diff --git a/.clang-format-ignore b/.clang-format-ignore new file mode 100644 index 0000000000000..a6c57f5fb2ffb --- /dev/null +++ b/.clang-format-ignore @@ -0,0 +1 @@ +*.json diff --git a/.cmake-format.py b/.cmake-format.py index 9827eecd329c4..ae092bc09f363 100644 --- a/.cmake-format.py +++ b/.cmake-format.py @@ -66,12 +66,6 @@ "HEADERS": '*', } }, - "o2_target_man_page": { - "kwargs": { - "NAME": '+', - "SECTION": '*', - } - }, "add_root_dictionary": { "kwargs": { "LINKDEF": '+', diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000000..30ad6d8f005b3 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +--- +# Dependabot configuration +# Reference: https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file + +version: 2 +updates: + - package-ecosystem: "github-actions" # See documentation for possible values + directory: "/" # Location of package manifests + schedule: + interval: "weekly" diff --git a/.github/workflows/async-auto-label.yml b/.github/workflows/async-auto-label.yml new file mode 100644 index 0000000000000..b0a17c7faba26 --- /dev/null +++ b/.github/workflows/async-auto-label.yml @@ -0,0 +1,17 @@ +--- +name: Apply requested async label + +'on': + issue_comment: + types: + - created + - edited + +permissions: {} + +jobs: + apply_async_labels: + name: Apply requested async label + uses: alisw/ali-bot/.github/workflows/async-auto-label.yml@master + permissions: + pull-requests: write # to update labels diff --git a/.github/workflows/async-list-label.yml b/.github/workflows/async-list-label.yml new file mode 100644 index 0000000000000..e0b4185c563b7 --- /dev/null +++ b/.github/workflows/async-list-label.yml @@ -0,0 +1,19 @@ +--- +name: Collect and print async labels + +'on': + pull_request_target: + types: + - opened + - reopened + branches: + - dev + +permissions: {} + +jobs: + list_async_labels: + name: Collect and print async labels + uses: alisw/ali-bot/.github/workflows/async-list-label.yml@master + permissions: + pull-requests: write # to update labels diff --git a/.github/workflows/clean-test.yml b/.github/workflows/clean-test.yml index de59435a9a34a..0f15301d4eed9 100644 --- a/.github/workflows/clean-test.yml +++ b/.github/workflows/clean-test.yml @@ -1,67 +1,62 @@ +--- name: Clean PR checks -on: + +'on': workflow_dispatch: inputs: pr: - description: PR to be cleaned - required: true - checks: - description: Checks to be cleaned - required: true - default: 'build/O2/o2,build/AliceO2/O2/o2/macOS,build/O2/fullCI,build/O2/o2-cs8,build/O2/o2-dataflow,build/O2/o2-dataflow-cs8' - owner: - description: Organization - required: true - default: 'AliceO2Group' - repo: - description: Repository + description: PR number in this repo to be cleaned + type: string # can't use number here required: true - default: 'AliceO2' + message: + description: Human-readable message displayed on the new pending status + type: string + required: false + default: '' + + # Warning: GitHub limits the total number of inputs to 10, so a maximum of + # 8 checks is allowed here! + # Warning: the check_* keys are magic and must consist of the string + # "check_" followed by the applicable check name exactly. The + # "description" field is only the human-readable label for the input. + 'check_build/AliceO2/O2/o2/macOS': + description: build/AliceO2/O2/o2/macOS + type: boolean + default: true + 'check_build/AliceO2/O2/o2/macOS-arm': + description: build/AliceO2/O2/o2/macOS-arm + type: boolean + default: true + 'check_build/O2/fullCI_slc9': + description: build/O2/fullCI + type: boolean + default: true + 'check_build/O2/o2-dataflow-cs8': + description: build/O2/o2-dataflow-cs8 + type: boolean + default: true + 'check_build/O2/o2/aarch64': + description: build/O2/o2/aarch64 + type: boolean + default: true + 'check_build/O2/o2_slc9': + description: build/O2/o2_slc9 + type: boolean + default: true + + +permissions: {} jobs: - cleanup_pr_checks: - runs-on: ubuntu-latest - steps: - - name: Set up Python 3.7 - uses: actions/setup-python@v1 - with: - python-version: 3.7 - - name: Install ali-bot - run: | - sudo apt-get update -y - sudo apt-get install -y libsasl2-dev python-dev libldap2-dev libssl-dev - python -m pip install --upgrade pip - pip install git+https://github.com/alisw/ali-bot@master - - uses: octokit/graphql-action@v2.x - id: get_last_commit_for_pr - with: - query: | - { - repository(owner: "${{ github.event.inputs.owner }}", name: "${{ github.event.inputs.repo }}") { - url - pullRequest(number:${{ github.event.inputs.pr }}) { - commits(last: 1) { - nodes { - commit { - oid - } - } - } - } - } - } - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cleanup tests - run: |- - set -x - cat <<\EOF > results.json - ${{ steps.get_last_commit_for_pr.outputs.data }} - EOF - COMMIT=$(jq -r '.repository.pullRequest.commits.nodes[].commit.oid' results.json) - echo $COMMIT - for check in `echo ${{ github.event.inputs.checks }} | tr , \\\\n`; do - set-github-status -c ${{ github.event.inputs.owner }}/${{ github.event.inputs.repo }}@$COMMIT -s $check/pending - done - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + clean: + name: Clean PR checks + uses: alisw/ali-bot/.github/workflows/clean-pr-checks.yml@master + with: + owner: ${{ github.event.repository.owner.login }} + repo: ${{ github.event.repository.name }} + pr: ${{ github.event.inputs.pr }} + message: ${{ github.event.inputs.message }} + checks: ${{ toJSON(github.event.inputs) }} + permissions: + pull-requests: read # to get last commit for pr (octokit/graphql-action) + statuses: write # for set-github-status diff --git a/.github/workflows/code-transformations.yml b/.github/workflows/code-transformations.yml index 7e8bdd11047c9..35493afda94f5 100644 --- a/.github/workflows/code-transformations.yml +++ b/.github/workflows/code-transformations.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v5 with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false @@ -44,7 +44,7 @@ jobs: COMMIT_FILES=$(git diff --diff-filter d --name-only $BASE_COMMIT) if [ -z "$COMMIT_FILES" ]; then echo "No files to check" >&2 - echo ::set-output name=clean::true + echo clean=true >> "$GITHUB_OUTPUT" exit 0 fi perl -p -i -e "${{ env.REFACTORING }}" $COMMIT_FILES @@ -52,13 +52,13 @@ jobs: if git diff --exit-code; then echo "Refactoring not needed." git push --set-upstream https://alibuild:$ALIBUILD_GITHUB_TOKEN@github.com/alibuild/AliceO2.git :alibot-refactoring-${{ github.event.pull_request.number }} -f || true - echo ::set-output name=clean::true + echo clean=true >> "$GITHUB_OUTPUT" else git commit -m "${{ env.RATIONALE }}" -a git show | cat git fetch https://github.com/AliceO2Group/AliceO2.git pull/${{ github.event.pull_request.number }}/head git push --set-upstream https://alibuild:$ALIBUILD_GITHUB_TOKEN@github.com/alibuild/AliceO2.git HEAD:refs/heads/alibot-refactoring-${{ github.event.pull_request.number }} -f - echo ::set-output name=clean::false + echo clean=false >> "$GITHUB_OUTPUT" fi - name: pull-request diff --git a/.github/workflows/datamodel-doc.yml b/.github/workflows/datamodel-doc.yml index 28a97407892b6..3ba015631aec6 100644 --- a/.github/workflows/datamodel-doc.yml +++ b/.github/workflows/datamodel-doc.yml @@ -10,20 +10,20 @@ jobs: steps: - name: Checkout O2 - uses: actions/checkout@v2 + uses: actions/checkout@v5 with: path: O2 persist-credentials: false - name: Checkout O2Physics - uses: actions/checkout@v2 + uses: actions/checkout@v5 with: repository: AliceO2Group/O2Physics path: O2Physics persist-credentials: false - name: Checkout documentation - uses: actions/checkout@v2 + uses: actions/checkout@v5 with: repository: AliceO2Group/analysis-framework path: analysis-framework @@ -40,7 +40,7 @@ jobs: git checkout -B auto-datamodel-doc - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v6 with: python-version: 3.x @@ -48,6 +48,7 @@ jobs: run: | python3 -m pip install --user -U numpy nltk python3 -m nltk.downloader -d ~/nltk_data punkt + python3 -m nltk.downloader -d ~/nltk_data punkt_tab - name: Generate documentation run: exec bash -exo pipefail O2/scripts/datamodel-doc/update-datamodel.sh @@ -59,6 +60,7 @@ jobs: run: | # git diff --quiet exits with 1 if any tracked files have changed, and # with 0 otherwise. + set -e if git diff --quiet; then exit # Nothing has changed, so no need to send a PR. fi @@ -68,9 +70,10 @@ jobs: git push -f origin auto-datamodel-doc # Send pull request - # We need to use "hub" ourselves because alisw/pull-request gets + # We need to use "gh" ourselves because alisw/pull-request gets # confused when multiple repos are checked out. - hub pull-request -b AliceO2Group:master -h alibuild:auto-datamodel-doc \ - --no-edit --no-maintainer-edits -m 'Automatic data model update' \ - -m "This update to the data model documentation was automatically created from tonight's O2 dev branch." || - : # If the PR already exists, hub fails, but we've just force-pushed, so we don't need a new PR. + GH_TOKEN="$GITHUB_TOKEN" gh pr create -R AliceO2Group/analysis-framework -B master \ + --no-maintainer-edit -t 'Automatic data model update' -b "This update \ + to the data model documentation was automatically created from \ + tonight's O2 dev branch." || true + # If the PR already exists, hub fails, but we've just force-pushed, so we don't need a new PR. diff --git a/.github/workflows/doxygen.yml b/.github/workflows/doxygen.yml index 38da67c793799..b1dbaf122b342 100644 --- a/.github/workflows/doxygen.yml +++ b/.github/workflows/doxygen.yml @@ -13,7 +13,7 @@ jobs: run: | sudo apt-get update -y sudo apt-get install -y doxygen doxygen-doc doxygen-latex doxygen-gui graphviz cmake - - uses: actions/checkout@v3 + - uses: actions/checkout@v5 with: ref: "dev" persist-credentials: false diff --git a/.github/workflows/first-timer.yml b/.github/workflows/first-timer.yml index 20b7ee6a070a8..54334d109bd49 100644 --- a/.github/workflows/first-timer.yml +++ b/.github/workflows/first-timer.yml @@ -8,7 +8,7 @@ jobs: nag_first_timer: runs-on: ubuntu-latest steps: - - uses: actions/first-interaction@v1 + - uses: actions/first-interaction@v3 with: repo-token: ${{ secrets.GITHUB_TOKEN }} pr-message: 'This seems to be your first PR. You will need a positive review in order for tests to start.' diff --git a/.github/workflows/pr-security-approval.yml b/.github/workflows/pr-security-approval.yml new file mode 100644 index 0000000000000..9ae70fdb9933d --- /dev/null +++ b/.github/workflows/pr-security-approval.yml @@ -0,0 +1,18 @@ +--- +name: Security approval + +'on': + pull_request_review: + types: + - edited + - submitted + +permissions: {} + +jobs: + clean: + name: Security approval + uses: alisw/ali-bot/.github/workflows/pr-security-approval.yml@master + permissions: + pull-requests: read # to get last commit for PR + statuses: write # for set-github-status diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bd3afeb09f947..2f692527ea5ce 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,11 +14,11 @@ jobs: steps: - name: Decide which branch to use run: | - cat << EOF - ::set-output name=branch::$(echo ${{ github.event.inputs.tag }}-patches | tr . - | sed -e's/-[0-9]*-patches$/-patches/') + cat << EOF >> "$GITHUB_OUTPUT" + branch=$(echo ${{ github.event.inputs.tag }}-patches | tr . - | sed -e's/-[0-9]*-patches$/-patches/') EOF id: decide_release_branch - - uses: actions/checkout@v2 + - uses: actions/checkout@v5 with: ref: "dev" - name: Tag branch (or create one before tagging if does not exists) diff --git a/.github/workflows/reports.yml b/.github/workflows/reports.yml index bf7347464476e..5a04e56382fb3 100644 --- a/.github/workflows/reports.yml +++ b/.github/workflows/reports.yml @@ -17,12 +17,12 @@ jobs: if: github.repository == 'AliceO2Group/AliceO2' steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.7 - uses: actions/setup-python@v1 + - uses: actions/checkout@v5 + - name: Set up Python 3.10 + uses: actions/setup-python@v6 with: - python-version: 3.7 - - uses: actions/cache@v2 + python-version: '3.10' + - uses: actions/cache@v5 name: Configure pip caching with: path: ~/.cache/pip @@ -81,7 +81,7 @@ jobs: run: | set -x mkdir -p doc/data - # We create new files once per month, mostly so that + # We create new files once per month, mostly so that # we can keep the query results small. It does not # matter if we get results from different months, # as what matters is how we merge them. @@ -96,7 +96,6 @@ jobs: # being published LAST_RELEASE="${{ github.event.inputs.LAST_RELEASE_DATE }}" MERGED_AFTER=${LAST_RELEASE:-$(date -v -14d +%Y-%m-%d)} - # Here we convert all the json files to per subsystem # logs, using the MERGED_AFTER date to further filter them. # Notice we can have duplicates in each file, @@ -106,7 +105,7 @@ jobs: for f in doc/data/*_prs.json; do for x in Algorithm Analysis Common DataFormats Detectors EventVisualisation Examples Framework Generators Steer Testing Utilities; do cat $f | jq ".repository.pullRequests.edges[].node | select(.files.edges[].node.path | test(\"$x\")) | del(.files) | select(.state == \"MERGED\" and .mergedAt >= \"${MERGED_AFTER}\")" > /tmp/${x}_prs.json - if [ ! X`jq -s length /tmp/${x}_prs.json` = X0 ]; then + if [ ! X`jq -s length /tmp/${x}_prs.json` = X0 ]; then cat /tmp/${x}_prs.json | jq -r '"- [#\(.number)](https://github.com/AliceO2Group/AliceO2/pull/\(.number)) \(.mergedAt | split("T")[0]): \(.title) by [@\(.author.login)](https://github.com/\(.author.login))"' | sort -u >> /tmp/${x}_prs.md fi done @@ -127,11 +126,6 @@ jobs: git config --global user.name "GitHub Action Bot" git commit -m "Updated README" -a || echo "No changes to commit" git push origin HEAD:changelog -f - GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} \ - hub pull-request -f -b dev -h changelog \ - --no-edit --no-maintainer-edits \ - 'Auto-generated changelog' \ - -m 'The following changelog has been automatically generated.' || - # If the PR already exists, the force-push will have updated it. - # It's fine if this step fails. - true + # If the PR already exists, the force-push will have updated it. + # It's fine if this step fails. + GH_TOKEN=${{ secrets.GITHUB_TOKEN }} gh pr create -B dev -H changelog -t 'Auto-generated changelog' -b 'The following changelog has been automatically generated.' || true diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 1f1387d4868ae..23f454aaca950 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -7,7 +7,7 @@ jobs: stale: runs-on: ubuntu-latest steps: - - uses: actions/stale@v1 + - uses: actions/stale@v10 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-pr-message: 'This PR did not have any update in the last 30 days. Is it still needed? Unless further action in will be closed in 5 days.' diff --git a/.gitignore b/.gitignore index dd3f634b9319b..d58d1e151800b 100644 --- a/.gitignore +++ b/.gitignore @@ -62,6 +62,8 @@ compile_commands.json .settings .vscode .ycm_extra_conf.py +Session.vim +CMakeLists.txt.user # Datafiles gphysi.dat @@ -80,6 +82,9 @@ bazel-* # direnv .envrc +# git wrappers +.sl + # LSP support on macOS with vim .clangd DataFormats/Detectors/CTP/include/DataFormatsCTP/Scalers.h diff --git a/Algorithm/CMakeLists.txt b/Algorithm/CMakeLists.txt index b245562c7cc93..ed7a42a96e528 100644 --- a/Algorithm/CMakeLists.txt +++ b/Algorithm/CMakeLists.txt @@ -11,9 +11,6 @@ o2_add_header_only_library(Algorithm INTERFACE_LINK_LIBRARIES O2::Headers) -o2_target_man_page(Algorithm NAME Algorithm SECTION 3) -o2_target_man_page(Algorithm NAME algorithm_parser SECTION 3) - o2_add_test(o2formatparser SOURCES test/o2formatparser.cxx COMPONENT_NAME Algorithm diff --git a/Algorithm/doc/Algorithm.3.in b/Algorithm/doc/Algorithm.3.in deleted file mode 100644 index eaf618ee68da2..0000000000000 --- a/Algorithm/doc/Algorithm.3.in +++ /dev/null @@ -1,12 +0,0 @@ -.\" Alice O2 manpage for module Algorithm -.TH "AliceO2" 3 "17 Jan 2017" "1.0" "Algorithm man page" - -.SH NAME -AliceO2 - module -.B Algorithm - -.SH DESCRIPTION -A collection of generic algorithms for Alice O2 - -.SH SEE ALSO -algorithm_parser(3) diff --git a/Algorithm/doc/algorithm_parser.3.in b/Algorithm/doc/algorithm_parser.3.in deleted file mode 100644 index 98f45df279669..0000000000000 --- a/Algorithm/doc/algorithm_parser.3.in +++ /dev/null @@ -1,135 +0,0 @@ -.\" Alice O2 manpage for parser algorithms -.TH "AliceO2" 3 "17 Jan 2017" "1.0" "Algorithm Parser man page" - -.SH NAME -AliceO2 - module -.B Algorithm -- data parsers - -.SH SYNOPSIS -.B ForwardParser< -.I SomeHeaderType -, -.I SomeTrailerType -.B > - -.B ReverseParser< -.I SomeHeaderType -, -.I SomeTrailerType -.B > - -.SS Public types -.TP 2 -// a compound of header, data, and trailer -.B struct FrameInfo { - using PtrT = const PayloadType*; - const HeaderType* header = nullptr; - const TrailerType* trailer = nullptr; - PtrT payload = nullptr; - size_t length = 0; - -.B }; - -.TP 2 -.B using CheckHeaderFct = std::function; -alias for callback checking the header, return true if the object is a valid header -.TP 2 -.B using CheckTrailerFct = std::function; -alias for callback checking the trailer -.TP 2 -.B using GetFrameSizeFct = std::function; -alias for callback to get the complete frame size including header, trailer and the data -.TP 2 -.B using InsertFct = std::function; -function callback to insert/handle one frame into, sequentially called for all frames if the whole block has a valid format - -.SS Public member functions -.TP 2 -.B template -.B int parse(const InputType* \fIbuffer\fB, size_t \fIbufferSize\fB, CheckHeaderFct \fIcheckHeader\fB, CheckTrailerFct \fIcheckTrailer\fB, GetFrameSizeFct \fIgetFrameSize\fB, InsertFct \fIinsert\fB) - -.SS Public member variables -.TP 2 -.B static const size_t headOffset = typesize::size; -the length offset due to header -.TP 2 -.B static const size_t tailOffset = typesize::size; -the length offset due to trailer -.TP 2 -.B static const size_t totalOffset = headOffset + tailOffset; -total length offset due to header and trailer - -.SH DESCRIPTION -Template utilities for parsing of data sequences. Each entry in the sequence consist of a header, variable payload, and optionally a trailer. The three parts are collected in the FrameInfo structure for every entry. - -Callback functions for checking header and trailer integrity, getting length of the current frame and handling of a frame. - -.SS ForwardParser -The size is expected to be part of the header, parsing starts at beginning of buffer. -Trailer type can be void, which is also the default template parameter. That -allows to define a frame consisting of only header and data. - -.SS ReverseParser -The size is expected to be part of the trailer, the parsing is thus in reverse direction. Also the insert callback is called with the entries starting form the end of the buffer. -An easy extension can be to reverse the order of the inserts, meaning that the entries are read from the beginning. - -.SH EXAMPLES -.SS ReverseParser example -.EX -using SomeParser = ReverseParser; -SomeParser parser; -std::vector frames; -parser.parse(ptr, size, - [] (const typename SomeParser::HeaderType& h) { - // check the header - return true; - }, - [] (const typename SomeParser::TrailerType& t) { - // check the trailer - return true; - }, - [] (const typename SomeParser::TrailerType& t) { - // get the size of the frame including payload - // and header and trailer size, e.g. payload size - // from a trailer member - return t.payloadSize + SomeParser::totalOffset; - }, - [&frames] (typename SomeParser::FrameInfo& info) { - frames.emplace_back(info); - return true; - } - ) -.EE - -.SS ForwardParser example with frame consisting of header and payload -.EX -using SomeParser = ForwardParser; -SomeParser parser; -std::vector frames; -parser.parse(ptr, size, - [] (const typename SomeParser::HeaderType& h) { - // check the header - return true; - }, - [] (const typename SomeParser::HeaderType& h) { - // get the size of the frame including payload - // and header and trailer size, e.g. payload size - // from a header member - return h.payloadSize + SomeParser::totalOffset; - }, - [&frames] (typename SomeParser::FrameInfo& info) { - frames.emplace_back(info); - return true; - } - ) -.EE - -.SH BUGS, CONTRIBUTIONS -Please add an issue to -.UR https://github.com/AliceO2Group/AliceO2/issues -.UE - -.SH SEE ALSO -.UR https://github.com/AliceO2Group/AliceO2/blob/dev/Algorithm/include/Algorithm/Parser.h -.UE diff --git a/Algorithm/include/Algorithm/BitstreamReader.h b/Algorithm/include/Algorithm/BitstreamReader.h deleted file mode 100644 index 0a112183ab5ef..0000000000000 --- a/Algorithm/include/Algorithm/BitstreamReader.h +++ /dev/null @@ -1,290 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -#ifndef BITSTREAMREADER_H -#define BITSTREAMREADER_H - -/// @file BitstreamReader.h -/// @author Matthias Richter -/// @since 2019-06-05 -/// @brief Utility class to provide bitstream access to an underlying resource - -#include -#include - -namespace o2 -{ -namespace algorithm -{ - -/// @class BitStreamReader -/// @brief Utility class to provide bitstream access to an underlying resource -/// -/// Allows to access bits of variable length, supports integral types and also -/// bitsets as target type. At the moment, the access is in direction MSB -> LSB. -/// -/// BitstreamReader reader(start, end); -/// while (reader.good() && not reader.eof()) { -/// // get an 8 bit value from the stream, moves the position -/// uint8_t ivalue; -/// reader.get(ivalue); -/// -/// // get a 13 bit bitset without moving the position -/// std::bitset<13> value; -/// reader.peek(value, value.size()); -/// // e.g. use 7 bits of the data -/// value >>= value.size() - 7; -/// // move position by the specific number of bits -/// reader.seek(7); -/// } -template -class BitstreamReader -{ - public: - using self_type = BitstreamReader; - // for the moment we simply use pointers, but with some traits this can be extended to - // containers - using value_type = BufferType; - using iterator = const value_type*; - static constexpr size_t value_size = sizeof(value_type) * 8; - BitstreamReader() = delete; - BitstreamReader(iterator start, iterator end) - : mStart(start), mEnd(end), mCurrent(mStart), mBitPosition(value_size) - { - } - ~BitstreamReader() = default; - - /// Check reader's state - /// @return true if not in error state - bool good() const - { - return mBitPosition > 0; - } - - /// Indicates end of data - /// @return true if end of resource is reached - bool eof() const - { - return mCurrent == mEnd && mBitPosition > 0; - } - - /// Reset the reader, start over at beginning - void reset() - { - mCurrent = mStart; - mBitPosition = value_size; - } - - /// Get the next N bits without moving the read position - /// if bitlength is smaller than the size of data type, result is aligned to LSB - /// TODO: this also works nicely for bitsets, but then the bitlength has to be specified - /// as template parameter, want to do a specific overload, but needs more work to catch - /// all cases. - /// @param v target variable passed by reference - /// @return number of poked bits - template - size_t peek(T& v) - { - static_assert(N <= sizeof(T) * 8); - return peek(v, N); - } - - /// Get the next n bits without moving the read position - /// if bitlength is smaller than the size of data type, result is aligned to LSB - /// @param v target variable passed by reference - /// @param bitlength number of bits to read - /// @return number of poked bits - template - size_t peek(T& v, size_t bitlength) - { - return peek(v, bitlength); - } - - /// Move read position - /// @param bitlength move count in number of bits - void seek(size_t bitlength) - { - while (good() && bitlength > 0 && mCurrent != mEnd) { - if (bitlength >= mBitPosition) { - bitlength -= mBitPosition; - mBitPosition = 0; - } else { - mBitPosition -= bitlength; - bitlength = 0; - } - if (mBitPosition == 0) { - mCurrent++; - mBitPosition = value_size; - } - } - - if (bitlength > 0) { - mBitPosition = 0; - } - } - - /// Get the next n bits and move the read position - template - T get() - { - T result; - peek(result); - seek(N); - return result; - } - - /// Get the next n and move the read position - template - T get(size_t bitlength = sizeof(T) * 8) - { - T result; - peek(result, bitlength); - seek(bitlength); - return result; - } - - /// @class Bits - /// @brief Helper class to get value of specified type which holds the number used bits - /// - /// The class holds both the extracted value access via peek method and the number of used - /// bits. The reader will be incremented when the object is destroyed. - /// The number of bits can be adjusted by using markUsed method - template - class Bits - { - public: - using field_type = FieldType; - static_assert(N <= sizeof(FieldType) * 8); - Bits() - : mParent(nullptr), mData(0), mLength(0) - { - } - Bits(ParentType* parent, FieldType&& data) - : mParent(parent), mData(std::move(data)), mLength(N) - { - } - Bits(Bits&& other) - : mParent(other.mParent), mData(std::move(other.mData)), mLength(other.mLength) - { - other.mParent = nullptr; - other.mLength = 0; - } - - ~Bits() - { - if (mParent) { - mParent->seek(mLength); - } - } - - auto& operator=(Bits&& other) - { - mParent = other.mParent; - mData = std::move(other.mData); - mLength = other.mLength; - other.mParent = nullptr; - other.mLength = 0; - - return *this; - } - - FieldType& operator*() - { - return mData; - } - - void markUsed(size_t length) - { - mLength = length; - } - - private: - ParentType* mParent; - FieldType mData; - size_t mLength; - }; - - /// Read an integral value from the stream - template ::value, int> = 0> - self_type& operator>>(T& target) - { - target = get(); - return *this; - } - - /// Read a bitstream value from the stream - template - self_type& operator>>(std::bitset& target) - { - target = get, N>(); - return *this; - } - - /// Read a Bits object from the stream - template - self_type& operator>>(Bits& target) - { - T bitfield; - peek(bitfield); - target = std::move(Bits(this, std::move(bitfield))); - return *this; - } - - private: - /// The internal peek method - template - size_t peek(T& result, size_t bitlength) - { - if constexpr (RuntimeCheck) { - // the runtime check is disabled if bitlength is derived at compile time - if (bitlength > sizeof(T) * 8) { - throw std::length_error(std::string("requested bit length ") + std::to_string(bitlength) + " does not fit size of result data type " + std::to_string(sizeof(T) * 8)); - } - } - result = 0; - size_t bitsToWrite = bitlength; - auto current = mCurrent; - auto bitsAvailable = mBitPosition; - while (bitsToWrite > 0 && current != mEnd) { - // extract available bits - value_type mask = ~value_type(0) >> (value_size - bitsAvailable); - if (bitsToWrite >= bitsAvailable) { - T value = (*current & mask) << (bitsToWrite - bitsAvailable); - result |= value; - bitsToWrite -= bitsAvailable; - bitsAvailable = 0; - } else { - value_type value = (*current & mask) >> (bitsAvailable - bitsToWrite); - result |= value; - bitsAvailable -= bitsToWrite; - bitsToWrite = 0; - } - if (bitsAvailable == 0) { - current++; - bitsAvailable = value_size; - } - } - - return bitlength - bitsToWrite; - } - - /// start of resource - iterator mStart; - /// end of resource - iterator mEnd; - /// current position in resource - iterator mCurrent; - /// bit position in current element - size_t mBitPosition; -}; -} // namespace algorithm -} // namespace o2 -#endif diff --git a/Algorithm/include/Algorithm/PageParser.h b/Algorithm/include/Algorithm/PageParser.h index e382fc318352e..3ca01d87bcba3 100644 --- a/Algorithm/include/Algorithm/PageParser.h +++ b/Algorithm/include/Algorithm/PageParser.h @@ -255,12 +255,12 @@ class PageParser return mElement; } // comparison - bool operator==(const SelfType& rh) + bool operator==(const SelfType& rh) const { return mPosition == rh.mPosition; } // comparison - bool operator!=(const SelfType& rh) + bool operator!=(const SelfType& rh) const { return mPosition != rh.mPosition; } diff --git a/Algorithm/test/pageparser.cxx b/Algorithm/test/pageparser.cxx index 14b24c670cfd6..7551c32d9d864 100644 --- a/Algorithm/test/pageparser.cxx +++ b/Algorithm/test/pageparser.cxx @@ -50,7 +50,7 @@ struct ClusterData { { } - bool operator==(const ClusterData& rhs) + bool operator==(const ClusterData& rhs) const { return clusterid == rhs.clusterid && x == rhs.x && y == rhs.y && z == rhs.z && e == rhs.e; } diff --git a/Algorithm/test/test_BitstreamReader.cxx b/Algorithm/test/test_BitstreamReader.cxx deleted file mode 100644 index 41e3b47f5f276..0000000000000 --- a/Algorithm/test/test_BitstreamReader.cxx +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// @file test_BitstreamReader.cxx -/// @author Matthias Richter -/// @since 2019-06-05 -/// @brief Test program for BitstreamReader utility - -#define BOOST_TEST_MODULE Algorithm BitstreamReader unit test -#define BOOST_TEST_MAIN -#define BOOST_TEST_DYN_LINK -#include -#include -#include -#include -#include -#include -#include "../include/Algorithm/BitstreamReader.h" - -namespace o2 -{ -namespace algorithm -{ - -BOOST_AUTO_TEST_CASE(test_BitstreamReader_basic) -{ - std::array data = {'d', 'e', 'a', 'd', 'b', 'e', 'e', 'f'}; - std::array expected7bit = {0x32, 0x19, 0x2c, 0x16, 0x23, 0x09, 0x4a, 0x65, 0x33, 0x0}; - auto reference = expected7bit.begin(); - constexpr size_t totalBits = data.size() * sizeof(decltype(data)::value_type) * 8; - size_t bitsRead = 0; - - BitstreamReader reader(data.data(), data.data() + data.size()); - while (bitsRead < totalBits) { - BOOST_REQUIRE(reference != expected7bit.end()); - BOOST_CHECK(reader.eof() == false); - uint8_t value; - reader.peek(value); - // we use 7 bits of the data - value >>= 1; - reader.seek(7); - bitsRead += 7; - // in the last call should there is not enough data - BOOST_CHECK(reader.good() == (bitsRead <= totalBits)); - BOOST_REQUIRE(reference != expected7bit.end()); - //std::cout << "value " << (int)value << " expected " << (int)*reference << std::endl; - BOOST_CHECK(value == *reference); - ++reference; - } -} - -BOOST_AUTO_TEST_CASE(test_BitstreamReader_operator) -{ - std::array data = {'d', 'e', 'a', 'd', 'b', 'e', 'e', 'f'}; - std::array expected7bit = {0x32, 0x19, 0x2c, 0x16, 0x23, 0x09, 0x4a, 0x65, 0x33, 0x0}; - auto reference = expected7bit.begin(); - constexpr size_t totalBits = data.size() * sizeof(decltype(data)::value_type) * 8; - size_t bitsRead = 0; - - BitstreamReader reader(data.data(), data.data() + data.size()); - while (bitsRead < totalBits) { - BOOST_REQUIRE(reference != expected7bit.end()); - BOOST_CHECK(reader.eof() == false); - { - decltype(reader)::Bits value; - reader >> value; - // we use 7 bits of the data - *value >>= 1; - value.markUsed(7); - //std::cout << "value " << (int)*value << " expected " << (int)*reference << std::endl; - BOOST_CHECK(*value == *reference); - } - bitsRead += 7; - // in the last call should there is not enough data - BOOST_CHECK(reader.good() == (bitsRead <= totalBits)); - BOOST_REQUIRE(reference != expected7bit.end()); - ++reference; - } -} - -BOOST_AUTO_TEST_CASE(test_BitstreamReader_bitset) -{ - std::array data = {'d', 'e', 'a', 'd', 'b', 'e', 'e', 'f'}; - std::array expected7bit = {0x32, 0x19, 0x2c, 0x16, 0x23, 0x09, 0x4a, 0x65, 0x33, 0x0}; - auto reference = expected7bit.begin(); - constexpr size_t totalBits = data.size() * sizeof(decltype(data)::value_type) * 8; - size_t bitsRead = 0; - - BitstreamReader reader(data.data(), data.data() + data.size()); - while (bitsRead < totalBits) { - BOOST_REQUIRE(reference != expected7bit.end()); - BOOST_CHECK(reader.eof() == false); - std::bitset<13> value; - reader.peek(value, value.size()); - // we use 7 bits of the data - value >>= value.size() - 7; - reader.seek(7); - bitsRead += 7; - // in the last call should there is not enough data - BOOST_CHECK(reader.good() == (bitsRead <= totalBits)); - BOOST_REQUIRE(reference != expected7bit.end()); - BOOST_CHECK_MESSAGE(value.to_ulong() == *reference, std::string("mismatch: value ") << value.to_ulong() << ", expected " << (int)*reference); - ++reference; - } - - reader.reset(); - std::bitset<16> aBitset; - reader >> aBitset; - BOOST_CHECK_MESSAGE(aBitset.to_ulong() == 0x6465, std::string("mismatch: value 0x") << std::hex << aBitset.to_ulong() << ", expected 0x6465"); -} - -} // namespace algorithm -} // namespace o2 diff --git a/CCDB/CMakeLists.txt b/CCDB/CMakeLists.txt index e1fdc0a8b9484..691c3311e117c 100644 --- a/CCDB/CMakeLists.txt +++ b/CCDB/CMakeLists.txt @@ -11,15 +11,17 @@ o2_add_library(CCDB SOURCES src/CcdbApi.cxx + src/CCDBDownloader.cxx src/BasicCCDBManager.cxx src/CCDBTimeStampUtils.cxx src/IdPath.cxx src/CCDBQuery.cxx PUBLIC_LINK_LIBRARIES CURL::libcurl - FairRoot::ParMQ ROOT::Hist O2::CommonUtils FairMQ::FairMQ libjalien::libjalienO2 + LibUV::LibUV + O2::Framework TARGETVARNAME targetName) o2_target_root_dictionary(CCDB @@ -30,7 +32,8 @@ o2_target_root_dictionary(CCDB include/CCDB/IdPath.h include/CCDB/BasicCCDBManager.h include/CCDB/CCDBTimeStampUtils.h - include/CCDB/CCDBQuery.h) + include/CCDB/CCDBQuery.h + include/CCDB/CCDBDownloader.h) o2_add_executable(inspectccdbfile COMPONENT_NAME ccdb @@ -42,6 +45,11 @@ o2_add_executable(upload SOURCES src/UploadTool.cxx PUBLIC_LINK_LIBRARIES O2::CCDB) +o2_add_executable(cleansemaphores + COMPONENT_NAME ccdb + SOURCES src/CleanCCDBSemaphores.cxx + PUBLIC_LINK_LIBRARIES O2::CCDB) + o2_add_executable(downloadccdbfile COMPONENT_NAME ccdb SOURCES src/DownloadCCDBFile.cxx @@ -77,3 +85,21 @@ o2_add_test(CcdbApiMultipleUrls COMPONENT_NAME ccdb PUBLIC_LINK_LIBRARIES O2::CCDB LABELS ccdb) + +o2_add_test(CcdbDownloader + SOURCES test/testCcdbApiDownloader.cxx + COMPONENT_NAME ccdb + PUBLIC_LINK_LIBRARIES O2::CCDB + LABELS ccdb) + +o2_add_test(CcdbApi-Headers + SOURCES test/testCcdbApiHeaders.cxx + COMPONENT_NAME ccdb + PUBLIC_LINK_LIBRARIES O2::CCDB + LABELS ccdb) + +# extra CcdbApi test which dispatches to CCDBDownloader (tmp until full move done) +#o2_add_test_command(NAME CcdbApi-MultiHandle +# WORKING_DIRECTORY ${SIMTESTDIR} +# COMMAND ${CMAKE_BINARY_DIR}/stage/tests/o2-test-ccdb-CcdbApi +# ENVIRONMENT "ALICEO2_ENABLE_MULTIHANDLE_CCDBAPI=1") diff --git a/CCDB/README.md b/CCDB/README.md index ce8d9e19f7b27..1ae5f29dcf0e2 100644 --- a/CCDB/README.md +++ b/CCDB/README.md @@ -13,7 +13,7 @@ in circumstances of reduced or no network connectivity. There are currently 2 different kinds of store/retrieve functions, which we expect to unify in the immediate future: 2. `storeAsTFile/retrieveFromTFile` API serializing a `TObject` in a ROOT `TFile`. -3. A strongly-typed `storeAsTFileAny/retrieveFromTFileAny` API allowing to handle any type T +3. A strongly-typed `storeAsTFileAny/retrieveFromTFileAny` API allowing to handle any type T having a ROOT dictionary. We encourage to use this API by default. ## Central and local instances of the CCDB @@ -31,18 +31,18 @@ If you access the CCDB with a web browser, add `/browse` at the end of the URL t ```c++ // init CcdbApi api; -map metadata; // can be empty +std::map metadata; // can be empty api.init("http://ccdb-test.cern.ch:8080"); // or http://localhost:8080 for a local installation // store abitrary user object in strongly typed manner auto deadpixels = new o2::FOO::DeadPixelMap(); api.storeAsTFileAny(deadpixels, "FOO/DeadPixels", metadata); // read like this (you have to specify the type) -auto deadpixelsback = api.retrieveFromTFileAny("FOO/DeadPixels", metadata); -// read like this to get the headers as well, and thus the metadata attached to the object -map headers; -auto deadpixelsback = api.retrieveFromTFileAny("FOO/DeadPixels", metadata /* constraint the objects retrieved to those matching the metadata */, -1 /* timestamp */, &headers /* the headers attached to the returned object */); +auto deadpixelsback = api.retrieveFromTFileAny("FOO/DeadPixels", metadata); +// read like this to get the headers as well, and thus the metadata attached to the object +std::map headers; +auto deadpixelsback = api.retrieveFromTFileAny("FOO/DeadPixels", metadata /* constraint the objects retrieved to those matching the metadata */, -1 /* timestamp */, &headers /* the headers attached to the returned object */); // finally, use this method to retrieve only the headers (and thus the metadata) -std::map headers = f.api.retrieveHeaders("FOO/DeadPixels", f.metadata); +std::map headers = api.retrieveHeaders("FOO/DeadPixels", metadata); ``` * creating a local snapshot and fetching objects therefrom @@ -50,7 +50,7 @@ std::map headers = f.api.retrieveHeaders("FOO/DeadPixe ```c++ // init CcdbApi api; -map metadata; // can be empty +std::map metadata; // can be empty api.init("http://ccdb-test.cern.ch:8080"); // or http://localhost:8080 for a local installation // create a local snapshot of everthing in or below the FOO folder valid for timestamp 12345 api.snapshot("FOO", "/tmp/CCDBSnapshot/", 12345); @@ -85,7 +85,7 @@ user code. This class The class was written for the use-case of transport MC simulation. Typical usage should be like ```c++ -// setup manager once (at start of processing) +// setup manager once (at start of processing) auto& mgr = o2::ccdb::BasicCCDBManager::instance(); mgr.setURL("http://ourccdbserverver.cern.ch"); mgr.setTimestamp(timestamp_which_we_want_to_anchor_to); @@ -111,6 +111,12 @@ This feature is useful to avoid using newer objects if the CCDB is updated in pa In cached mode, the manager can check that local objects are still valid by requiring `mgr.setLocalObjectValidityChecking(true)`, in this case a CCDB query is performed only if the cached object is no longer valid. +If you want the headers/metadata for the object retrieved from the CCDB there is an optional paramater to `BasicCCDBManager::getForTimeStamp`. These headers are also cached (when caching is enabled) and is updated when a CCDB query is sent. +```c++ +std::map headers; +mgr.getForTimeStamp(path, timstamp, metadata, &headers); +``` + ## Future ideas / todo: - [ ] offer improved error handling / exceptions @@ -129,26 +135,26 @@ A few prototypic command line tools are offered. These can be used in scriptable and facilitate the following tasks: 1. Upload and annotate a generic C++ object serialized in a ROOT file - + ```bash o2-ccdb-upload -f myRootFile.root --key histogram1 --path /Detector1/QA/ --meta "Description=Foo;Author=Person1;Uploader=Person2" ``` This will upload the object serialized in `myRootFile.root` under the key `histogram1`. Object will be put to the CCDB path `/Detector1/QA`. For full list of options see `o2-ccdb-upload --help`. - + 2. Download a CCDB object to a local ROOT file (including its meta information) - + ```bash o2-ccdb-downloadccdbfile --path /Detector1/QA/ --dest /tmp/CCDB --timestamp xxx ``` This will download the CCDB object under path given by `--path` to a directory given by `--dest` on the disc. (The final filename will be `/tmp/CCDB/Detector1/QA/snapshot.root` for the moment). All meta-information as well as the information associated to this query will be appended to the file. - + For full list of options see `o2-ccdb-downloadccdbfile --help`. - + 3. Inspect the content of a ROOT file and print summary about type of contained (CCDB) objects and its meta information - + ```bash o2-ccdb-inspectccdbfile filename ``` diff --git a/CCDB/include/CCDB/BasicCCDBManager.h b/CCDB/include/CCDB/BasicCCDBManager.h index f6e5bb299cc11..fd0fe7aa6d05b 100644 --- a/CCDB/include/CCDB/BasicCCDBManager.h +++ b/CCDB/include/CCDB/BasicCCDBManager.h @@ -18,10 +18,17 @@ #include "CCDB/CcdbApi.h" #include "CCDB/CCDBTimeStampUtils.h" #include "CommonUtils/NameConf.h" +#include "Framework/DataTakingContext.h" +#include "Framework/DefaultsHelpers.h" +#include "Framework/ServiceRegistryRef.h" +#include "Framework/DataProcessingStats.h" #include +#include #include +#include #include #include +#include class TGeoManager; // we need to forward-declare those classes which should not be cleaned up @@ -46,7 +53,20 @@ class CCDBManagerInstance std::string uuid; long startvalidity = 0; long endvalidity = -1; - bool isValid(long ts) { return ts < endvalidity && ts > startvalidity; } + long cacheValidFrom = 0; // time for which the object was cached + long cacheValidUntil = -1; // object is guaranteed to be valid till this time (modulo new updates) + size_t minSize = -1ULL; + size_t maxSize = 0; + int queries = 0; + int fetches = 0; + int failures = 0; + std::map cacheOfHeaders; + bool isValid(long ts) { return ts < endvalidity && ts >= startvalidity; } + bool isCacheValid(long ts) + { + LOGP(debug, "isCacheValid : {} : {} : {} --> {}", cacheValidFrom, ts, cacheValidUntil, ts < cacheValidUntil && ts >= cacheValidFrom); + return ts < cacheValidUntil && ts >= cacheValidFrom; + } void clear() { noCleanupPtr = nullptr; @@ -54,6 +74,7 @@ class CCDBManagerInstance uuid = ""; startvalidity = 0; endvalidity = -1; + cacheOfHeaders.clear(); } }; @@ -63,6 +84,7 @@ class CCDBManagerInstance CCDBManagerInstance(std::string const& path) : mCCDBAccessor{} { mCCDBAccessor.init(path); + mDeplMode = o2::framework::DefaultsHelpers::deploymentMode(); } /// set a URL to query from void setURL(const std::string& url); @@ -81,19 +103,31 @@ class CCDBManagerInstance /// query timestamp long getTimestamp() const { return mTimestamp; } - /// retrieve an object of type T from CCDB as stored under path and timestamp + /// retrieve an object of type T from CCDB as stored under path and timestamp. Optional to get the headers. + template + T* getForTimeStamp(std::string const& path, long timestamp, std::map* headers = nullptr); + + /// retrieve an object of type T from CCDB as stored under path and using the timestamp in the middle of the run template - T* getForTimeStamp(std::string const& path, long timestamp); + T* getForRun(std::string const& path, int runNumber, bool setRunMetadata = false); /// retrieve an object of type T from CCDB as stored under path, timestamp and metaData template - T* getSpecific(std::string const& path, long timestamp = -1, MD metaData = MD()) + T* getSpecific(std::string const& path, long timestamp = -1, MD metaData = MD(), std::map* headers = nullptr) { // TODO: add some error info/handling when failing mMetaData = metaData; - return getForTimeStamp(path, timestamp); + auto obj = getForTimeStamp(path, timestamp, headers); + return obj; } + /// retrieve an object of type T from CCDB as stored under path and using the timestamp in the middle of the run + metadata. The run number is provided separately to conform to typical analysis use (in which case metadata does not include runNumber) + template + T* getSpecificForRun(std::string const& path, int runNumber, MD const& metaData = MD()); + + /// detect online processing modes (i.e. CCDB objects may be updated in the lifetime of the manager) + bool isOnline() const { return mDeplMode == o2::framework::DeploymentMode::OnlineAUX || mDeplMode == o2::framework::DeploymentMode::OnlineDDS || mDeplMode == o2::framework::DeploymentMode::OnlineECS; } + /// retrieve an object of type T from CCDB as stored under path; will use the timestamp member template T* get(std::string const& path) @@ -101,6 +135,9 @@ class CCDBManagerInstance return getForTimeStamp(path, mTimestamp); } + // gain access to underlaying CCDB layer (to allow for more complex queries without need to reinit another API) + CcdbApi& getCCDBAccessor() { return mCCDBAccessor; } + bool isHostReachable() const { return mCCDBAccessor.isHostReachable(); } /// clear all entries in the cache @@ -127,7 +164,7 @@ class CCDBManagerInstance if (!isCachingEnabled()) { return false; } - return mCache[path].isValid(timestamp); + return (mCheckObjValidityEnabled && mCache[path].isValid(timestamp)) || mCache[path].isCacheValid(timestamp); // use stricter check } /// check if checks of object validity before CCDB query is enabled @@ -159,9 +196,21 @@ class CCDBManagerInstance /// set the fatal property (when false; nullptr object responses will not abort) void setFatalWhenNull(bool b) { mFatalWhenNull = b; } - /// a convenience function for MC to fetch - /// valid timestamps given an ALICE run number - std::pair getRunDuration(int runnumber) const; + /// A convenience function for MC to fetch + /// valid start and end timestamps for recorded TF data given an ALICE run number. + /// In absence of STF/ETF fields in the RCT with fall back to CTP SOX/EOX then to + /// ECS SOR/EOR. + /// On error it fatals (if fatal == true) or else returns the pair -1, -1. + std::pair getRunDuration(int runnumber, bool fatal = true); + static std::pair getRunDuration(o2::ccdb::CcdbApi const& api, int runnumber, bool fatal = true); + static std::pair getRunDuration(const MD& headers); + std::string getSummaryString() const; + + size_t getFetchedSize() const { return mFetchedSize; } + + void report(bool longrep = false); + + void endOfStream(); private: // method to print (fatal) error @@ -175,56 +224,155 @@ class CCDBManagerInstance bool mCanDefault = false; // whether default is ok --> useful for testing purposes done standalone/isolation bool mCachingEnabled = true; // whether caching is enabled bool mCheckObjValidityEnabled = false; // wether the validity of cached object is checked before proceeding to a CCDB API query + bool mFatalWhenNull = true; // if nullptr blob replies should be treated as fatal (can be set by user) long mCreatedNotAfter = 0; // upper limit for object creation timestamp (TimeMachine mode) - If-Not-After HTTP header long mCreatedNotBefore = 0; // lower limit for object creation timestamp (TimeMachine mode) - If-Not-Before HTTP header - bool mFatalWhenNull = true; // if nullptr blob replies should be treated as fatal (can be set by user) - + long mTimerMS = 0; // timer for queries + size_t mFetchedSize = 0; // total fetched size + int mQueries = 0; // total number of object queries + int mFetches = 0; // total number of succesful fetches from CCDB + int mFailures = 0; // total number of failed fetches + o2::framework::DeploymentMode mDeplMode; // O2 deployment mode ClassDefNV(CCDBManagerInstance, 1); }; template -T* CCDBManagerInstance::getForTimeStamp(std::string const& path, long timestamp) +T* CCDBManagerInstance::getForTimeStamp(std::string const& path, long timestamp, std::map* headers) { + mHeaders.clear(); // we clear at the beginning; to allow to retrieve the header information in a subsequent call T* ptr = nullptr; + mQueries++; + auto start = std::chrono::system_clock::now(); if (!isCachingEnabled()) { - ptr = mCCDBAccessor.retrieveFromTFileAny(path, mMetaData, timestamp, nullptr, "", + ptr = mCCDBAccessor.retrieveFromTFileAny(path, mMetaData, timestamp, &mHeaders, "", mCreatedNotAfter ? std::to_string(mCreatedNotAfter) : "", mCreatedNotBefore ? std::to_string(mCreatedNotBefore) : ""); - if (!ptr && mFatalWhenNull) { - reportFatal(std::string("Got nullptr from CCDB for path ") + path + std::string(" and timestamp ") + std::to_string(timestamp)); - } - return ptr; - } - auto& cached = mCache[path]; - if (mCheckObjValidityEnabled && cached.isValid(timestamp)) { - return reinterpret_cast(cached.noCleanupPtr ? cached.noCleanupPtr : cached.objPtr.get()); - } - ptr = mCCDBAccessor.retrieveFromTFileAny(path, mMetaData, timestamp, &mHeaders, cached.uuid, - mCreatedNotAfter ? std::to_string(mCreatedNotAfter) : "", - mCreatedNotBefore ? std::to_string(mCreatedNotBefore) : ""); - if (ptr) { // new object was shipped, old one (if any) is not valid anymore - if constexpr (std::is_same::value || std::is_base_of::value) { - // some special objects cannot be cached to shared_ptr since root may delete their raw global pointer - cached.noCleanupPtr = ptr; + if (!ptr) { + if (mFatalWhenNull) { + reportFatal(std::string("Got nullptr from CCDB for path ") + path + std::string(" and timestamp ") + std::to_string(timestamp)); + } + mFailures++; } else { - cached.objPtr.reset(ptr); + mFetches++; + auto sh = mHeaders.find("fileSize"); + if (sh != mHeaders.end()) { + size_t s = atol(sh->second.c_str()); + mFetchedSize += s; + } + } + + if (headers) { + *headers = mHeaders; + } + } else { + auto& cached = mCache[path]; + cached.queries++; + if ((!isOnline() && cached.isCacheValid(timestamp)) || (mCheckObjValidityEnabled && cached.isValid(timestamp))) { + // Give back the cached/saved headers + if (headers) { + *headers = cached.cacheOfHeaders; + } + return reinterpret_cast(cached.noCleanupPtr ? cached.noCleanupPtr : cached.objPtr.get()); + } + ptr = mCCDBAccessor.retrieveFromTFileAny(path, mMetaData, timestamp, &mHeaders, cached.uuid, + mCreatedNotAfter ? std::to_string(mCreatedNotAfter) : "", + mCreatedNotBefore ? std::to_string(mCreatedNotBefore) : ""); + // update the cached headers + for (auto const& h : mHeaders) { + cached.cacheOfHeaders[h.first] = h.second; + } + // return the cached headers + if (headers) { + *headers = cached.cacheOfHeaders; } - cached.uuid = mHeaders["ETag"]; - cached.startvalidity = std::stol(mHeaders["Valid-From"]); - cached.endvalidity = std::stol(mHeaders["Valid-Until"]); - } else if (mHeaders.count("Error")) { // in case of errors the pointer is 0 and headers["Error"] should be set - clearCache(path); // in case of any error clear cache for this object - } else { // the old object is valid + + if (ptr) { // new object was shipped, old one (if any) is not valid anymore + cached.fetches++; + mFetches++; + if constexpr (std::is_same::value || std::is_base_of::value) { + // some special objects cannot be cached to shared_ptr since root may delete their raw global pointer + cached.noCleanupPtr = ptr; + } else { + cached.objPtr.reset(ptr); + } + cached.uuid = mHeaders["ETag"]; + + try { + if (mHeaders.find("Valid-From") != mHeaders.end()) { + cached.startvalidity = std::stol(mHeaders["Valid-From"]); + } else { + // if meta-information missing assume infinit validity + // (should happen only for locally created objects) + cached.startvalidity = 0; + } + if (mHeaders.find("Valid-Until") != mHeaders.end()) { + cached.endvalidity = std::stol(mHeaders["Valid-Until"]); + } else { + cached.endvalidity = std::numeric_limits::max(); + } + cached.cacheValidFrom = timestamp; + } catch (std::exception const& e) { + reportFatal("Failed to read validity from CCDB response (Valid-From : " + mHeaders["Valid-From"] + std::string(" Valid-Until: ") + mHeaders["Valid-Until"] + std::string(")")); + } + auto sh = mHeaders.find("fileSize"); + if (sh != mHeaders.end()) { + size_t s = atol(sh->second.c_str()); + mFetchedSize += s; + cached.minSize = std::min(s, cached.minSize); + cached.maxSize = std::max(s, cached.minSize); + } + } else if (mHeaders.count("Error")) { // in case of errors the pointer is 0 and headers["Error"] should be set + cached.failures++; + cached.clear(); // in case of any error clear cache for this object + } + // the old object is valid, fetch cache end of validity ptr = reinterpret_cast(cached.noCleanupPtr ? cached.noCleanupPtr : cached.objPtr.get()); + if (mHeaders.find("Cache-Valid-Until") != mHeaders.end()) { + cached.cacheValidUntil = std::stol(mHeaders["Cache-Valid-Until"]); + } else { + cached.cacheValidUntil = -1; + } + mMetaData.clear(); + if (!ptr) { + if (mFatalWhenNull) { + reportFatal(std::string("Got nullptr from CCDB for path ") + path + std::string(" and timestamp ") + std::to_string(timestamp)); + } + mFailures++; + } } - mHeaders.clear(); - mMetaData.clear(); - if (!ptr && mFatalWhenNull) { - reportFatal(std::string("Got nullptr from CCDB for path ") + path + std::string(" and timestamp ") + std::to_string(timestamp)); + auto end = std::chrono::system_clock::now(); + mTimerMS += std::chrono::duration_cast(end - start).count(); + auto *ref = o2::framework::ServiceRegistryRef::globalDeviceRef(); + if (ref && ref->active()) { + auto& stats = ref->get(); + stats.updateStats({(int)o2::framework::ProcessingStatsId::CCDB_CACHE_HIT, o2::framework::DataProcessingStats::Op::Set, (int64_t)mQueries - mFailures - mFetches}); + stats.updateStats({(int)o2::framework::ProcessingStatsId::CCDB_CACHE_MISS, o2::framework::DataProcessingStats::Op::Set, (int64_t)mFetches}); + stats.updateStats({(int)o2::framework::ProcessingStatsId::CCDB_CACHE_FAILURE, o2::framework::DataProcessingStats::Op::Set, (int64_t)mFailures}); } return ptr; } +template +T* CCDBManagerInstance::getForRun(std::string const& path, int runNumber, bool setRunMetadata) +{ + auto metaData = setRunMetadata ? MD{{"runNumber", std::to_string(runNumber)}} : MD{}; + mMetaData = metaData; + return getSpecificForRun(path, runNumber, metaData); +} + +template +T* CCDBManagerInstance::getSpecificForRun(std::string const& path, int runNumber, MD const& metaData) +{ + auto [start, stop] = getRunDuration(runNumber, mFatalWhenNull); + if (start < 0 || stop < 0) { + if (mFatalWhenNull) { + reportFatal(std::string("Failed to get run duration for run ") + std::to_string(runNumber) + std::string(" from CCDB")); + } + return nullptr; + } + return getSpecific(path, start / 2 + stop / 2, metaData); +} + class BasicCCDBManager : public CCDBManagerInstance { public: @@ -237,8 +385,19 @@ class BasicCCDBManager : public CCDBManagerInstance private: using CCDBManagerInstance::CCDBManagerInstance; + BasicCCDBManager(std::string const& url) : CCDBManagerInstance(url) + { + const char* t = getenv("ALICEO2_CCDB_CONDITION_NOT_AFTER"); + if (t) { + auto timeaslong = strtol(t, nullptr, 10); + if (timeaslong != 0L) { + LOG(info) << "CCDB Time-machine constrained detected. Setting condition-not-after constrained to timestamp " << timeaslong; + setCreatedNotAfter(timeaslong); + } + } + } }; } // namespace o2::ccdb -#endif //O2_BASICCCDBMANAGER_H +#endif // O2_BASICCCDBMANAGER_H diff --git a/CCDB/include/CCDB/CCDBDownloader.h b/CCDB/include/CCDB/CCDBDownloader.h new file mode 100644 index 0000000000000..6c057a537a096 --- /dev/null +++ b/CCDB/include/CCDB/CCDBDownloader.h @@ -0,0 +1,435 @@ +// Copyright 2019-2023 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +#ifndef O2_CCDBDOWNLOADER_H_ +#define O2_CCDBDOWNLOADER_H_ + +#if !defined(__CINT__) && !defined(__MAKECINT__) && !defined(__ROOTCLING__) && !defined(__CLING__) +#include "MemoryResources/MemoryResources.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +typedef struct uv_loop_s uv_loop_t; +typedef struct uv_timer_s uv_timer_t; +typedef struct uv_poll_s uv_poll_t; +typedef struct uv_signal_s uv_signal_t; +typedef struct uv_async_s uv_async_t; +typedef struct uv_handle_s uv_handle_t; + +namespace o2::ccdb +{ + +#if !defined(__CINT__) && !defined(__MAKECINT__) && !defined(__ROOTCLING__) && !defined(__CLING__) +struct HeaderObjectPair_t { + std::multimap header; + o2::pmr::vector* object = nullptr; + int counter = 0; +}; + +typedef struct DownloaderRequestData { + std::vector hosts; + std::vector locations; + std::string path; + long timestamp; + HeaderObjectPair_t hoPair; + std::map* headers; + std::string userAgent; + curl_slist* optionsList; + + std::function localContentCallback; +} DownloaderRequestData; +#endif + +/* + Some functions below aren't member functions of CCDBDownloader because both curl and libuv require callback functions which have to be either static or non-member. + Because non-static functions are used in the functions below, they must be non-member. +*/ + +/** + * uv_walk callback which is used to close passed handle. + * + * @param handle Handle to be closed. + * @param arg Argument required by callback template. Is not used in this implementation. + */ +void closeHandles(uv_handle_t* handle, void* arg); + +/** + * Called by CURL in order to open a new socket. Newly opened sockets are assigned a timeout timer and added to socketTimerMap. + * + * @param clientp Pointer to the CCDBDownloader instance which controls the socket. + * @param purpose Purpose of opened socket. This parameter is unused but required by the callback template. + * @param address Structure containing information about family, type and protocol for the socket. + */ +curl_socket_t opensocketCallback(void* clientp, curlsocktype purpose, struct curl_sockaddr* address); + +/** + * Delete the handle. + * + * @param handle Handle assigned to this callback. + */ +void onUVClose(uv_handle_t* handle); + +enum DownloaderErrorLevel { + MINOR, + SEVERE +}; + +/// A class encapsulating and performing simple CURL requests in terms of a so-called CURL multi-handle. +/// A multi-handle allows to use a connection pool (connection cache) in the CURL layer even +/// with short-lived CURL easy-handles. Thereby the overhead of connection to servers can be +/// significantly reduced. For more info, see for instance https://everything.curl.dev/libcurl/connectionreuse. +/// +/// Further, this class adds functionality on top +/// of simple CURL (aysync requests, timeout handling, event loop, etc). +class CCDBDownloader +{ + public: + /** + * Timer starts for each socket when its respective transfer finishes, and is stopped when another transfer starts for that handle. + * When the timer runs out it closes the socket. The period for which socket stays open is defined by socketTimeoutMS. + */ + std::unordered_map mSocketTimerMap; + + /** + * The UV loop which handles transfers. Can be created internally or provided through a constructor. + */ + uv_loop_t* mUVLoop; + + /** + * Map used to store active uv_handles belonging to the CcdbDownloader. If internal uv_loop is used, then all uv_handles should be marked in this map. + */ + std::unordered_map mHandleMap; + + /** + * Time for which sockets will stay open after last download finishes + */ + int mKeepaliveTimeoutMS = 100; + + /** + * Time for connection to start before it times out. + */ + int mConnectionTimeoutMS = 60000; + + /** + * Time for request to finish before it times out. + */ + int mRequestTimeoutMS = 300000; + + /** + * Head start of IPv6 in regards to IPv4. + */ + int mHappyEyeballsHeadstartMS = 500; + + /** + * Max number of handles that can be used at the same time + */ + int mMaxHandlesInUse = 3; + + /** + * Variable denoting whether an external or internal uv_loop is being used. + */ + bool mExternalLoop; + + CCDBDownloader(uv_loop_t* uv_loop = nullptr); + ~CCDBDownloader(); + + /** + * Perform on a single handle in a blocking manner. Has the same effect as curl_easy_perform(). + * + * @param handle Handle to be performed on. It can be reused or cleaned after perform finishes. + */ + CURLcode perform(CURL* handle); + + /** + * Perform on a batch of handles in a blocking manner. Has the same effect as calling curl_easy_perform() on all handles in the vector. + * @param handleVector Handles to be performed on. + */ + std::vector batchBlockingPerform(std::vector const& handleVector); + + /** + * Schedules an asynchronous transfer but doesn't perform it. + * + * @param handle Handle to be performed on. + * @param requestCounter Counter shared by a batch of CURL handles. + */ + void asynchSchedule(CURL* handle, size_t* requestCounter); + + /** + * Limits the number of parallel connections. Should be used only if no transfers are happening. + */ + void setMaxParallelConnections(int limit); + + /** + * Limits the time a socket and its connection will be opened after transfer finishes. + */ + void setKeepaliveTimeoutTime(int timeoutMS); + + /** + * Setter for the connection timeout. + */ + void setConnectionTimeoutTime(int timeoutMS); + + /** + * Setter for the request timeout. + */ + void setRequestTimeoutTime(int timeoutMS); + + /** + * Setter for the happy eyeballs headstart. + */ + void setHappyEyeballsHeadstartTime(int headstartMS); + + /** + * Sets the timeout values selected for the offline environment. + */ + void setOfflineTimeoutSettings(); + + /** + * Sets the timeout values selected for the online environment. + */ + void setOnlineTimeoutSettings(); + + /** + * Run the uvLoop once. + * + * @param noWait Using this flag will cause the loop to run only if sockets have pendind data. + */ + void runLoop(bool noWait); + + /** + * Returns a message describing the transfer an it's result. + */ + std::string prepareLogMessage(std::string host_url, std::string userAgent, const std::string& path, long ts, const std::map* headers, long httpCode) const; + + /** + * Leaves only the protocol and host part of the url, discrading path and metadata. + */ + std::string trimHostUrl(std::string full_host_url) const; + + private: + /** + * Recognizes whether the address is a full url, or a partial one (like for example "/Task/Detector/1") and combines it with potentialHost if needed. + */ + std::string prepareRedirectedURL(std::string address, std::string potentialHost) const; + + /** + * Updates the locations vector with the the locations. + * + * @param headerMap Map containing response headers. + * @param locations Location list to be updated. + * @param locIndex Index of the next locaiton to be tried. + */ + void updateLocations(std::multimap* headerMap, std::vector* locations, int* locIndex) const; + + std::string mUserAgentId = "CCDBDownloader"; + /** + * Sets up internal UV loop. + */ + void setupInternalUVLoop(); + + /** + * Current amount of handles which are performed on. + */ + int mHandlesInUse = 0; + + /** + * Multi handle which controlls all network flow. + */ + CURLM* mCurlMultiHandle = nullptr; + + /** + * The timeout clock that is be used by CURL. + */ + uv_timer_t* mTimeoutTimer; + + /** + * Queue of handles awaiting their transfers to start. + */ + std::vector mHandlesToBeAdded; + + /** + * Types of requests. + */ + enum RequestType { + BLOCKING, + ASYNCHRONOUS + }; + + /** + * Information about a socket. + */ + typedef struct curl_context_s { + uv_poll_t* poll_handle; + curl_socket_t sockfd = -1; + CCDBDownloader* CD = nullptr; + } curl_context_t; + + /** + * Structure used for CURLMOPT_SOCKETDATA, which gives context for handleSocket + */ + typedef struct DataForSocket { + CCDBDownloader* CD; + CURLM* curlm; + } DataForSocket; + + DataForSocket mSocketData; + +#if !defined(__CINT__) && !defined(__MAKECINT__) && !defined(__ROOTCLING__) && !defined(__CLING__) + /** + * Structure which is stored in a easy_handle. It carries information about the request which the easy_handle is part of. + */ + typedef struct PerformData { + CURLcode* codeDestination; + size_t* requestsLeft; + RequestType type; + int hostInd; + int locInd; + DownloaderRequestData* requestData; + curl_slist** options; + } PerformData; +#endif + + /** + * Called by CURL in order to close a socket. It will be called by CURL even if a timeout timer closed the socket beforehand. + * + * @param clientp Pointer to the CCDBDownloader instance which controls the socket. + * @param item File descriptor of the socket. + */ + static void closesocketCallback(void* clientp, curl_socket_t item); + +#if !defined(__CINT__) && !defined(__MAKECINT__) && !defined(__ROOTCLING__) && !defined(__CLING__) + // Returns a new location string or an empty string if all locations under current host have been accessedd + std::string getNewLocation(PerformData* performData, std::vector& locations) const; + + // Reschedules the transfer to be performed with a different host. + void tryNewHost(PerformData* performData, CURL* easy_handle); + + // Retrieves content from either alien, cvmfs or local storage using a callback to CCDBApi. + void getLocalContent(PerformData* performData, std::string& newLocation, bool& contentRetrieved, std::vector& locations); + + // Continues a transfer via a http redirect. + void httpRedirect(PerformData* performData, std::string& newLocation, CURL* easy_handle); + + // Continues a transfer via a redirect. The redirect can point to a local file, alien file or a http address. + void followRedirect(PerformData* performData, CURL* easy_handle, std::vector& locations, bool& rescheduled, bool& contentRetrieved); +#endif + + /** + * Is used to react to polling file descriptors in poll_handle. + * + * @param handle Handle assigned to this callback. + * @param status Used to signal errors. + * @param events Bitmask used to describe events on the socket. + */ + static void curlPerform(uv_poll_t* handle, int status, int events); + + /** + * Used by CURL to react to action happening on a socket. + */ + static int handleSocket(CURL* easy, curl_socket_t s, int action, void* userp, void* socketp); + + /** + * Close socket assigned to the timer handle. + * + * @param handle Handle which is assigned to this callback. + */ + static void closeSocketByTimer(uv_timer_t* handle); + + /** + * Start new transfers, terminate expired transfers. + * + * @param req Handle which is assigned to this callback. + */ + static void curlTimeout(uv_timer_t* req); + + /** + * Free curl context assigned to the handle. + * + * @param handle Handle assigned to this callback. + */ + static void curlCloseCB(uv_handle_t* handle); + + /** + * Close poll handle assigned to the socket contained in the context and free data within the handle. + * + * @param context Structure containing information about socket and handle to be closed. + */ + static void destroyCurlContext(curl_context_t* context); + + /** + * Connect curl timer with uv timer. + * + * @param multi Multi handle for which the timeout will be set + * @param timeout_ms Time until timeout + * @param userp Pointer to the uv_timer_t handle that is used for timeout. + */ + static int startTimeout(CURLM* multi, long timeout_ms, void* userp); + + /** + * Create a new multi_handle for the downloader + */ + void initializeMultiHandle(); + + /** + * Release resources reserver for the transfer, mark transfer as complete, passe the CURLcode to the destination and launche callbacks if it is specified in PerformData. + * + * @param handle The easy_handle for which the transfer completed + * @param curlCode The code produced for the handle by the transfer + */ + void transferFinished(CURL* handle, CURLcode curlCode); + + /** + * Check message queue inside curl multi handle. + */ + void checkMultiInfo(); + +#if !defined(__CINT__) && !defined(__MAKECINT__) && !defined(__ROOTCLING__) && !defined(__CLING__) + /** + * Set openSocketCallback and closeSocketCallback with appropriate arguments. Stores data inside the CURL handle. + */ + void setHandleOptions(CURL* handle, PerformData* data); +#endif + + /** + * Create structure holding information about a socket including a poll handle assigned to it + * + * @param socketfd File descriptor of socket for which the structure will be created + */ + curl_context_t* createCurlContext(curl_socket_t sockfd); + + /** + * If multi_handles uses less then maximum number of handles then add handles from the queue. + */ + void checkHandleQueue(); +}; + +/** + * Structure assigned to a uv_timer_t before adding it to socketTimerMap. It stores the information about the socket connected to the timer. + */ +typedef struct DataForClosingSocket { + CCDBDownloader* CD; + curl_socket_t socket; +} DataForClosingSocket; + +} // namespace o2::ccdb + +#endif // O2_CCDB_CCDBDOWNLOADER_H diff --git a/CCDB/include/CCDB/CcdbApi.h b/CCDB/include/CCDB/CcdbApi.h index b1383b06bf6f8..4dab11d5972d8 100644 --- a/CCDB/include/CCDB/CcdbApi.h +++ b/CCDB/include/CCDB/CcdbApi.h @@ -30,11 +30,14 @@ #if !defined(__CINT__) && !defined(__MAKECINT__) && !defined(__ROOTCLING__) && !defined(__CLING__) #include "MemoryResources/MemoryResources.h" +#include #include #else class TJAlienCredentials; #endif +#include "CCDB/CCDBDownloader.h" + class TFile; class TGrid; @@ -61,6 +64,14 @@ class CcdbApi //: public DatabaseInterface /// \brief Default destructor virtual ~CcdbApi(); + // Delete copy and copy assignment constructor + CcdbApi(const CcdbApi&) = delete; + void operator=(const CcdbApi&) = delete; + + const std::string getUniqueAgentID() const { return mUniqueAgentID; } + + static bool checkAlienToken(); + /** * Initialize connection to CCDB * @@ -114,34 +125,34 @@ class CcdbApi //: public DatabaseInterface static std::unique_ptr> createObjectImage(const void* obj, std::type_info const& tinfo, CcdbObjectInfo* info = nullptr); /** - * Store into the CCDB a TFile containing the ROOT object. - * - * @param rootObject Raw pointer to the object to store. - * @param path The path where the object is going to be stored. - * @param metadata Key-values representing the metadata for this object. - * @param startValidityTimestamp Start of validity. If omitted, current timestamp is used. - * @param endValidityTimestamp End of validity. If omitted, current timestamp + 1 day is used. - * @return 0 -> ok, - * positive number -> curl error (https://curl.se/libcurl/c/libcurl-errors.html), - * -1 : object bigger than maxSize, - * -2 : curl initialization error - */ + * Store into the CCDB a TFile containing the ROOT object. + * + * @param rootObject Raw pointer to the object to store. + * @param path The path where the object is going to be stored. + * @param metadata Key-values representing the metadata for this object. + * @param startValidityTimestamp Start of validity. If omitted, current timestamp is used. + * @param endValidityTimestamp End of validity. If omitted, current timestamp + 1 day is used. + * @return 0 -> ok, + * positive number -> curl error (https://curl.se/libcurl/c/libcurl-errors.html), + * -1 : object bigger than maxSize, + * -2 : curl initialization error + */ int storeAsTFile(const TObject* rootObject, std::string const& path, std::map const& metadata, long startValidityTimestamp = -1, long endValidityTimestamp = -1, std::vector::size_type maxSize = 0 /*bytes*/) const; /** - * Store into the CCDB a TFile containing an object of type T (which needs to have a ROOT dictionary) - * - * @param obj Raw pointer to the object to store. - * @param path The path where the object is going to be stored. - * @param metadata Key-values representing the metadata for this object. - * @param startValidityTimestamp Start of validity. If omitted, current timestamp is used. - * @param endValidityTimestamp End of validity. If omitted, current timestamp + 1 day is used. - * @return 0 -> ok, - * positive number -> curl error (https://curl.se/libcurl/c/libcurl-errors.html), - * -1 : object bigger than maxSize, - * -2 : curl initialization error - */ + * Store into the CCDB a TFile containing an object of type T (which needs to have a ROOT dictionary) + * + * @param obj Raw pointer to the object to store. + * @param path The path where the object is going to be stored. + * @param metadata Key-values representing the metadata for this object. + * @param startValidityTimestamp Start of validity. If omitted, current timestamp is used. + * @param endValidityTimestamp End of validity. If omitted, current timestamp + 1 day is used. + * @return 0 -> ok, + * positive number -> curl error (https://curl.se/libcurl/c/libcurl-errors.html), + * -1 : object bigger than maxSize, + * -2 : curl initialization error + */ template int storeAsTFileAny(const T* obj, std::string const& path, std::map const& metadata, long startValidityTimestamp = -1, long endValidityTimestamp = -1, std::vector::size_type maxSize = 0 /*bytes*/) const @@ -203,8 +214,9 @@ class CcdbApi //: public DatabaseInterface * @param metadata The metadata to update * @param timestamp The timestamp to select the object * @param id The id, if any, to select the object + * @return anithing non-0 is CURL error code or -1 */ - void updateMetadata(std::string const& path, std::map const& metadata, long timestamp, std::string const& id = "", long newEOV = 0); + int updateMetadata(std::string const& path, std::map const& metadata, long timestamp, std::string const& id = "", long newEOV = 0); /** * Return the listing of objects, and in some cases subfolders, matching this path. @@ -227,7 +239,7 @@ class CcdbApi //: public DatabaseInterface * @param returnFormat The format of the returned string -> one of "text/plain (default)", "application/json", "text/xml" * @return The listing of folder and/or objects in the format requested */ - std::string list(std::string const& path = "", bool latestOnly = false, std::string const& returnFormat = "text/plain") const; + std::string list(std::string const& path = "", bool latestOnly = false, std::string const& returnFormat = "text/plain", long createdNotAfter = -1, long createdNotBefore = -1) const; /** * Make a local snapshot of all valid objects, given a timestamp, of the CCDB under a given local path. @@ -243,12 +255,12 @@ class CcdbApi //: public DatabaseInterface bool isHostReachable() const; /** - * Helper function to extract the list of sub-folders from a list reply into a vector container. - * Can be used to achieve full recursive traversal/listing of the CCDB. - * - * @param reply The reply that we got from a GET/browse sort of request. - * @return The vector of sub-folders. - */ + * Helper function to extract the list of sub-folders from a list reply into a vector container. + * Can be used to achieve full recursive traversal/listing of the CCDB. + * + * @param reply The reply that we got from a GET/browse sort of request. + * @return The vector of sub-folders. + */ std::vector parseSubFolders(std::string const& reply) const; /** @@ -269,7 +281,7 @@ class CcdbApi //: public DatabaseInterface * @return: True in case operation successful or false if there was a failure/problem. */ bool retrieveBlob(std::string const& path, std::string const& targetdir, std::map const& metadata, long timestamp, - bool preservePathStructure = true, std::string const& localFileName = "snapshot.root") const; + bool preservePathStructure = true, std::string const& localFileName = "snapshot.root", std::string const& createdNotAfter = "", std::string const& createdNotBefore = "", std::map* headers = nullptr) const; /** * Retrieve the headers of a CCDB entry, if it exists. @@ -336,14 +348,57 @@ class CcdbApi //: public DatabaseInterface TObject* retrieveFromTFile(std::string const& path, std::map const& metadata, long timestamp, std::map* headers, std::string const& etag, const std::string& createdNotAfter, const std::string& createdNotBefore) const; + void loadFileToMemory(std::vector& dest, std::string const& path, + std::map const& metadata, long timestamp, + std::map* headers, std::string const& etag, + const std::string& createdNotAfter, const std::string& createdNotBefore, bool considerSnapshot = true) const; #if !defined(__CINT__) && !defined(__MAKECINT__) && !defined(__ROOTCLING__) && !defined(__CLING__) - void loadFileToMemory(o2::pmr::vector& dest, const std::string& path, std::map* localHeaders = nullptr) const; + typedef struct RequestContext { + o2::pmr::vector& dest; + std::string path; + std::map const& metadata; + long timestamp; + std::map& headers; + std::string etag; + std::string createdNotAfter; + std::string createdNotBefore; + bool considerSnapshot; + + RequestContext(o2::pmr::vector& d, + std::map const& m, + std::map& h) + : dest(d), metadata(m), headers(h) {} + } RequestContext; + + // Stores file associated with requestContext as a snapshot. + void saveSnapshot(RequestContext& requestContext) const; + + // Schedules download via CCDBDownloader, but doesn't perform it until mUVLoop is ran. + void scheduleDownload(RequestContext& requestContext, size_t* requestCounter) const; + + void getFromSnapshot(bool createSnapshot, std::string const& path, + long timestamp, std::map& headers, + std::string& snapshotpath, o2::pmr::vector& dest, int& fromSnapshot, std::string const& etag) const; + void releaseNamedSemaphore(boost::interprocess::named_semaphore* sem, std::string const& path) const; + boost::interprocess::named_semaphore* createNamedSemaphore(std::string const& path) const; + static std::string determineSemaphoreName(std::string const& basedir, std::string const& objectpath); + // queries and optionally removes a named semaphore from the system + // returns true when successful (either found or found + removed) + static bool removeSemaphore(std::string const& name, bool remove = false); + static void removeLeakingSemaphores(std::string const& basedir, bool remove = false); + + void loadFileToMemory(o2::pmr::vector& dest, const std::string& path, std::map* localHeaders = nullptr, bool fetchLocalMetaData = true) const; void loadFileToMemory(o2::pmr::vector& dest, std::string const& path, std::map const& metadata, long timestamp, std::map* headers, std::string const& etag, const std::string& createdNotAfter, const std::string& createdNotBefore, bool considerSnapshot = true) const; - void navigateURLsAndLoadFileToMemory(o2::pmr::vector& dest, CURL* curl_handle, std::string const& url, std::map* headers) const; + + // Loads files from alien and cvmfs into given destination. + bool loadLocalContentToMemory(o2::pmr::vector& dest, std::string& url) const; + + // add annotated flattened headers in the end of the blob + static void appendFlatHeader(o2::pmr::vector& dest, const std::map& headers); // the failure to load the file to memory is signaled by 0 size and non-0 capacity static bool isMemoryFileInvalid(const o2::pmr::vector& v) { return v.size() == 0 && v.capacity() > 0; } @@ -358,9 +413,39 @@ class CcdbApi //: public DatabaseInterface } return obj; } + + /** + * Retrieves files either as snapshot or schedules them to be downloaded via CCDBDownloader. + * + * @param requestContext Structure giving details about the transfer. + * @param fromSnapshot After navigateSourcesAndLoadFile returns signals whether file was retrieved from snapshot. + * @param requestCounter Pointer to the variable storing the number of requests to be done. + */ + void navigateSourcesAndLoadFile(RequestContext& requestContext, int& fromSnapshot, size_t* requestCounter) const; + + /** + * Retrieves files described via RequestContexts into memory. Downloads are performed in parallel via CCDBDownloader. + * + * @param requestContext Structure giving details about the transfer. + */ + void vectoredLoadFileToMemory(std::vector& requestContext) const; #endif private: + // Sets the unique agent ID + void setUniqueAgentID(); + + /** + * Schedules download of data associated with the curl_handle. Doing that increments the requestCounter by 1. Requests are performed by running the mUVLoop + * + * @param handle CURL handle associated with the request. + * @param requestCounter Pointer to the variable storing the number of requests to be done. + */ + void asynchPerform(CURL* handle, size_t* requestCounter) const; + + // internal helper function to update a CCDB file with meta information + static void updateMetaInformationInLocalFile(std::string const& filename, std::map const* headers, CCDBQuery const* querysummary = nullptr); + // report what file is read and for which purpose void logReading(const std::string& path, long ts, const std::map* headers, const std::string& comment) const; @@ -439,6 +524,23 @@ class CcdbApi //: public DatabaseInterface long timestamp = -1, std::map* headers = nullptr, std::string const& etag = "", const std::string& createdNotAfter = "", const std::string& createdNotBefore = "") const; + /** + * Run the uvLoop belonging to mDownloader once. + * + * @param noWait Using this flag will cause the loop to run only if sockets have pendind data. + */ + void runDownloaderLoop(bool noWait); + /** + * Set the number of times curl should retry in case of failure and the delay between thte attempts. + * @param numberRetries + * @param delay + */ + void setCurlRetriesParameters(int numberRetries, int delay = 100000 /* microseconds */) + { + mCurlRetries = numberRetries; + mCurlDelayRetries = delay; + } + private: /** * A helper function to extract object from a local ROOT file @@ -449,12 +551,12 @@ class CcdbApi //: public DatabaseInterface void* extractFromLocalFile(std::string const& filename, std::type_info const& tinfo, std::map* headers) const; /** - * Helper function to download binary content from alien:// storage + * Helper function to download binary content from alien://, cvmfs or local storage * @param fullUrl The alien URL * @param tcl The TClass object describing the serialized type * @return raw pointer to created object */ - void* downloadAlienContent(std::string const& fullUrl, std::type_info const& tinfo) const; + void* downloadFilesystemContent(std::string const& fullUrl, std::type_info const& tinfo, std::map* headers) const; // initialize the TGrid (Alien connection) bool initTGrid() const; @@ -474,15 +576,12 @@ class CcdbApi //: public DatabaseInterface // convert type_info to TClass, throw on failure static TClass* tinfo2TClass(std::type_info const& tinfo); - // split string on delimiters and return tokens as vector - std::vector splitString(const std::string& str, const char* delimiters); - typedef size_t (*CurlWriteCallback)(void*, size_t, size_t, void*); void initCurlOptionsForRetrieve(CURL* curlHandle, void* pointer, CurlWriteCallback writeCallback, bool followRedirect = true) const; - void initHeadersForRetrieve(CURL* curlHandle, long timestamp, std::map* headers, std::string const& etag, - const std::string& createdNotAfter, const std::string& createdNotBefore) const; + /// initialize HTTPS header information for the CURL handle. Needs to be given an existing curl_slist* pointer to work with (may be nullptr), which needs to be free by the caller. + void initCurlHTTPHeaderOptionsForRetrieve(CURL* curlHandle, curl_slist*& option_list, long timestamp, std::map* headers, std::string const& etag, const std::string& createdNotAfter, const std::string& createdNotBefore) const; bool receiveToFile(FILE* fileHandle, std::string const& path, std::map const& metadata, long timestamp, std::map* headers = nullptr, std::string const& etag = "", @@ -497,9 +596,9 @@ class CcdbApi //: public DatabaseInterface const std::string& createdNotAfter, const std::string& createdNotBefore, bool followRedirect, CurlWriteCallback writeCallback) const; /** - * Initialize hostsPool - * @param hosts string with hosts separated by "," or ";" - */ + * Initialize hostsPool + * @param hosts string with hosts separated by "," or ";" + */ void initHostsPool(std::string hosts); std::string getHostUrl(int hostIndex) const; @@ -516,6 +615,22 @@ class CcdbApi //: public DatabaseInterface return getSnapshotDir(topdir, path) + '/' + sfile; } + template // can be either std::map or std::multimap + static size_t getFlatHeaderSize(const MAP& Headers) + { + size_t hsize = sizeof(int) + sizeof(FlatHeaderAnnot); // annotation size + for (auto& h : Headers) { + hsize += h.first.length() + h.second.length() + 2; // 2*(string_buffer + terminating null character) + } + return hsize; + } + + // tmp helper and single point of entry for a CURL perform call + // helps to switch between easy handle perform and multi handles in a single place + CURLcode CURL_perform(CURL* handle) const; + + mutable CCDBDownloader* mDownloader = nullptr; //! the multi-handle (async) CURL downloader + bool mIsCCDBDownloaderPreferred = false; /// Base URL of the CCDB (with port) std::string mUniqueAgentID{}; // Unique User-Agent ID communicated to server for logging std::string mUrl{}; @@ -527,6 +642,12 @@ class CcdbApi //: public DatabaseInterface mutable TGrid* mAlienInstance = nullptr; // a cached connection to TGrid (needed for Alien locations) bool mNeedAlienToken = true; // On EPN and FLP we use a local cache and don't need the alien token static std::unique_ptr mJAlienCredentials; // access JAliEn credentials + int mCurlRetries = 3; + int mCurlDelayRetries = 100000; // in microseconds + size_t mCurlTimeoutDownload = 15; // download timeout in seconds, can be configured via ALICEO2_CCDB_CURL_TIMEOUT_DOWNLOAD, updated according to the deployment mode + size_t mCurlTimeoutUpload = 15; // upload timeout in seconds, can be configured via ALICEO2_CCDB_CURL_TIMEOUT_UPLOAD, updated according to the deployment mode + + static constexpr char FlatHeaderAnnot[] = "$HEADER$"; // annotation for flat header ClassDefNV(CcdbApi, 1); }; @@ -558,4 +679,4 @@ typename std::enable_if::value, } // namespace ccdb } // namespace o2 -#endif //PROJECT_CCDBAPI_H +#endif // PROJECT_CCDBAPI_H diff --git a/CCDB/include/CCDB/CcdbObjectInfo.h b/CCDB/include/CCDB/CcdbObjectInfo.h index 438f531b66f38..117ca1123104f 100644 --- a/CCDB/include/CCDB/CcdbObjectInfo.h +++ b/CCDB/include/CCDB/CcdbObjectInfo.h @@ -44,8 +44,8 @@ class CcdbObjectInfo } CcdbObjectInfo(std::string path, std::string objType, std::string flName, std::map metadata, - long startValidityTimestamp, long endValidityTimestamp, bool adjustableEOV = true) - : mObjType(std::move(objType)), mFileName(std::move(flName)), mPath(std::move(path)), mMD(std::move(metadata)), mStart(startValidityTimestamp), mEnd(endValidityTimestamp) + long startValidityTimestamp, long endValidityTimestamp, bool adjustableEOV = true, bool validateUpload = false) + : mObjType(std::move(objType)), mFileName(std::move(flName)), mPath(std::move(path)), mMD(std::move(metadata)), mStart(startValidityTimestamp), mEnd(endValidityTimestamp), mValidateUpload(validateUpload) { if (adjustableEOV) { setAdjustableEOV(); @@ -82,7 +82,10 @@ class CcdbObjectInfo } } + void setValidateUpload(bool v) { mValidateUpload = v; } + bool isAdjustableEOV() const { return mAdjustableEOV; } + bool getValidateUpload() const { return mValidateUpload; } [[nodiscard]] long getStartValidityTimestamp() const { return mStart; } void setStartValidityTimestamp(long start) { mStart = start; } @@ -90,6 +93,16 @@ class CcdbObjectInfo [[nodiscard]] long getEndValidityTimestamp() const { return mEnd; } void setEndValidityTimestamp(long end) { mEnd = end; } + bool operator<(const CcdbObjectInfo& other) const + { + return mStart < other.mStart; + } + + bool operator>(const CcdbObjectInfo& other) const + { + return mStart > other.mStart; + } + private: std::string mObjType{}; // object type (e.g. class) std::string mFileName{}; // file name in the CCDB @@ -98,9 +111,23 @@ class CcdbObjectInfo long mStart = 0; // start of the validity of the object long mEnd = 0; // end of the validity of the object bool mAdjustableEOV = false; // each new object may override EOV of object it overrides to its own SOV - ClassDefNV(CcdbObjectInfo, 2); + bool mValidateUpload = false; // request to validate the upload by querying its header + ClassDefNV(CcdbObjectInfo, 3); }; } // namespace o2::ccdb +namespace std +{ +// defining std::hash for InteractionRecord to be used with std containers +template <> +struct hash { + public: + size_t operator()(const o2::ccdb::CcdbObjectInfo& info) const + { + return info.getStartValidityTimestamp(); + } +}; +} // namespace std + #endif // O2_CCDB_CCDBOBJECTINFO_H_ diff --git a/CCDB/src/BasicCCDBManager.cxx b/CCDB/src/BasicCCDBManager.cxx index f0af44ab0dd3d..d55fdad960d3a 100644 --- a/CCDB/src/BasicCCDBManager.cxx +++ b/CCDB/src/BasicCCDBManager.cxx @@ -13,8 +13,10 @@ // Created by Sandro Wenzel on 2019-08-14. // #include "CCDB/BasicCCDBManager.h" +#include "Framework/ServiceRegistryRef.h" +#include "Framework/DataProcessingStats.h" #include -#include "FairLogger.h" +#include #include namespace o2 @@ -32,15 +34,105 @@ void CCDBManagerInstance::reportFatal(std::string_view err) LOG(fatal) << err; } -std::pair CCDBManagerInstance::getRunDuration(int runnumber) const +std::pair CCDBManagerInstance::getRunDuration(const std::map& headers) { - auto response = mCCDBAccessor.retrieveHeaders("RCT/Info/RunInformation", std::map(), runnumber); - if (response.size() == 0 || response.find("SOR") == response.end() || response.find("EOR") == response.end()) { - LOG(fatal) << "Empty or missing response from query to RCT/Info/RunInformation for run " << runnumber; + if (headers.size() != 0) { + std::string report{}; + auto strt = headers.find("STF"); + auto stop = headers.find("ETF"); + long valStrt = (strt == headers.end()) ? -1L : boost::lexical_cast(strt->second); + long valStop = (stop == headers.end()) ? -1L : boost::lexical_cast(stop->second); + if (valStrt < 0 || valStop < 0) { + report += "Missing STF/EFT -> use SOX/EOX;"; + strt = headers.find("SOX"); + valStrt = (strt == headers.end()) ? -1L : boost::lexical_cast(strt->second); + if (valStrt < 1) { + report += fmt::format(" Missing/invalid SOX -> use SOR"); + strt = headers.find("SOR"); + valStrt = (strt == headers.end()) ? -1L : boost::lexical_cast(strt->second); + } + stop = headers.find("EOX"); + valStop = (stop == headers.end()) ? -1L : boost::lexical_cast(stop->second); + if (valStop < 1) { + report += fmt::format(" | Missing/invalid EOX -> use EOR"); + stop = headers.find("EOR"); + valStop = (stop == headers.end()) ? -1L : boost::lexical_cast(stop->second); + } + if (!report.empty()) { + LOGP(warn, "{}", report); + } + } + return std::make_pair(valStrt, valStop); } - auto sor = boost::lexical_cast(response["SOR"]); - auto eor = boost::lexical_cast(response["EOR"]); - return std::make_pair(sor, eor); + return std::make_pair(-1L, -1L); +} + +std::pair CCDBManagerInstance::getRunDuration(o2::ccdb::CcdbApi const& api, int runnumber, bool fatal) +{ + auto headers = api.retrieveHeaders("RCT/Info/RunInformation", std::map(), runnumber); + auto response = getRunDuration(headers); + if ((response.first <= 0 || response.second < response.first) && fatal) { + LOG(fatal) << "Empty, missing or invalid response from query to RCT/Info/RunInformation for run " << runnumber; + } + return response; +} + +std::pair CCDBManagerInstance::getRunDuration(int runnumber, bool fatal) +{ + mQueries++; + if (!isCachingEnabled()) { + return CCDBManagerInstance::getRunDuration(mCCDBAccessor, runnumber, fatal); + } + auto& cached = mCache["RCT-Run-Info HeaderOnly"]; + std::pair rd; + cached.queries++; + if (cached.startvalidity != runnumber) { // need to fetch + rd = CCDBManagerInstance::getRunDuration(mCCDBAccessor, runnumber, fatal); + cached.objPtr = std::make_shared>(rd); + cached.startvalidity = runnumber; + cached.endvalidity = runnumber + 1; + cached.minSize = cached.maxSize = 0; + cached.fetches++; + } else { + rd = *reinterpret_cast*>(cached.objPtr.get()); + } + return rd; +} + +std::string CCDBManagerInstance::getSummaryString() const +{ + std::string res = fmt::format("{} queries, {} bytes", mQueries, fmt::group_digits(mFetchedSize)); + if (mCachingEnabled) { + res += fmt::format(" for {} objects", mCache.size()); + } + res += fmt::format(", {} good fetches (and {} failed ones", mFetches, mFailures); + if (mCachingEnabled && mFailures) { + int nfailObj = 0; + for (const auto& obj : mCache) { + if (obj.second.failures) { + nfailObj++; + } + } + res += fmt::format(" for {} objects", nfailObj); + } + res += fmt::format(") in {} ms, instance: {}", fmt::group_digits(mTimerMS), mCCDBAccessor.getUniqueAgentID()); + return res; +} + +void CCDBManagerInstance::report(bool longrep) +{ + LOG(info) << "CCDBManager summary: " << getSummaryString(); + if (longrep && mCachingEnabled) { + LOGP(info, "CCDB cache miss/hit/failures"); + for (const auto& obj : mCache) { + LOGP(info, " {}: {}/{}/{} ({}-{} bytes)", obj.first, obj.second.fetches, obj.second.queries - obj.second.fetches - obj.second.failures, obj.second.failures, obj.second.minSize, obj.second.maxSize); + } + } +} + +void CCDBManagerInstance::endOfStream() +{ + report(true); } } // namespace ccdb diff --git a/CCDB/src/CCDBDownloader.cxx b/CCDB/src/CCDBDownloader.cxx new file mode 100644 index 0000000000000..2f033a50b36e7 --- /dev/null +++ b/CCDB/src/CCDBDownloader.cxx @@ -0,0 +1,772 @@ +// Copyright 2019-2023 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include +#include "CommonUtils/StringUtils.h" +#include "CCDB/CCDBTimeStampUtils.h" +#include "Framework/Signpost.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +O2_DECLARE_DYNAMIC_STACKTRACE_LOG(ccdb_downloader); + +namespace o2::ccdb +{ + +void uvErrorCheck(int code, DownloaderErrorLevel level) +{ + if (code != 0) { + char buf[1000]; + uv_strerror_r(code, buf, 1000); + O2_SIGNPOST_ID_GENERATE(sid, ccdb_downloader); + if (level == SEVERE) { + O2_SIGNPOST_EVENT_EMIT_ERROR(ccdb_downloader, sid, "CCDBDownloader", "UV error - %{public}s", buf); + } else { + O2_SIGNPOST_EVENT_EMIT_WARN(ccdb_downloader, sid, "CCDBDownloader", "UV minor error - %{public}s", buf); + } + } +} + +void curlEasyErrorCheck(CURLcode code) +{ + if (code != CURLE_OK) { + O2_SIGNPOST_ID_GENERATE(sid, ccdb_downloader); + O2_SIGNPOST_EVENT_EMIT_ERROR(ccdb_downloader, sid, "CCDBDownloader", "CURL error - %{public}s", curl_easy_strerror(code)); + } +} + +void curlMultiErrorCheck(CURLMcode code) +{ + if (code != CURLM_OK) { + O2_SIGNPOST_ID_GENERATE(sid, ccdb_downloader); + O2_SIGNPOST_EVENT_EMIT_ERROR(ccdb_downloader, sid, "CCDBDownloader", "CURL error - %{public}s", curl_multi_strerror(code)); + } +} +namespace +{ +std::string uniqueAgentID() +{ + std::string host = boost::asio::ip::host_name(); + char const* jobID = getenv("ALIEN_PROC_ID"); + if (jobID) { + return fmt::format("{}-{}-{}-{}", host, getCurrentTimestamp() / 1000, o2::utils::Str::getRandomString(6), jobID); + } else { + return fmt::format("{}-{}-{}", host, getCurrentTimestamp() / 1000, o2::utils::Str::getRandomString(6)); + } +} +} // namespace + +CCDBDownloader::CCDBDownloader(uv_loop_t* uv_loop) + : mUserAgentId(uniqueAgentID()) +{ + if (uv_loop) { + mExternalLoop = true; + mUVLoop = uv_loop; + } else { + mExternalLoop = false; + setupInternalUVLoop(); + } + + // Preparing timer to be used by curl + mTimeoutTimer = (uv_timer_t*)malloc(sizeof(*mTimeoutTimer)); + mTimeoutTimer->data = this; + uvErrorCheck(uv_timer_init(mUVLoop, mTimeoutTimer), SEVERE); + mHandleMap[(uv_handle_t*)mTimeoutTimer] = true; + + initializeMultiHandle(); +} + +void CCDBDownloader::setupInternalUVLoop() +{ + mUVLoop = new uv_loop_t(); + uvErrorCheck(uv_loop_init(mUVLoop), SEVERE); +} + +void CCDBDownloader::initializeMultiHandle() +{ + mCurlMultiHandle = curl_multi_init(); + curlMultiErrorCheck(curl_multi_setopt(mCurlMultiHandle, CURLMOPT_SOCKETFUNCTION, handleSocket)); + auto socketData = &mSocketData; + socketData->curlm = mCurlMultiHandle; + socketData->CD = this; + curlMultiErrorCheck(curl_multi_setopt(mCurlMultiHandle, CURLMOPT_SOCKETDATA, socketData)); + curlMultiErrorCheck(curl_multi_setopt(mCurlMultiHandle, CURLMOPT_TIMERFUNCTION, startTimeout)); + curlMultiErrorCheck(curl_multi_setopt(mCurlMultiHandle, CURLMOPT_TIMERDATA, mTimeoutTimer)); + curlMultiErrorCheck(curl_multi_setopt(mCurlMultiHandle, CURLMOPT_MAX_TOTAL_CONNECTIONS, mMaxHandlesInUse)); +} + +CCDBDownloader::~CCDBDownloader() +{ + // Loop has been ordered to stop via signalToClose() + curlMultiErrorCheck(curl_multi_cleanup(mCurlMultiHandle)); + + if (!mExternalLoop) { + // Schedule all handles to close. Execute loop to allow them to execute their destructors. + while (uv_loop_alive(mUVLoop) || (uv_loop_close(mUVLoop) == UV_EBUSY)) { + uv_walk(mUVLoop, closeHandles, this); + uv_run(mUVLoop, UV_RUN_ONCE); + } + delete mUVLoop; + } +} + +void closeHandles(uv_handle_t* handle, void* arg) +{ + auto CD = (CCDBDownloader*)arg; + // Close only handles belonging to the Downloader + if (CD->mHandleMap.find(handle) != CD->mHandleMap.end()) { + if (!uv_is_closing(handle)) { + uv_close(handle, onUVClose); + } + } +} + +void onUVClose(uv_handle_t* handle) +{ + if (handle != nullptr) { + free(handle); + } +} + +void CCDBDownloader::closesocketCallback(void* clientp, curl_socket_t item) +{ + auto CD = (CCDBDownloader*)clientp; + if (CD->mExternalLoop) { + // If external uv loop is used then the keepalive mechanism is active. + if (CD->mSocketTimerMap.find(item) != CD->mSocketTimerMap.end()) { + auto timer = CD->mSocketTimerMap[item]; + uvErrorCheck(uv_timer_stop(timer), SEVERE); + // we are getting rid of the uv_timer_t pointer ... so we need + // to free possibly attached user data pointers as well. Counteracts action of opensocketCallback + if (timer->data) { + delete (DataForClosingSocket*)timer->data; + } + CD->mSocketTimerMap.erase(item); + if (close(item) == -1) { + O2_SIGNPOST_ID_GENERATE(sid, ccdb_downloader); + O2_SIGNPOST_EVENT_EMIT_ERROR(ccdb_downloader, sid, "CCDBDownloader", "CCDBDownloader: Socket failed to close"); + } + } + } else { + if (close(item) == -1) { + O2_SIGNPOST_ID_GENERATE(sid, ccdb_downloader); + O2_SIGNPOST_EVENT_EMIT_ERROR(ccdb_downloader, sid, "CCDBDownloader", "CCDBDownloader: Socket failed to close"); + } + } +} + +curl_socket_t opensocketCallback(void* clientp, curlsocktype purpose, struct curl_sockaddr* address) +{ + auto CD = (CCDBDownloader*)clientp; + auto sock = socket(address->family, address->socktype, address->protocol); + if (sock == -1) { + O2_SIGNPOST_ID_GENERATE(sid, ccdb_downloader); + O2_SIGNPOST_EVENT_EMIT_ERROR(ccdb_downloader, sid, "CCDBDownloader", "CCDBDownloader: Socket failed to open"); + } + + if (CD->mExternalLoop) { + CD->mSocketTimerMap[sock] = (uv_timer_t*)malloc(sizeof(*CD->mSocketTimerMap[sock])); + uvErrorCheck(uv_timer_init(CD->mUVLoop, CD->mSocketTimerMap[sock]), SEVERE); + CD->mHandleMap[(uv_handle_t*)CD->mSocketTimerMap[sock]] = true; + + auto data = new DataForClosingSocket(); + data->CD = CD; + data->socket = sock; + CD->mSocketTimerMap[sock]->data = data; + } + + return sock; +} + +void CCDBDownloader::closeSocketByTimer(uv_timer_t* handle) +{ + auto data = (DataForClosingSocket*)handle->data; + auto CD = data->CD; + auto sock = data->socket; + + if (CD->mSocketTimerMap.find(sock) != CD->mSocketTimerMap.end()) { + uvErrorCheck(uv_timer_stop(CD->mSocketTimerMap[sock]), SEVERE); + CD->mSocketTimerMap.erase(sock); + if (close(sock) == -1) { + O2_SIGNPOST_ID_GENERATE(sid, ccdb_downloader); + O2_SIGNPOST_EVENT_EMIT_ERROR(ccdb_downloader, sid, "CCDBDownloader", "CCDBDownloader: Socket failed to close"); + } + delete data; + } +} + +void CCDBDownloader::curlTimeout(uv_timer_t* handle) +{ + auto CD = (CCDBDownloader*)handle->data; + int running_handles; + curl_multi_socket_action(CD->mCurlMultiHandle, CURL_SOCKET_TIMEOUT, 0, &running_handles); + CD->checkMultiInfo(); +} + +void CCDBDownloader::curlPerform(uv_poll_t* handle, int status, int events) +{ + uvErrorCheck(status, MINOR); + int running_handles; + int flags = 0; + if (events & UV_READABLE) { + flags |= CURL_CSELECT_IN; + } + if (events & UV_WRITABLE) { + flags |= CURL_CSELECT_OUT; + } + + auto context = (CCDBDownloader::curl_context_t*)handle->data; + + curlMultiErrorCheck(curl_multi_socket_action(context->CD->mCurlMultiHandle, context->sockfd, flags, &running_handles)); + context->CD->checkMultiInfo(); +} + +int CCDBDownloader::handleSocket(CURL* easy, curl_socket_t s, int action, void* userp, void* socketp) +{ + auto socketData = (CCDBDownloader::DataForSocket*)userp; + auto CD = (CCDBDownloader*)socketData->CD; + CCDBDownloader::curl_context_t* curl_context; + int events = 0; + + switch (action) { + case CURL_POLL_IN: + case CURL_POLL_OUT: + case CURL_POLL_INOUT: + + curl_context = socketp ? (CCDBDownloader::curl_context_t*)socketp : CD->createCurlContext(s); + curlMultiErrorCheck(curl_multi_assign(socketData->curlm, s, (void*)curl_context)); + + if (action != CURL_POLL_IN) { + events |= UV_WRITABLE; + } + if (action != CURL_POLL_OUT) { + events |= UV_READABLE; + } + + if (CD->mExternalLoop && CD->mSocketTimerMap.find(s) != CD->mSocketTimerMap.end()) { + uvErrorCheck(uv_timer_stop(CD->mSocketTimerMap[s]), SEVERE); + } + + uvErrorCheck(uv_poll_start(curl_context->poll_handle, events, curlPerform), SEVERE); + break; + case CURL_POLL_REMOVE: + if (socketp) { + if (CD->mExternalLoop) { + // If external loop is used then start the keepalive timeout. + if (CD->mSocketTimerMap.find(s) != CD->mSocketTimerMap.end()) { + uvErrorCheck(uv_timer_start(CD->mSocketTimerMap[s], closeSocketByTimer, CD->mKeepaliveTimeoutMS, 0), SEVERE); + } + } + uvErrorCheck(uv_poll_stop(((CCDBDownloader::curl_context_t*)socketp)->poll_handle), SEVERE); + CD->destroyCurlContext((CCDBDownloader::curl_context_t*)socketp); + curlMultiErrorCheck(curl_multi_assign(socketData->curlm, s, nullptr)); + } + break; + default: + abort(); + } + + return 0; +} + +void CCDBDownloader::setMaxParallelConnections(int limit) +{ + mMaxHandlesInUse = limit; +} + +void CCDBDownloader::setKeepaliveTimeoutTime(int timeoutMS) +{ + mKeepaliveTimeoutMS = timeoutMS; +} + +void CCDBDownloader::setConnectionTimeoutTime(int timeoutMS) +{ + mConnectionTimeoutMS = timeoutMS; +} + +void CCDBDownloader::setRequestTimeoutTime(int timeoutMS) +{ + mRequestTimeoutMS = timeoutMS; +} + +void CCDBDownloader::setHappyEyeballsHeadstartTime(int headstartMS) +{ + mHappyEyeballsHeadstartMS = headstartMS; +} + +void CCDBDownloader::setOfflineTimeoutSettings() +{ + setConnectionTimeoutTime(60000); + setRequestTimeoutTime(300000); + setHappyEyeballsHeadstartTime(500); +} + +void CCDBDownloader::setOnlineTimeoutSettings() +{ + setConnectionTimeoutTime(5000); + setRequestTimeoutTime(30000); + setHappyEyeballsHeadstartTime(500); +} + +CCDBDownloader::curl_context_t* CCDBDownloader::createCurlContext(curl_socket_t sockfd) +{ + curl_context_t* context; + + context = (curl_context_t*)malloc(sizeof(*context)); + context->CD = this; + context->sockfd = sockfd; + context->poll_handle = (uv_poll_t*)malloc(sizeof(*context->poll_handle)); + + uvErrorCheck(uv_poll_init_socket(mUVLoop, context->poll_handle, sockfd), SEVERE); + mHandleMap[(uv_handle_t*)(context->poll_handle)] = true; + context->poll_handle->data = context; + + return context; +} + +void CCDBDownloader::curlCloseCB(uv_handle_t* handle) +{ + auto* context = (curl_context_t*)handle->data; + free(context->poll_handle); + free(context); +} + +void CCDBDownloader::destroyCurlContext(curl_context_t* context) +{ + uv_close((uv_handle_t*)context->poll_handle, curlCloseCB); +} + +void CCDBDownloader::tryNewHost(PerformData* performData, CURL* easy_handle) +{ + auto requestData = performData->requestData; + std::string newUrl = requestData->hosts.at(performData->hostInd) + "/" + requestData->path + "/" + std::to_string(requestData->timestamp); + LOG(debug) << "Connecting to another host " << newUrl << "\n"; + requestData->hoPair.header.clear(); + curl_easy_setopt(easy_handle, CURLOPT_URL, newUrl.c_str()); + mHandlesToBeAdded.push_back(easy_handle); +} + +void CCDBDownloader::getLocalContent(PerformData* performData, std::string& newLocation, bool& contentRetrieved, std::vector& locations) +{ + auto requestData = performData->requestData; + LOG(debug) << "Redirecting to local content " << newLocation << "\n"; + if (requestData->localContentCallback(newLocation)) { + contentRetrieved = true; + LOG(debug) << "Local content retrieved succesfully: " << newLocation << " n"; + } else { + // Prepare next redirect url + newLocation = getNewLocation(performData, locations); + LOG(debug) << "Failed to retrieve local content: " << newLocation << "\n"; + } +} + +std::string CCDBDownloader::getNewLocation(PerformData* performData, std::vector& locations) const +{ + auto requestData = performData->requestData; + if (performData->locInd < locations.size()) { + std::string newLocation = locations.at(performData->locInd++); + std::string hostUrl = requestData->hosts.at(performData->hostInd); + std::string newUrl = prepareRedirectedURL(newLocation, hostUrl); + return newUrl; + } else { + return ""; + } +} + +void CCDBDownloader::httpRedirect(PerformData* performData, std::string& newLocation, CURL* easy_handle) +{ + auto requestData = performData->requestData; + LOG(debug) << "Trying content location " << newLocation << "\n"; + curl_easy_setopt(easy_handle, CURLOPT_URL, newLocation.c_str()); + mHandlesToBeAdded.push_back(easy_handle); +} + +void CCDBDownloader::followRedirect(PerformData* performData, CURL* easy_handle, std::vector& locations, bool& rescheduled, bool& contentRetrieved) +{ + std::string newLocation = getNewLocation(performData, locations); + while (!contentRetrieved && (newLocation.find("alien:/", 0) != std::string::npos || newLocation.find("file:/", 0) != std::string::npos)) { + getLocalContent(performData, newLocation, contentRetrieved, locations); + } + if (!contentRetrieved && newLocation != "") { + httpRedirect(performData, newLocation, easy_handle); + rescheduled = true; + } +} + +std::string CCDBDownloader::trimHostUrl(std::string full_host_url) const +{ + CURLU* host_url = curl_url(); + curl_url_set(host_url, CURLUPART_URL, full_host_url.c_str(), 0); + + // Get host part (the only critical part) + char* host; + CURLUcode host_result = curl_url_get(host_url, CURLUPART_HOST, &host, 0); + if (host_result != CURLUE_OK) { + LOG(error) << "CCDBDownloader: Malformed url detected when processing redirect, could not identify the host part: " << full_host_url; + curl_url_cleanup(host_url); + return ""; + } + // Get scheme (protocol) part + char* scheme; + CURLUcode scheme_result = curl_url_get(host_url, CURLUPART_SCHEME, &scheme, 0); + // Get port + char* port; + CURLUcode port_result = curl_url_get(host_url, CURLUPART_PORT, &port, 0); + + curl_url_cleanup(host_url); + + // Assemble parts + std::string trimmed_url = ""; + if (scheme_result == CURLUE_OK) { + trimmed_url += scheme + std::string("://"); + free(scheme); + } + trimmed_url += host; + free(host); + if (port_result == CURLUE_OK) { + trimmed_url += std::string(":") + port; + free(port); + } + return trimmed_url; +} + +std::string CCDBDownloader::prepareRedirectedURL(std::string address, std::string potentialHost) const +{ + // If it is an alien or local address it does not need preparation + if (address.find("alien:/") != std::string::npos || address.find("file:/") != std::string::npos) { + return address; + } + // Check if URL contains a scheme (protocol) + CURLU* redirected_url = curl_url(); + curl_url_set(redirected_url, CURLUPART_URL, address.c_str(), 0); + char* scheme; + CURLUcode scheme_result = curl_url_get(redirected_url, CURLUPART_SCHEME, &scheme, 0); + curl_free(scheme); + curl_url_cleanup(redirected_url); + if (scheme_result == CURLUE_OK) { + // The redirected_url contains a scheme (protocol) so there is no need for preparation + return address; + } + // If the address doesn't contain a scheme it means it is a relative url. We need to append it to the trimmed host url + // The host url must be trimmed from it's path (if it ends in one) as otherwise the redirection url would be appended after said path + return trimHostUrl(potentialHost) + address; +} + +void CCDBDownloader::transferFinished(CURL* easy_handle, CURLcode curlCode) +{ + mHandlesInUse--; + PerformData* performData; + curlEasyErrorCheck(curl_easy_getinfo(easy_handle, CURLINFO_PRIVATE, &performData)); + + curlMultiErrorCheck(curl_multi_remove_handle(mCurlMultiHandle, easy_handle)); + *performData->codeDestination = curlCode; + + bool rescheduled = false; + bool contentRetrieved = false; + + if (curlCode != 0) { + LOG(error) << "CCDBDownloader CURL transfer error - " << curl_easy_strerror(curlCode) << "\n"; + } + + switch (performData->type) { + case BLOCKING: { + --(*performData->requestsLeft); + } break; + case ASYNCHRONOUS: { + DownloaderRequestData* requestData = performData->requestData; + if (requestData->headers) { + for (auto& p : requestData->hoPair.header) { + (*requestData->headers)[p.first] = p.second; + } + } + // Log that transfer finished + long httpCode; + curl_easy_getinfo(easy_handle, CURLINFO_RESPONSE_CODE, &httpCode); + char* url; + curl_easy_getinfo(easy_handle, CURLINFO_EFFECTIVE_URL, &url); + LOG(debug) << "Transfer for " << url << " finished with code " << httpCode << "\n"; + std::string currentHost = requestData->hosts[performData->hostInd]; + std::string loggingMessage = prepareLogMessage(currentHost, requestData->userAgent, requestData->path, requestData->timestamp, requestData->headers, httpCode); + + // Get new locations based on received headers + updateLocations(&(requestData->hoPair.header), &requestData->locations, &performData->locInd); + + // React to received http code + if (200 <= httpCode && httpCode < 400) { + LOG(debug) << loggingMessage; + if (304 == httpCode) { + LOGP(debug, "Object exists but I am not serving it since it's already in your possession"); + contentRetrieved = true; + } else if (300 <= httpCode && httpCode < 400 && performData->locInd < requestData->locations.size()) { + followRedirect(performData, easy_handle, requestData->locations, rescheduled, contentRetrieved); + } else if (200 <= httpCode && httpCode < 300) { + contentRetrieved = true; // Can be overruled by following error check + } + } else { + LOG(error) << loggingMessage; + } + + // Check for errors + if (curlCode != 0) { + contentRetrieved = false; + } + + // Check if content was retrieved or scheduled to be retrieved + if (!rescheduled && !contentRetrieved) { + // Current location failed without providing 3xx http code, try next redirect for the same host + if (performData->locInd < requestData->locations.size()) { + followRedirect(performData, easy_handle, requestData->locations, rescheduled, contentRetrieved); + } + } + + // Check again because content might have been retrieved or rescheduled via a redirect + if (!rescheduled && !contentRetrieved) { + // Ran out of locations to redirect, try new host + if (++performData->hostInd < requestData->hosts.size()) { + tryNewHost(performData, easy_handle); + rescheduled = true; + } else { + LOG(error) << "File " << requestData->path << " could not be retrieved. No more hosts to try."; + } + } + + if (!rescheduled) { + // No more transfers will be done for this request, do cleanup specific for ASYNCHRONOUS calls + if (!contentRetrieved) { + if (requestData->hoPair.object) { + requestData->hoPair.object->clear(); + } + if (requestData->headers) { + (*requestData->headers)["Error"] = "An error occurred during retrieval"; + } + LOGP(alarm, "Curl request to {}, response code: {}", url, httpCode); + } else { + if (requestData->headers && requestData->headers->find("fileSize") == requestData->headers->end()) { + (*requestData->headers)["fileSize"] = fmt::format("{}", requestData->hoPair.object ? requestData->hoPair.object->size() : 0); + } + } + --(*performData->requestsLeft); + curl_slist_free_all(*performData->options); + delete requestData; + delete performData->codeDestination; + curl_easy_cleanup(easy_handle); + } + } break; + } + if (!rescheduled) { + // No more transfers will be done for this request, do general cleanup + delete performData; + } + + checkHandleQueue(); + + // Calling timeout starts a new download if a new easy_handle was added. + int running_handles; + curlMultiErrorCheck(curl_multi_socket_action(mCurlMultiHandle, CURL_SOCKET_TIMEOUT, 0, &running_handles)); + checkMultiInfo(); +} + +void CCDBDownloader::checkMultiInfo() +{ + CURLMsg* message; + int pending; + + while ((message = curl_multi_info_read(mCurlMultiHandle, &pending))) { + switch (message->msg) { + case CURLMSG_DONE: { + CURLcode code = message->data.result; + transferFinished(message->easy_handle, code); + } break; + + default: + fprintf(stderr, "CURLMSG default\n"); + break; + } + } +} + +int CCDBDownloader::startTimeout(CURLM* multi, long timeout_ms, void* userp) +{ + auto timeout = (uv_timer_t*)userp; + + if (timeout_ms < 0) { + uvErrorCheck(uv_timer_stop(timeout), SEVERE); + } else { + if (timeout_ms == 0) { + timeout_ms = 1; // Calling curlTimeout when timeout = 0 could create an infinite loop + } + uvErrorCheck(uv_timer_start(timeout, curlTimeout, timeout_ms, 0), SEVERE); + } + return 0; +} + +void CCDBDownloader::setHandleOptions(CURL* handle, PerformData* data) +{ + curlEasyErrorCheck(curl_easy_setopt(handle, CURLOPT_PRIVATE, data)); + curlEasyErrorCheck(curl_easy_setopt(handle, CURLOPT_CLOSESOCKETFUNCTION, closesocketCallback)); + curlEasyErrorCheck(curl_easy_setopt(handle, CURLOPT_CLOSESOCKETDATA, this)); + curlEasyErrorCheck(curl_easy_setopt(handle, CURLOPT_OPENSOCKETFUNCTION, opensocketCallback)); + curlEasyErrorCheck(curl_easy_setopt(handle, CURLOPT_OPENSOCKETDATA, this)); + + curlEasyErrorCheck(curl_easy_setopt(handle, CURLOPT_TIMEOUT_MS, mRequestTimeoutMS)); + curlEasyErrorCheck(curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT_MS, mConnectionTimeoutMS)); + curlEasyErrorCheck(curl_easy_setopt(handle, CURLOPT_HAPPY_EYEBALLS_TIMEOUT_MS, mHappyEyeballsHeadstartMS)); + curlEasyErrorCheck(curl_easy_setopt(handle, CURLOPT_USERAGENT, mUserAgentId.c_str())); +} + +void CCDBDownloader::checkHandleQueue() +{ + if (mHandlesToBeAdded.size() > 0) { + // Add handles without going over the limit + while (mHandlesToBeAdded.size() > 0 && mHandlesInUse < mMaxHandlesInUse) { + curlMultiErrorCheck(curl_multi_add_handle(mCurlMultiHandle, mHandlesToBeAdded.front())); + mHandlesInUse++; + mHandlesToBeAdded.erase(mHandlesToBeAdded.begin()); + } + } +} + +void CCDBDownloader::runLoop(bool noWait) +{ + uv_run(mUVLoop, noWait ? UV_RUN_NOWAIT : UV_RUN_ONCE); +} + +CURLcode CCDBDownloader::perform(CURL* handle) +{ + std::vector handleVector; + handleVector.push_back(handle); + return batchBlockingPerform(handleVector).back(); +} + +void CCDBDownloader::updateLocations(std::multimap* headerMap, std::vector* locations, int* locIndex) const +{ + std::vector newLocations; + + auto iter = headerMap->find("Location"); + if (iter != headerMap->end()) { + auto range = headerMap->equal_range("Location"); + for (auto it = range.first; it != range.second; ++it) { + if (std::find(locations->begin(), locations->end(), it->second) == locations->end()) { + if (std::find(newLocations.begin(), newLocations.end(), it->second) == newLocations.end()) { + newLocations.push_back(it->second); + } + } + } + } + + // add alternative locations (not yet included) + auto iter2 = headerMap->find("Content-Location"); + if (iter2 != headerMap->end()) { + auto range = headerMap->equal_range("Content-Location"); + for (auto it = range.first; it != range.second; ++it) { + if (std::find(locations->begin(), locations->end(), it->second) == locations->end()) { + if (std::find(newLocations.begin(), newLocations.end(), it->second) == newLocations.end()) { + newLocations.push_back(it->second); + } + } + } + } + + // Insert location list at the current location index. This assures that the provided locations will be tried first. + locations->insert(locations->begin() + (*locIndex), newLocations.begin(), newLocations.end()); +} + +std::vector CCDBDownloader::batchBlockingPerform(std::vector const& handleVector) +{ + std::vector codeVector(handleVector.size()); + size_t requestsLeft = handleVector.size(); + + for (int i = 0; i < handleVector.size(); i++) { + auto* data = new CCDBDownloader::PerformData(); + data->codeDestination = &codeVector[i]; + codeVector[i] = CURLE_FAILED_INIT; + + data->type = BLOCKING; + data->requestsLeft = &requestsLeft; + setHandleOptions(handleVector[i], data); + mHandlesToBeAdded.push_back(handleVector[i]); + } + checkHandleQueue(); + while (requestsLeft > 0) { + uv_run(mUVLoop, UV_RUN_ONCE); + } + + return codeVector; +} + +void CCDBDownloader::asynchSchedule(CURL* handle, size_t* requestCounter) +{ + (*requestCounter)++; + + CURLcode* codeVector = new CURLcode(); + + // Get data about request + DownloaderRequestData* requestData; + std::multimap* headerMap; + std::vector* hostsPool; + curl_easy_getinfo(handle, CURLINFO_PRIVATE, &requestData); + headerMap = &(requestData->hoPair.header); + hostsPool = &(requestData->hosts); + auto* options = &(requestData->optionsList); + + // Prepare temporary data about transfer + auto* data = new CCDBDownloader::PerformData(); // Freed in transferFinished + data->codeDestination = codeVector; + *codeVector = CURLE_FAILED_INIT; + + data->type = ASYNCHRONOUS; + data->requestsLeft = requestCounter; + data->hostInd = 0; + data->locInd = 0; + data->requestData = requestData; + data->options = options; + + // Prepare handle and schedule download + setHandleOptions(handle, data); + mHandlesToBeAdded.push_back(handle); + + checkHandleQueue(); + + // return codeVector; +} + +std::string CCDBDownloader::prepareLogMessage(std::string host_url, std::string userAgent, const std::string& path, long ts, const std::map* headers, long httpCode) const +{ + std::string upath{path}; + if (headers) { + auto ent = headers->find("Valid-From"); + if (ent != headers->end()) { + upath += "/" + ent->second; + } + ent = headers->find("ETag"); + if (ent != headers->end()) { + upath += "/" + ent->second; + } + } + upath.erase(remove(upath.begin(), upath.end(), '\"'), upath.end()); + return fmt::format("CcdbDownloader finished transfer {}{}{} for {} (agent_id: {}) with http code: {}", host_url, (host_url.back() == '/') ? "" : "/", upath, (ts < 0) ? getCurrentTimestamp() : ts, userAgent, httpCode); +} + +} // namespace o2::ccdb diff --git a/CCDB/src/CcdbApi.cxx b/CCDB/src/CcdbApi.cxx index f856dc6109ccd..42bc13904bf61 100644 --- a/CCDB/src/CcdbApi.cxx +++ b/CCDB/src/CcdbApi.cxx @@ -16,12 +16,15 @@ #include "CCDB/CcdbApi.h" #include "CCDB/CCDBQuery.h" + #include "CommonUtils/StringUtils.h" #include "CommonUtils/FileSystemUtils.h" #include "CommonUtils/MemFileHelper.h" -#include "MemoryResources/MemoryResources.h" +#include "Framework/DefaultsHelpers.h" +#include "Framework/DataTakingContext.h" #include #include +#include #include #include #include @@ -30,19 +33,24 @@ #include #include #include -#include +#include #include #include #include #include #include #include -#include #include #include #include #include #include +#include +#include +#include +#include "rapidjson/document.h" +#include "rapidjson/writer.h" +#include "rapidjson/stringbuffer.h" namespace o2::ccdb { @@ -52,15 +60,83 @@ using namespace std; std::mutex gIOMutex; // to protect TMemFile IO operations unique_ptr CcdbApi::mJAlienCredentials = nullptr; +/** + * Object, encapsulating a semaphore, regulating + * concurrent (multi-process) access to CCDB snapshot files. + * Intended to be used with smart pointers to achieve automatic resource + * cleanup after the smart pointer goes out of scope. + */ +class CCDBSemaphore +{ + public: + CCDBSemaphore(std::string const& cachepath, std::string const& path); + ~CCDBSemaphore(); + + private: + boost::interprocess::named_semaphore* mSem = nullptr; + std::string mSemName{}; // name under which semaphore is kept by the OS kernel +}; + +// Small registry class with the purpose that a static object +// ensures cleanup of registered semaphores even when programs +// "crash". +class SemaphoreRegistry +{ + public: + SemaphoreRegistry() = default; + ~SemaphoreRegistry(); + void add(CCDBSemaphore const* ptr); + void remove(CCDBSemaphore const* ptr); + + private: + std::unordered_set mStore; +}; +static SemaphoreRegistry gSemaRegistry; + CcdbApi::CcdbApi() { - std::string host = boost::asio::ip::host_name(); - mUniqueAgentID = fmt::format("{}-{}-{}", host, getCurrentTimestamp() / 1000, o2::utils::Str::getRandomString(6)); + using namespace o2::framework; + setUniqueAgentID(); + + DeploymentMode deploymentMode = DefaultsHelpers::deploymentMode(); + mIsCCDBDownloaderPreferred = 0; + if (deploymentMode == DeploymentMode::OnlineDDS && deploymentMode == DeploymentMode::OnlineECS && deploymentMode == DeploymentMode::OnlineAUX && deploymentMode == DeploymentMode::FST) { + mIsCCDBDownloaderPreferred = 1; + } + if (getenv("ALICEO2_ENABLE_MULTIHANDLE_CCDBAPI")) { // todo rename ALICEO2_ENABLE_MULTIHANDLE_CCDBAPI to ALICEO2_PREFER_MULTIHANDLE_CCDBAPI + mIsCCDBDownloaderPreferred = atoi(getenv("ALICEO2_ENABLE_MULTIHANDLE_CCDBAPI")); + } + mDownloader = new CCDBDownloader(); } CcdbApi::~CcdbApi() { curl_global_cleanup(); + delete mDownloader; +} + +void CcdbApi::setUniqueAgentID() +{ + mUniqueAgentID = TAlienUserAgent::BasedOnEnvironment().ToString(); +} + +bool CcdbApi::checkAlienToken() +{ +#ifdef __APPLE__ + LOG(debug) << "On macOS we simply rely on TGrid::Connect(\"alien\")."; + return true; +#endif + if (getenv("ALICEO2_CCDB_NOTOKENCHECK") && atoi(getenv("ALICEO2_CCDB_NOTOKENCHECK"))) { + return true; + } + if (getenv("JALIEN_TOKEN_CERT")) { + return true; + } + auto returncode = system("LD_PRELOAD= alien-token-info &> /dev/null"); + if (returncode == -1) { + LOG(error) << "..."; + } + return returncode == 0; } void CcdbApi::curlInit() @@ -70,10 +146,23 @@ void CcdbApi::curlInit() CcdbApi::mJAlienCredentials = std::make_unique(); CcdbApi::mJAlienCredentials->loadCredentials(); CcdbApi::mJAlienCredentials->selectPreferedCredentials(); + + // allow to configure the socket timeout of CCDBDownloader (for some tuning studies) + if (getenv("ALICEO2_CCDB_SOCKET_TIMEOUT")) { + auto timeoutMS = atoi(getenv("ALICEO2_CCDB_SOCKET_TIMEOUT")); + if (timeoutMS >= 0) { + LOG(info) << "Setting socket timeout to " << timeoutMS << " milliseconds"; + mDownloader->setKeepaliveTimeoutTime(timeoutMS); + } + } } void CcdbApi::init(std::string const& host) { + if (host.empty()) { + throw std::invalid_argument("Empty url passed CcdbApi, cannot initialize. Aborting."); + } + // if host is prefixed with "file://" this is a local snapshot // in this case we init the API in snapshot (readonly) mode constexpr const char* SNAPSHOTPREFIX = "file://"; @@ -94,21 +183,26 @@ void CcdbApi::init(std::string const& host) // In addition, we can monitor exactly which objects are fetched and what is their content. // One can also distribute so obtained caches to sites without network access. // + // THE INFORMATION BELOW IS TEMPORARILY WRONG: the functionality of checking the validity if IGNORE_VALIDITYCHECK_OF_CCDB_LOCALCACHE + // is NOT set is broken. At the moment the code is modified to behave as if the IGNORE_VALIDITYCHECK_OF_CCDB_LOCALCACHE is always set + // whenever the ALICEO2_CCDB_LOCALCACHE is defined. + // // When used with the DPL CCDB fetcher (i.e. loadFileToMemory is called), in order to prefer the available snapshot w/o its validity // check an extra variable IGNORE_VALIDITYCHECK_OF_CCDB_LOCALCACHE must be defined, otherwhise the object will be fetched from the // server after the validity check and new snapshot will be created if needed std::string snapshotReport{}; const char* cachedir = getenv("ALICEO2_CCDB_LOCALCACHE"); + namespace fs = std::filesystem; if (cachedir) { if (cachedir[0] == 0) { - mSnapshotCachePath = "."; + mSnapshotCachePath = fs::weakly_canonical(fs::absolute(".")); } else { - mSnapshotCachePath = cachedir; + mSnapshotCachePath = fs::weakly_canonical(fs::absolute(cachedir)); } snapshotReport = fmt::format("(cache snapshots to dir={}", mSnapshotCachePath); } - if (getenv("IGNORE_VALIDITYCHECK_OF_CCDB_LOCALCACHE")) { + if (cachedir) { // || getenv("IGNORE_VALIDITYCHECK_OF_CCDB_LOCALCACHE")) { mPreferSnapshotCache = true; if (mSnapshotCachePath.empty()) { LOGP(fatal, "IGNORE_VALIDITYCHECK_OF_CCDB_LOCALCACHE is defined but the ALICEO2_CCDB_LOCALCACHE is not"); @@ -119,22 +213,92 @@ void CcdbApi::init(std::string const& host) snapshotReport += ')'; } - mNeedAlienToken = host != "http://o2-ccdb.internal" && host != "http://localhost:8084" && host != "http://127.0.0.1:8084"; + mNeedAlienToken = (host.find("https://") != std::string::npos) || (host.find("alice-ccdb.cern.ch") != std::string::npos); + + // Set the curl timeout. It can be forced with an env var or it has different defaults based on the deployment mode. + if (getenv("ALICEO2_CCDB_CURL_TIMEOUT_DOWNLOAD")) { + auto timeout = atoi(getenv("ALICEO2_CCDB_CURL_TIMEOUT_DOWNLOAD")); + if (timeout >= 0) { // if valid int + mCurlTimeoutDownload = timeout; + } + } else { // set a default depending on the deployment mode + o2::framework::DeploymentMode deploymentMode = o2::framework::DefaultsHelpers::deploymentMode(); + if (deploymentMode == o2::framework::DeploymentMode::OnlineDDS || + deploymentMode == o2::framework::DeploymentMode::OnlineAUX || + deploymentMode == o2::framework::DeploymentMode::OnlineECS) { + mCurlTimeoutDownload = 15; + } else if (deploymentMode == o2::framework::DeploymentMode::Grid || + deploymentMode == o2::framework::DeploymentMode::FST) { + mCurlTimeoutDownload = 15; + } else if (deploymentMode == o2::framework::DeploymentMode::Local) { + mCurlTimeoutDownload = 5; + } + } + + if (getenv("ALICEO2_CCDB_CURL_TIMEOUT_UPLOAD")) { + auto timeout = atoi(getenv("ALICEO2_CCDB_CURL_TIMEOUT_UPLOAD")); + if (timeout >= 0) { // if valid int + mCurlTimeoutUpload = timeout; + } + } else { // set a default depending on the deployment mode + o2::framework::DeploymentMode deploymentMode = o2::framework::DefaultsHelpers::deploymentMode(); + if (deploymentMode == o2::framework::DeploymentMode::OnlineDDS || + deploymentMode == o2::framework::DeploymentMode::OnlineAUX || + deploymentMode == o2::framework::DeploymentMode::OnlineECS) { + mCurlTimeoutUpload = 3; + } else if (deploymentMode == o2::framework::DeploymentMode::Grid || + deploymentMode == o2::framework::DeploymentMode::FST) { + mCurlTimeoutUpload = 20; + } else if (deploymentMode == o2::framework::DeploymentMode::Local) { + mCurlTimeoutUpload = 20; + } + } + if (mDownloader) { + mDownloader->setRequestTimeoutTime(mCurlTimeoutDownload * 1000L); + } + + LOGP(debug, "Curl timeouts are set to: download={:2}, upload={:2} seconds", mCurlTimeoutDownload, mCurlTimeoutUpload); + + LOGP(info, "Init CcdApi with UserAgentID: {}, Host: {}{}, Curl timeouts: upload:{} download:{}", mUniqueAgentID, host, + mInSnapshotMode ? "(snapshot readonly mode)" : snapshotReport.c_str(), mCurlTimeoutUpload, mCurlTimeoutDownload); +} + +void CcdbApi::runDownloaderLoop(bool noWait) +{ + mDownloader->runLoop(noWait); +} - LOGP(info, "Init CcdApi with UserAgentID: {}, Host: {}{}", mUniqueAgentID, host, - mInSnapshotMode ? "(snapshot readonly mode)" : snapshotReport.c_str()); +// A helper function used in a few places. Updates a ROOT file with meta/header information. +void CcdbApi::updateMetaInformationInLocalFile(std::string const& filename, std::map const* headers, CCDBQuery const* querysummary) +{ + std::lock_guard guard(gIOMutex); + auto oldlevel = gErrorIgnoreLevel; + gErrorIgnoreLevel = 6001; // ignoring error messages here (since we catch with IsZombie) + TFile snapshotfile(filename.c_str(), "UPDATE"); + // The assumption is that the blob is a ROOT file + if (!snapshotfile.IsZombie()) { + if (querysummary && !snapshotfile.Get(CCDBQUERY_ENTRY)) { + snapshotfile.WriteObjectAny(querysummary, TClass::GetClass(typeid(*querysummary)), CCDBQUERY_ENTRY); + } + if (headers && !snapshotfile.Get(CCDBMETA_ENTRY)) { + snapshotfile.WriteObjectAny(headers, TClass::GetClass(typeid(*headers)), CCDBMETA_ENTRY); + } + snapshotfile.Write(); + snapshotfile.Close(); + } + gErrorIgnoreLevel = oldlevel; } /** - * Keep only the alphanumeric characters plus '_' plus '/' from the string passed in argument. + * Keep only the alphanumeric characters plus '_' plus '/' plus '.' from the string passed in argument. * @param objectName * @return a new string following the rule enounced above. */ std::string sanitizeObjectName(const std::string& objectName) { - string tmpObjectName = objectName; + std::string tmpObjectName = objectName; tmpObjectName.erase(std::remove_if(tmpObjectName.begin(), tmpObjectName.end(), - [](auto const& c) -> bool { return (!std::isalnum(c) && c != '_' && c != '/'); }), + [](auto const& c) -> bool { return (!std::isalnum(c) && c != '_' && c != '/' && c != '.'); }), tmpObjectName.end()); return tmpObjectName; } @@ -173,6 +337,10 @@ int CcdbApi::storeAsTFile_impl(const void* obj, std::type_info const& tinfo, std std::vector::size_type maxSize) const { // We need the TClass for this type; will verify if dictionary exists + if (!obj) { + LOGP(error, "nullptr is provided for object {}/{}/{}", path, startValidityTimestamp, endValidityTimestamp); + return -1; + } CcdbObjectInfo info; auto img = createObjectImage(obj, tinfo, &info); return storeAsBinaryFile(img->data(), img->size(), info.getFileName(), info.getObjectType(), @@ -201,6 +369,10 @@ int CcdbApi::storeAsBinaryFile(const char* buffer, size_t size, const std::strin sanitizedEndValidityTimestamp = getFutureTimestamp(60 * 60 * 24 * 1); } if (mInSnapshotMode) { // write local file + if (filename.empty() || buffer == nullptr || size == 0) { + LOGP(alarm, "Snapshot mode does not support headers-only upload"); + return -3; + } auto pthLoc = getSnapshotDir(mSnapshotTopPath, path); o2::utils::createDirectoriesIfAbsent(pthLoc); auto flLoc = getSnapshotFile(mSnapshotTopPath, path, filename); @@ -216,6 +388,11 @@ int CcdbApi::storeAsBinaryFile(const char* buffer, size_t size, const std::strin if (!outf.good()) { throw std::runtime_error(fmt::format("Failed to write local CCDB file {}", flLoc)); } else { + std::map metaheader(metadata); + // add time validity information + metaheader["Valid-From"] = std::to_string(startValidityTimestamp); + metaheader["Valid-Until"] = std::to_string(endValidityTimestamp); + updateMetaInformationInLocalFile(flLoc.c_str(), &metaheader); std::string metaStr{}; for (const auto& mentry : metadata) { metaStr += fmt::format("{}={};", mentry.first, mentry.second); @@ -236,15 +413,17 @@ int CcdbApi::storeAsBinaryFile(const char* buffer, size_t size, const std::strin checkMetadataKeys(metadata); if (curl != nullptr) { - struct curl_httppost* formpost = nullptr; - struct curl_httppost* lastptr = nullptr; - curl_formadd(&formpost, - &lastptr, - CURLFORM_COPYNAME, "send", - CURLFORM_BUFFER, filename.c_str(), - CURLFORM_BUFFERPTR, buffer, //.Buffer(), - CURLFORM_BUFFERLENGTH, size, - CURLFORM_END); + auto mime = curl_mime_init(curl); + auto field = curl_mime_addpart(mime); + curl_mime_name(field, "send"); + if (!filename.empty()) { + curl_mime_filedata(field, filename.c_str()); + } + if (buffer != nullptr && size > 0) { + curl_mime_data(field, buffer, size); + } else { + curl_mime_data(field, "", 0); + } struct curl_slist* headerlist = nullptr; static const char buf[] = "Expect:"; @@ -252,24 +431,29 @@ int CcdbApi::storeAsBinaryFile(const char* buffer, size_t size, const std::strin curlSetSSLOptions(curl); + curl_easy_setopt(curl, CURLOPT_MIMEPOST, mime); curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headerlist); - curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); curl_easy_setopt(curl, CURLOPT_USERAGENT, mUniqueAgentID.c_str()); + curl_easy_setopt(curl, CURLOPT_TIMEOUT, mCurlTimeoutUpload); CURLcode res = CURL_LAST; for (size_t hostIndex = 0; hostIndex < hostsPool.size() && res > 0; hostIndex++) { - string fullUrl = getFullUrlForStorage(curl, path, objectType, metadata, sanitizedStartValidityTimestamp, sanitizedEndValidityTimestamp, hostIndex); + std::string fullUrl = getFullUrlForStorage(curl, path, objectType, metadata, sanitizedStartValidityTimestamp, sanitizedEndValidityTimestamp, hostIndex); LOG(debug3) << "Full URL Encoded: " << fullUrl; /* what URL that receives this POST */ curl_easy_setopt(curl, CURLOPT_URL, fullUrl.c_str()); /* Perform the request, res will get the return code */ - res = curl_easy_perform(curl); + res = CURL_perform(curl); /* Check for errors */ if (res != CURLE_OK) { - LOGP(alarm, "curl_easy_perform() failed: {}", curl_easy_strerror(res)); + if (res == CURLE_OPERATION_TIMEDOUT) { + LOGP(alarm, "curl_easy_perform() timed out. Consider increasing the timeout using the env var `ALICEO2_CCDB_CURL_TIMEOUT_UPLOAD` (seconds), current one is {}", mCurlTimeoutUpload); + } else { // generic message + LOGP(alarm, "curl_easy_perform() failed: {}", curl_easy_strerror(res)); + } returnValue = res; } } @@ -277,10 +461,10 @@ int CcdbApi::storeAsBinaryFile(const char* buffer, size_t size, const std::strin /* always cleanup */ curl_easy_cleanup(curl); - /* then cleanup the formpost chain */ - curl_formfree(formpost); /* free slist */ curl_slist_free_all(headerlist); + /* free mime */ + curl_mime_free(mime); } else { LOGP(alarm, "curl initialization failure"); returnValue = -2; @@ -292,35 +476,39 @@ int CcdbApi::storeAsTFile(const TObject* rootObject, std::string const& path, st long startValidityTimestamp, long endValidityTimestamp, std::vector::size_type maxSize) const { // Prepare file + if (!rootObject) { + LOGP(error, "nullptr is provided for object {}/{}/{}", path, startValidityTimestamp, endValidityTimestamp); + return -1; + } CcdbObjectInfo info; auto img = createObjectImage(rootObject, &info); return storeAsBinaryFile(img->data(), img->size(), info.getFileName(), info.getObjectType(), path, metadata, startValidityTimestamp, endValidityTimestamp, maxSize); } -string CcdbApi::getFullUrlForStorage(CURL* curl, const string& path, const string& objtype, - const map& metadata, - long startValidityTimestamp, long endValidityTimestamp, int hostIndex) const +std::string CcdbApi::getFullUrlForStorage(CURL* curl, const std::string& path, const std::string& objtype, + const std::map& metadata, + long startValidityTimestamp, long endValidityTimestamp, int hostIndex) const { // Prepare timestamps - string startValidityString = getTimestampString(startValidityTimestamp < 0 ? getCurrentTimestamp() : startValidityTimestamp); - string endValidityString = getTimestampString(endValidityTimestamp < 0 ? getFutureTimestamp(60 * 60 * 24 * 1) : endValidityTimestamp); + std::string startValidityString = getTimestampString(startValidityTimestamp < 0 ? getCurrentTimestamp() : startValidityTimestamp); + std::string endValidityString = getTimestampString(endValidityTimestamp < 0 ? getFutureTimestamp(60 * 60 * 24 * 1) : endValidityTimestamp); // Get url - string url = getHostUrl(hostIndex); + std::string url = getHostUrl(hostIndex); // Build URL - string fullUrl = url + "/" + path + "/" + startValidityString + "/" + endValidityString + "/"; + std::string fullUrl = url + "/" + path + "/" + startValidityString + "/" + endValidityString + "/"; // Add type as part of metadata // we need to URL encode the object type, since in case it has special characters (like the "<", ">" for templated classes) it won't work otherwise char* objtypeEncoded = curl_easy_escape(curl, objtype.c_str(), objtype.size()); - fullUrl += "ObjectType=" + string(objtypeEncoded) + "/"; + fullUrl += "ObjectType=" + std::string(objtypeEncoded) + "/"; curl_free(objtypeEncoded); // Add general metadata for (auto& kv : metadata) { - string mfirst = kv.first; - string msecond = kv.second; + std::string mfirst = kv.first; + std::string msecond = kv.second; // same trick for the metadata as for the object type char* mfirstEncoded = curl_easy_escape(curl, mfirst.c_str(), mfirst.size()); char* msecondEncoded = curl_easy_escape(curl, msecond.c_str(), msecond.size()); - fullUrl += string(mfirstEncoded) + "=" + string(msecondEncoded) + "/"; + fullUrl += std::string(mfirstEncoded) + "=" + std::string(msecondEncoded) + "/"; curl_free(mfirstEncoded); curl_free(msecondEncoded); } @@ -328,26 +516,26 @@ string CcdbApi::getFullUrlForStorage(CURL* curl, const string& path, const strin } // todo make a single method of the one above and below -string CcdbApi::getFullUrlForRetrieval(CURL* curl, const string& path, const map& metadata, long timestamp, int hostIndex) const +std::string CcdbApi::getFullUrlForRetrieval(CURL* curl, const std::string& path, const std::map& metadata, long timestamp, int hostIndex) const { if (mInSnapshotMode) { return getSnapshotFile(mSnapshotTopPath, path); } // Prepare timestamps - string validityString = getTimestampString(timestamp < 0 ? getCurrentTimestamp() : timestamp); + std::string validityString = getTimestampString(timestamp < 0 ? getCurrentTimestamp() : timestamp); // Get host url - string hostUrl = getHostUrl(hostIndex); + std::string hostUrl = getHostUrl(hostIndex); // Build URL - string fullUrl = hostUrl + "/" + path + "/" + validityString + "/"; + std::string fullUrl = hostUrl + "/" + path + "/" + validityString + "/"; // Add metadata for (auto& kv : metadata) { - string mfirst = kv.first; - string msecond = kv.second; + std::string mfirst = kv.first; + std::string msecond = kv.second; // trick for the metadata in case it contains special characters char* mfirstEncoded = curl_easy_escape(curl, mfirst.c_str(), mfirst.size()); char* msecondEncoded = curl_easy_escape(curl, msecond.c_str(), msecond.size()); - fullUrl += string(mfirstEncoded) + "=" + string(msecondEncoded) + "/"; + fullUrl += std::string(mfirstEncoded) + "=" + std::string(msecondEncoded) + "/"; curl_free(mfirstEncoded); curl_free(msecondEncoded); } @@ -355,8 +543,8 @@ string CcdbApi::getFullUrlForRetrieval(CURL* curl, const string& path, const map } /** - * Struct to store the data we will receive from the CCDB with CURL. - */ + * Struct to store the data we will receive from the CCDB with CURL. + */ struct MemoryStruct { char* memory; unsigned int size; @@ -476,36 +664,67 @@ size_t header_map_callback(char* buffer, size_t size, size_t nitems, void* userd if (index != std::string::npos) { const auto key = boost::algorithm::trim_copy(header.substr(0, index)); const auto value = boost::algorithm::trim_copy(header.substr(index + 1)); - headers->insert(std::make_pair(key, value)); + LOGP(debug, "Adding #{} {} -> {}", headers->size(), key, value); + bool insert = true; + if (key == "Content-Length") { + auto cl = headers->find("Content-Length"); + if (cl != headers->end()) { + if (std::stol(cl->second) < stol(value)) { + headers->erase(key); + } else { + insert = false; + } + } + } + + // Keep only the first ETag encountered + if (key == "ETag") { + auto cl = headers->find("ETag"); + if (cl != headers->end()) { + insert = false; + } + } + + // Keep only the first Content-Type encountered + if (key == "Content-Type") { + auto cl = headers->find("Content-Type"); + if (cl != headers->end()) { + insert = false; + } + } + + if (insert) { + headers->insert(std::make_pair(key, value)); + } } return size * nitems; } } // namespace -void CcdbApi::initHeadersForRetrieve(CURL* curlHandle, long timestamp, std::map* headers, std::string const& etag, - const std::string& createdNotAfter, const std::string& createdNotBefore) const +void CcdbApi::initCurlHTTPHeaderOptionsForRetrieve(CURL* curlHandle, curl_slist*& option_list, long timestamp, std::map* headers, std::string const& etag, + const std::string& createdNotAfter, const std::string& createdNotBefore) const { - struct curl_slist* list = nullptr; + // struct curl_slist* list = nullptr; if (!etag.empty()) { - list = curl_slist_append(list, ("If-None-Match: " + etag).c_str()); + option_list = curl_slist_append(option_list, ("If-None-Match: " + etag).c_str()); } if (!createdNotAfter.empty()) { - list = curl_slist_append(list, ("If-Not-After: " + createdNotAfter).c_str()); + option_list = curl_slist_append(option_list, ("If-Not-After: " + createdNotAfter).c_str()); } if (!createdNotBefore.empty()) { - list = curl_slist_append(list, ("If-Not-Before: " + createdNotBefore).c_str()); + option_list = curl_slist_append(option_list, ("If-Not-Before: " + createdNotBefore).c_str()); } if (headers != nullptr) { - list = curl_slist_append(list, ("If-None-Match: " + to_string(timestamp)).c_str()); + option_list = curl_slist_append(option_list, ("If-None-Match: " + to_string(timestamp)).c_str()); curl_easy_setopt(curlHandle, CURLOPT_HEADERFUNCTION, header_map_callback<>); curl_easy_setopt(curlHandle, CURLOPT_HEADERDATA, headers); } - if (list) { - curl_easy_setopt(curlHandle, CURLOPT_HTTPHEADER, list); + if (option_list) { + curl_easy_setopt(curlHandle, CURLOPT_HTTPHEADER, option_list); } curl_easy_setopt(curlHandle, CURLOPT_USERAGENT, mUniqueAgentID.c_str()); @@ -532,27 +751,30 @@ bool CcdbApi::receiveObject(void* dataHolder, std::string const& path, std::map< CURL* curlHandle; curlHandle = curl_easy_init(); + curl_easy_setopt(curlHandle, CURLOPT_USERAGENT, mUniqueAgentID.c_str()); if (curlHandle != nullptr) { curlSetSSLOptions(curlHandle); initCurlOptionsForRetrieve(curlHandle, dataHolder, writeCallback, followRedirect); - initHeadersForRetrieve(curlHandle, timestamp, headers, etag, createdNotAfter, createdNotBefore); + curl_slist* option_list = nullptr; + initCurlHTTPHeaderOptionsForRetrieve(curlHandle, option_list, timestamp, headers, etag, createdNotAfter, createdNotBefore); long responseCode = 0; CURLcode curlResultCode = CURL_LAST; for (size_t hostIndex = 0; hostIndex < hostsPool.size() && (responseCode >= 400 || curlResultCode > 0); hostIndex++) { - string fullUrl = getFullUrlForRetrieval(curlHandle, path, metadata, timestamp, hostIndex); + std::string fullUrl = getFullUrlForRetrieval(curlHandle, path, metadata, timestamp, hostIndex); curl_easy_setopt(curlHandle, CURLOPT_URL, fullUrl.c_str()); - curlResultCode = curl_easy_perform(curlHandle); + curlResultCode = CURL_perform(curlHandle); if (curlResultCode != CURLE_OK) { LOGP(alarm, "curl_easy_perform() failed: {}", curl_easy_strerror(curlResultCode)); } else { curlResultCode = curl_easy_getinfo(curlHandle, CURLINFO_RESPONSE_CODE, &responseCode); if ((curlResultCode == CURLE_OK) && (responseCode < 300)) { + curl_slist_free_all(option_list); curl_easy_cleanup(curlHandle); return true; } else { @@ -565,6 +787,7 @@ bool CcdbApi::receiveObject(void* dataHolder, std::string const& path, std::map< } } + curl_slist_free_all(option_list); curl_easy_cleanup(curlHandle); } return false; @@ -616,7 +839,7 @@ TObject* CcdbApi::retrieveFromTFile(std::string const& path, std::map const& metadata, - long timestamp, bool preservePath, std::string const& localFileName) const + long timestamp, bool preservePath, std::string const& localFileName, std::string const& createdNotAfter, std::string const& createdNotBefore, std::map* outHeaders) const { // we setup the target path for this blob @@ -625,14 +848,14 @@ bool CcdbApi::retrieveBlob(std::string const& path, std::string const& targetdir try { o2::utils::createDirectoriesIfAbsent(fulltargetdir); } catch (std::exception e) { - LOGP(error, fmt::format("Could not create local snapshot cache directory {}, reason: {}", fulltargetdir, e.what())); + LOGP(error, "Could not create local snapshot cache directory {}, reason: {}", fulltargetdir, e.what()); return false; } o2::pmr::vector buff; std::map headers; // avoid creating snapshot via loadFileToMemory itself - loadFileToMemory(buff, path, metadata, timestamp, &headers, "", "", "", false); + loadFileToMemory(buff, path, metadata, timestamp, &headers, "", createdNotAfter, createdNotBefore, false); if ((headers.count("Error") != 0) || (buff.empty())) { LOGP(error, "Unable to find object {}/{}, Aborting", path, timestamp); return false; @@ -662,18 +885,10 @@ bool CcdbApi::retrieveBlob(std::string const& path, std::string const& targetdir } } CCDBQuery querysummary(path, metadata, timestamp); - { - std::lock_guard guard(gIOMutex); - auto oldlevel = gErrorIgnoreLevel; - gErrorIgnoreLevel = 6001; // ignoring error messages here (since we catch with IsZombie) - TFile snapshotfile(targetpath.c_str(), "UPDATE"); - // The assumption is that the blob is a ROOT file - if (!snapshotfile.IsZombie()) { - snapshotfile.WriteObjectAny(&querysummary, TClass::GetClass(typeid(querysummary)), CCDBQUERY_ENTRY); - snapshotfile.WriteObjectAny(&headers, TClass::GetClass(typeid(metadata)), CCDBMETA_ENTRY); - snapshotfile.Close(); - } - gErrorIgnoreLevel = oldlevel; + + updateMetaInformationInLocalFile(targetpath.c_str(), &headers, &querysummary); + if (outHeaders) { + *outHeaders = std::move(headers); } return true; } @@ -682,7 +897,7 @@ void CcdbApi::snapshot(std::string const& ccdbrootpath, std::string const& local { // query all subpaths to ccdbrootpath const auto allfolders = getAllFolders(ccdbrootpath); - std::map metadata; + std::map metadata; for (auto& folder : allfolders) { retrieveBlob(folder, localDir, metadata, timestamp); } @@ -746,26 +961,37 @@ void* CcdbApi::extractFromLocalFile(std::string const& filename, std::type_info if ((isSnapshotMode() || mPreferSnapshotCache) && headers->find("ETag") == headers->end()) { // generate dummy ETag to profit from the caching (*headers)["ETag"] = filename; } + if (headers->find("fileSize") == headers->end()) { + (*headers)["fileSize"] = fmt::format("{}", f.GetEND()); + } } return extractFromTFile(f, tcl); } bool CcdbApi::initTGrid() const { - if (mNeedAlienToken && !mAlienInstance) { - mAlienInstance = TGrid::Connect("alien"); + if (mNeedAlienToken && !gGrid) { + static bool allowNoToken = getenv("ALICEO2_CCDB_NOTOKENCHECK") && atoi(getenv("ALICEO2_CCDB_NOTOKENCHECK")); + if (!allowNoToken && !checkAlienToken()) { + LOG(fatal) << "Alien Token Check failed - Please get an alien token before running with https CCDB endpoint, or alice-ccdb.cern.ch!"; + } + TGrid::Connect("alien"); static bool errorShown = false; - if (!mAlienInstance && errorShown == false) { - LOG(error) << "TGrid::Connect returned nullptr. May be due to missing alien token"; + if (!gGrid && errorShown == false) { + if (allowNoToken) { + LOG(error) << "TGrid::Connect returned nullptr. May be due to missing alien token"; + } else { + LOG(fatal) << "TGrid::Connect returned nullptr. May be due to missing alien token"; + } errorShown = true; } } - return mAlienInstance != nullptr; + return gGrid != nullptr; } -void* CcdbApi::downloadAlienContent(std::string const& url, std::type_info const& tinfo) const +void* CcdbApi::downloadFilesystemContent(std::string const& url, std::type_info const& tinfo, std::map* headers) const { - if (!initTGrid()) { + if ((url.find("alien:/", 0) != std::string::npos) && !initTGrid()) { return nullptr; } std::lock_guard guard(gIOMutex); @@ -773,6 +999,9 @@ void* CcdbApi::downloadAlienContent(std::string const& url, std::type_info const if (memfile) { auto cl = tinfo2TClass(tinfo); auto content = extractFromTFile(*memfile, cl); + if (headers && headers->find("fileSize") == headers->end()) { + (*headers)["fileSize"] = fmt::format("{}", memfile->GetEND()); + } delete memfile; return content; } @@ -799,7 +1028,7 @@ void* CcdbApi::interpretAsTMemFileAndExtract(char* contentptr, size_t contentsiz } // navigate sequence of URLs until TFile content is found; object is extracted and returned -void* CcdbApi::navigateURLsAndRetrieveContent(CURL* curl_handle, std::string const& url, std::type_info const& tinfo, std::map* headers) const +void* CcdbApi::navigateURLsAndRetrieveContent(CURL* curl_handle, std::string const& url, std::type_info const& tinfo, std::map* headers) const { // a global internal data structure that can be filled with HTTP header information // static --> to avoid frequent alloc/dealloc as optimization @@ -807,8 +1036,8 @@ void* CcdbApi::navigateURLsAndRetrieveContent(CURL* curl_handle, std::string con static thread_local std::multimap headerData; // let's see first of all if the url is something specific that curl cannot handle - if (url.find("alien:/", 0) != std::string::npos) { - return downloadAlienContent(url, tinfo); + if ((url.find("alien:/", 0) != std::string::npos) || (url.find("file:/", 0) != std::string::npos)) { + return downloadFilesystemContent(url, tinfo, headers); } // add other final cases here // example root:// @@ -826,7 +1055,7 @@ void* CcdbApi::navigateURLsAndRetrieveContent(CURL* curl_handle, std::string con curlSetSSLOptions(curl_handle); - auto res = curl_easy_perform(curl_handle); + auto res = CURL_perform(curl_handle); long response_code = -1; void* content = nullptr; bool errorflag = false; @@ -839,6 +1068,9 @@ void* CcdbApi::navigateURLsAndRetrieveContent(CURL* curl_handle, std::string con if (200 <= response_code && response_code < 300) { // good response and the content is directly provided and should have been dumped into "chunk" content = interpretAsTMemFileAndExtract(chunk.memory, chunk.size, tinfo); + if (headers && headers->find("fileSize") == headers->end()) { + (*headers)["fileSize"] = fmt::format("{}", chunk.size); + } } else if (response_code == 304) { // this means the object exist but I am not serving // it since it's already in your possession @@ -914,18 +1146,7 @@ void* CcdbApi::retrieveFromTFile(std::type_info const& tinfo, std::string const& { if (!mSnapshotCachePath.empty()) { // protect this sensitive section by a multi-process named semaphore - boost::interprocess::named_semaphore* sem = nullptr; - std::hash hasher; - const auto semhashedstring = "aliceccdb" + std::to_string(hasher(mSnapshotCachePath + path)).substr(0, 16); - try { - sem = new boost::interprocess::named_semaphore(boost::interprocess::open_or_create_t{}, semhashedstring.c_str(), 1); - } catch (std::exception e) { - LOG(warn) << "Exception occurred during CCDB (cache) semaphore setup; Continuing without"; - sem = nullptr; - } - if (sem) { - sem->wait(); // wait until we can enter (no one else there) - } + auto semaphore_barrier = std::make_unique(mSnapshotCachePath, path); std::string logfile = mSnapshotCachePath + "/log"; std::fstream out(logfile, ios_base::out | ios_base::app); if (out.is_open()) { @@ -943,14 +1164,7 @@ void* CcdbApi::retrieveFromTFile(std::type_info const& tinfo, std::string const& } else { out << "CCDB-access[" << getpid() << "] ... " << mUniqueAgentID << "serving from local snapshot " << snapshotfile << "\n"; } - if (sem) { - sem->post(); - if (sem->try_wait()) { - // if nobody else is waiting remove the semaphore resource - sem->post(); - boost::interprocess::named_semaphore::remove(semhashedstring.c_str()); - } - } + auto res = extractFromLocalFile(snapshotfile, tinfo, headers); if (!snapshoting) { // if snapshot was created at this call, the log was already done logReading(path, timestamp, headers, "retrieve from snapshot"); @@ -961,14 +1175,19 @@ void* CcdbApi::retrieveFromTFile(std::type_info const& tinfo, std::string const& // normal mode follows CURL* curl_handle = curl_easy_init(); - string fullUrl = getFullUrlForRetrieval(curl_handle, path, metadata, timestamp); // todo check if function still works correctly in case mInSnapshotMode + curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, mUniqueAgentID.c_str()); + std::string fullUrl = getFullUrlForRetrieval(curl_handle, path, metadata, timestamp); // todo check if function still works correctly in case mInSnapshotMode // if we are in snapshot mode we can simply open the file; extract the object and return if (mInSnapshotMode) { - return extractFromLocalFile(fullUrl, tinfo, headers); - logReading(path, timestamp, headers, "retrieve from snapshot"); + auto res = extractFromLocalFile(fullUrl, tinfo, headers); + if (res) { + logReading(path, timestamp, headers, "retrieve from snapshot"); + } + return res; } - initHeadersForRetrieve(curl_handle, timestamp, headers, etag, createdNotAfter, createdNotBefore); + curl_slist* option_list = nullptr; + initCurlHTTPHeaderOptionsForRetrieve(curl_handle, option_list, timestamp, headers, etag, createdNotAfter, createdNotBefore); auto content = navigateURLsAndRetrieveContent(curl_handle, fullUrl, tinfo, headers); for (size_t hostIndex = 1; hostIndex < hostsPool.size() && !(content); hostIndex++) { @@ -978,6 +1197,7 @@ void* CcdbApi::retrieveFromTFile(std::type_info const& tinfo, std::string const& if (content) { logReading(path, timestamp, headers, "retrieve"); } + curl_slist_free_all(option_list); curl_easy_cleanup(curl_handle); return content; } @@ -997,7 +1217,7 @@ size_t CurlWrite_CallbackFunc_StdString2(void* contents, size_t size, size_t nme return size * nmemb; } -std::string CcdbApi::list(std::string const& path, bool latestOnly, std::string const& returnFormat) const +std::string CcdbApi::list(std::string const& path, bool latestOnly, std::string const& returnFormat, long createdNotAfter, long createdNotBefore) const { CURL* curl; CURLcode res = CURL_LAST; @@ -1007,15 +1227,22 @@ std::string CcdbApi::list(std::string const& path, bool latestOnly, std::string if (curl != nullptr) { curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, CurlWrite_CallbackFunc_StdString2); curl_easy_setopt(curl, CURLOPT_WRITEDATA, &result); + curl_easy_setopt(curl, CURLOPT_USERAGENT, mUniqueAgentID.c_str()); struct curl_slist* headers = nullptr; - headers = curl_slist_append(headers, (string("Accept: ") + returnFormat).c_str()); - headers = curl_slist_append(headers, (string("Content-Type: ") + returnFormat).c_str()); + headers = curl_slist_append(headers, (std::string("Accept: ") + returnFormat).c_str()); + headers = curl_slist_append(headers, (std::string("Content-Type: ") + returnFormat).c_str()); + if (createdNotAfter >= 0) { + headers = curl_slist_append(headers, ("If-Not-After: " + std::to_string(createdNotAfter)).c_str()); + } + if (createdNotBefore >= 0) { + headers = curl_slist_append(headers, ("If-Not-Before: " + std::to_string(createdNotBefore)).c_str()); + } curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); curlSetSSLOptions(curl); - string fullUrl; + std::string fullUrl; // Perform the request, res will get the return code for (size_t hostIndex = 0; hostIndex < hostsPool.size() && res != CURLE_OK; hostIndex++) { fullUrl = getHostUrl(hostIndex); @@ -1023,9 +1250,9 @@ std::string CcdbApi::list(std::string const& path, bool latestOnly, std::string fullUrl += path; curl_easy_setopt(curl, CURLOPT_URL, fullUrl.c_str()); - res = curl_easy_perform(curl); + res = CURL_perform(curl); if (res != CURLE_OK) { - LOGP(alarm, "curl_easy_perform() failed: {}", curl_easy_strerror(res)); + LOGP(alarm, "CURL_perform() failed: {}", curl_easy_strerror(res)); } } curl_slist_free_all(headers); @@ -1052,6 +1279,7 @@ void CcdbApi::deleteObject(std::string const& path, long timestamp) const curl = curl_easy_init(); if (curl != nullptr) { curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE"); + curl_easy_setopt(curl, CURLOPT_USERAGENT, mUniqueAgentID.c_str()); curlSetSSLOptions(curl); for (size_t hostIndex = 0; hostIndex < hostsPool.size(); hostIndex++) { @@ -1059,9 +1287,9 @@ void CcdbApi::deleteObject(std::string const& path, long timestamp) const curl_easy_setopt(curl, CURLOPT_URL, fullUrl.str().c_str()); // Perform the request, res will get the return code - res = curl_easy_perform(curl); + res = CURL_perform(curl); if (res != CURLE_OK) { - LOGP(alarm, "curl_easy_perform() failed: {}", curl_easy_strerror(res)); + LOGP(alarm, "CURL_perform() failed: {}", curl_easy_strerror(res)); } curl_easy_cleanup(curl); } @@ -1074,19 +1302,20 @@ void CcdbApi::truncate(std::string const& path) const CURLcode res; stringstream fullUrl; for (size_t i = 0; i < hostsPool.size(); i++) { - string url = getHostUrl(i); + std::string url = getHostUrl(i); fullUrl << url << "/truncate/" << path; curl = curl_easy_init(); + curl_easy_setopt(curl, CURLOPT_USERAGENT, mUniqueAgentID.c_str()); if (curl != nullptr) { curl_easy_setopt(curl, CURLOPT_URL, fullUrl.str().c_str()); curlSetSSLOptions(curl); // Perform the request, res will get the return code - res = curl_easy_perform(curl); + res = CURL_perform(curl); if (res != CURLE_OK) { - LOGP(alarm, "curl_easy_perform() failed: {}", curl_easy_strerror(res)); + LOGP(alarm, "CURL_perform() failed: {}", curl_easy_strerror(res)); } curl_easy_cleanup(curl); } @@ -1105,12 +1334,13 @@ bool CcdbApi::isHostReachable() const bool result = false; curl = curl_easy_init(); + curl_easy_setopt(curl, CURLOPT_USERAGENT, mUniqueAgentID.c_str()); if (curl) { for (size_t hostIndex = 0; hostIndex < hostsPool.size() && res != CURLE_OK; hostIndex++) { curl_easy_setopt(curl, CURLOPT_URL, mUrl.data()); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data); curlSetSSLOptions(curl); - res = curl_easy_perform(curl); + res = CURL_perform(curl); result = (res == CURLE_OK); } @@ -1156,53 +1386,140 @@ size_t header_callback(char* buffer, size_t size, size_t nitems, void* userdata) } } // namespace -std::map CcdbApi::retrieveHeaders(std::string const& path, std::map const& metadata, long timestamp) const +bool stdmap_to_jsonfile(std::map const& meta, std::string const& filename) { - CURL* curl = curl_easy_init(); - CURLcode res = CURL_LAST; - string fullUrl = getFullUrlForRetrieval(curl, path, metadata, timestamp); - std::map headers; - if (curl != nullptr) { - struct curl_slist* list = nullptr; - list = curl_slist_append(list, ("If-None-Match: " + std::to_string(timestamp)).c_str()); + // create directory structure if necessary + auto p = std::filesystem::path(filename).parent_path(); + if (!std::filesystem::exists(p)) { + std::filesystem::create_directories(p); + } - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list); + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + writer.StartObject(); + for (const auto& pair : meta) { + writer.Key(pair.first.c_str()); + writer.String(pair.second.c_str()); + } + writer.EndObject(); - /* get us the resource without a body! */ - curl_easy_setopt(curl, CURLOPT_NOBODY, 1L); - curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_map_callback<>); - curl_easy_setopt(curl, CURLOPT_HEADERDATA, &headers); - curl_easy_setopt(curl, CURLOPT_USERAGENT, mUniqueAgentID.c_str()); + // Write JSON to file + std::ofstream file(filename); + if (file.is_open()) { + file << buffer.GetString(); + file.close(); + } else { + return false; + } + return true; +} - curlSetSSLOptions(curl); +bool jsonfile_to_stdmap(std::map& meta, std::string const& filename) +{ + // Read JSON from file + std::ifstream file(filename); + if (!file.is_open()) { + std::cerr << "Failed to open file for reading." << std::endl; + return false; + } - // Perform the request, res will get the return code + std::string jsonStr((std::istreambuf_iterator(file)), std::istreambuf_iterator()); - long httpCode = 404; - CURLcode getCodeRes = CURL_LAST; - for (size_t hostIndex = 0; hostIndex < hostsPool.size() && (httpCode >= 400 || res > 0 || getCodeRes > 0); hostIndex++) { - curl_easy_setopt(curl, CURLOPT_URL, fullUrl.c_str()); - res = curl_easy_perform(curl); - if (res != CURLE_OK && res != CURLE_UNSUPPORTED_PROTOCOL) { - // We take out the unsupported protocol error because we are only querying - // header info which is returned in any case. Unsupported protocol error - // occurs sometimes because of redirection to alien for blobs. - LOG(error) << "curl_easy_perform() failed: " << curl_easy_strerror(res); - } + // Parse JSON + rapidjson::Document document; + document.Parse(jsonStr.c_str()); + + if (document.HasParseError()) { + std::cerr << "Error parsing JSON" << std::endl; + return false; + } + + // Convert JSON to std::map + for (auto itr = document.MemberBegin(); itr != document.MemberEnd(); ++itr) { + meta[itr->name.GetString()] = itr->value.GetString(); + } + return true; +} + +std::map CcdbApi::retrieveHeaders(std::string const& path, std::map const& metadata, long timestamp) const +{ + // lambda that actually does the call to the CCDB server + auto do_remote_header_call = [this, &path, &metadata, timestamp]() -> std::map { + CURL* curl = curl_easy_init(); + CURLcode res = CURL_LAST; + std::string fullUrl = getFullUrlForRetrieval(curl, path, metadata, timestamp); + std::map headers; + + if (curl != nullptr) { + struct curl_slist* list = nullptr; + list = curl_slist_append(list, ("If-None-Match: " + std::to_string(timestamp)).c_str()); + + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list); + + /* get us the resource without a body! */ + curl_easy_setopt(curl, CURLOPT_NOBODY, 1L); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_map_callback<>); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, &headers); + curl_easy_setopt(curl, CURLOPT_USERAGENT, mUniqueAgentID.c_str()); + + curlSetSSLOptions(curl); - getCodeRes = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &httpCode); + // Perform the request, res will get the return code + long httpCode = 404; + CURLcode getCodeRes = CURL_LAST; + for (size_t hostIndex = 0; hostIndex < hostsPool.size() && (httpCode >= 400 || res > 0 || getCodeRes > 0); hostIndex++) { + curl_easy_setopt(curl, CURLOPT_URL, fullUrl.c_str()); + res = CURL_perform(curl); + if (res != CURLE_OK && res != CURLE_UNSUPPORTED_PROTOCOL) { + // We take out the unsupported protocol error because we are only querying + // header info which is returned in any case. Unsupported protocol error + // occurs sometimes because of redirection to alien for blobs. + LOG(error) << "CURL_perform() failed: " << curl_easy_strerror(res); + } + getCodeRes = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &httpCode); + } + if (httpCode == 404) { + headers.clear(); + } + curl_easy_cleanup(curl); } + return headers; + }; + + if (!mSnapshotCachePath.empty()) { + // protect this sensitive section by a multi-process named semaphore + auto semaphore_barrier = std::make_unique(mSnapshotCachePath + std::string("_headers"), path); - if (httpCode == 404) { - headers.clear(); + std::string logfile = mSnapshotCachePath + "/log"; + std::fstream out(logfile, ios_base::out | ios_base::app); + if (out.is_open()) { + out << "CCDB-header-access[" << getpid() << "] of " << mUniqueAgentID << " to " << path << " timestamp " << timestamp << "\n"; } + auto snapshotfile = getSnapshotFile(mSnapshotCachePath, path + "/" + std::to_string(timestamp), "header.json"); + if (!std::filesystem::exists(snapshotfile)) { + out << "CCDB-header-access[" << getpid() << "] ... " << mUniqueAgentID << " storing to snapshot " << snapshotfile << "\n"; - curl_easy_cleanup(curl); - } + // if file not already here and valid --> snapshot it + auto meta = do_remote_header_call(); - return headers; + // cache the result + if (!stdmap_to_jsonfile(meta, snapshotfile)) { + LOG(warn) << "Failed to cache the header information to disc"; + } + return meta; + } else { + out << "CCDB-header-access[" << getpid() << "] ... " << mUniqueAgentID << "serving from local snapshot " << snapshotfile << "\n"; + std::map meta; + if (!jsonfile_to_stdmap(meta, snapshotfile)) { + LOG(warn) << "Failed to read cached information from disc"; + return do_remote_header_call(); + } + return meta; + } + } + return do_remote_header_call(); } bool CcdbApi::getCCDBEntryHeaders(std::string const& url, std::string const& etag, std::vector& headers, const std::string& agentID) @@ -1308,9 +1625,11 @@ TClass* CcdbApi::tinfo2TClass(std::type_info const& tinfo) return cl; } -void CcdbApi::updateMetadata(std::string const& path, std::map const& metadata, long timestamp, std::string const& id, long newEOV) +int CcdbApi::updateMetadata(std::string const& path, std::map const& metadata, long timestamp, std::string const& id, long newEOV) { + int ret = -1; CURL* curl = curl_easy_init(); + curl_easy_setopt(curl, CURLOPT_USERAGENT, mUniqueAgentID.c_str()); if (curl != nullptr) { CURLcode res; stringstream fullUrl; @@ -1325,12 +1644,12 @@ void CcdbApi::updateMetadata(std::string const& path, std::map CcdbApi::splitString(const std::string& str, const char* delimiters) -{ - std::vector tokens; - char stringForStrTok[str.length() + 1]; - strcpy(stringForStrTok, str.c_str()); - char* token = strtok(stringForStrTok, delimiters); - while (token != nullptr) { - tokens.emplace_back(token); - token = strtok(nullptr, delimiters); - } - return tokens; + return ret; } void CcdbApi::initHostsPool(std::string hosts) { - hostsPool = splitString(hosts, ",;"); + hostsPool.clear(); + auto splitted = hosts | std::views::transform([](char c) { return (c == ';') ? ',' : c; }) | std::views::split(','); + for (auto&& part : splitted) { + hostsPool.emplace_back(part.begin(), part.end()); + } } std::string CcdbApi::getHostUrl(int hostIndex) const @@ -1377,224 +1691,332 @@ std::string CcdbApi::getHostUrl(int hostIndex) const return hostsPool.at(hostIndex); } -void CcdbApi::loadFileToMemory(o2::pmr::vector& dest, std::string const& path, - std::map const& metadata, long timestamp, - std::map* headers, std::string const& etag, - const std::string& createdNotAfter, const std::string& createdNotBefore, bool considerSnapshot) const +void CcdbApi::scheduleDownload(RequestContext& requestContext, size_t* requestCounter) const { - LOGP(debug, "loadFileToMemory {} ETag=[{}]", path, etag); - - // if we are in snapshot mode we can simply open the file, unless the etag is non-empty: - // this would mean that the object was is already fetched and in this mode we don't to validity checks! - bool createSnapshot = considerSnapshot && !mSnapshotCachePath.empty(); // create snaphot if absent - int fromSnapshot = 0; - boost::interprocess::named_semaphore* sem = nullptr; - std::string semhashedstring{}, snapshotpath{}, logfile{}; - std::unique_ptr logStream; - auto sem_release = [&sem, &semhashedstring, path, this]() { - if (sem) { - sem->post(); - if (sem->try_wait()) { // if nobody else is waiting remove the semaphore resource - sem->post(); - boost::interprocess::named_semaphore::remove(semhashedstring.c_str()); - } - } + auto data = new DownloaderRequestData(); // Deleted in transferFinished of CCDBDownloader.cxx + data->hoPair.object = &requestContext.dest; + + std::function localContentCallback = [this, &requestContext](std::string url) { + return this->loadLocalContentToMemory(requestContext.dest, url); }; - if (createSnapshot) { // create named semaphore - std::hash hasher; - semhashedstring = "aliceccdb" + std::to_string(hasher(mSnapshotCachePath + path)).substr(0, 16); + auto writeCallback = [](void* contents, size_t size, size_t nmemb, void* chunkptr) { + auto& ho = *static_cast(chunkptr); + auto& chunk = *ho.object; + size_t realsize = size * nmemb, sz = 0; + ho.counter++; try { - sem = new boost::interprocess::named_semaphore(boost::interprocess::open_or_create_t{}, semhashedstring.c_str(), 1); + if (chunk.capacity() < chunk.size() + realsize) { + // estimate headers size when converted to annotated text string + const char hannot[] = "header"; + size_t hsize = getFlatHeaderSize(ho.header); + auto cl = ho.header.find("Content-Length"); + if (cl != ho.header.end()) { + size_t sizeFromHeader = std::stol(cl->second); + sz = hsize + std::max(chunk.size() * (sizeFromHeader ? 1 : 2) + realsize, sizeFromHeader); + } else { + sz = hsize + std::max(chunk.size() * 2, chunk.size() + realsize); + // LOGP(debug, "SIZE IS NOT IN HEADER, allocate {}", sz); + } + chunk.reserve(sz); + } + char* contC = (char*)contents; + chunk.insert(chunk.end(), contC, contC + realsize); } catch (std::exception e) { - LOG(warn) << "Exception occurred during CCDB (cache) semaphore setup; Continuing without"; - sem = nullptr; + // LOGP(alarm, "failed to reserve {} bytes in CURL write callback (realsize = {}): {}", sz, realsize, e.what()); + realsize = 0; } - if (sem) { - sem->wait(); // wait until we can enter (no one else there) + return realsize; + }; + + CURL* curl_handle = curl_easy_init(); + curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, mUniqueAgentID.c_str()); + std::string fullUrl = getFullUrlForRetrieval(curl_handle, requestContext.path, requestContext.metadata, requestContext.timestamp); + curl_slist* options_list = nullptr; + initCurlHTTPHeaderOptionsForRetrieve(curl_handle, options_list, requestContext.timestamp, &requestContext.headers, + requestContext.etag, requestContext.createdNotAfter, requestContext.createdNotBefore); + + data->headers = &requestContext.headers; + data->hosts = hostsPool; + data->path = requestContext.path; + data->timestamp = requestContext.timestamp; + data->localContentCallback = localContentCallback; + data->userAgent = mUniqueAgentID; + data->optionsList = options_list; + + curl_easy_setopt(curl_handle, CURLOPT_URL, fullUrl.c_str()); + initCurlOptionsForRetrieve(curl_handle, (void*)(&data->hoPair), writeCallback, false); + curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, header_map_callbackhoPair.header)>); + curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, (void*)&(data->hoPair.header)); + curl_easy_setopt(curl_handle, CURLOPT_PRIVATE, (void*)data); + curlSetSSLOptions(curl_handle); + + asynchPerform(curl_handle, requestCounter); +} + +std::string CcdbApi::determineSemaphoreName(std::string const& basedir, std::string const& ccdbpath) +{ + std::hash hasher; + std::string semhashedstring = "aliceccdb" + std::to_string(hasher(basedir + ccdbpath)).substr(0, 16); + return semhashedstring; +} + +boost::interprocess::named_semaphore* CcdbApi::createNamedSemaphore(std::string const& path) const +{ + std::string semhashedstring = determineSemaphoreName(mSnapshotCachePath, path); + // LOG(info) << "Creating named semaphore with name " << semhashedstring.c_str(); + try { + return new boost::interprocess::named_semaphore(boost::interprocess::open_or_create_t{}, semhashedstring.c_str(), 1); + } catch (std::exception e) { + LOG(warn) << "Exception occurred during CCDB (cache) semaphore setup; Continuing without"; + return nullptr; + } +} + +void CcdbApi::releaseNamedSemaphore(boost::interprocess::named_semaphore* sem, std::string const& path) const +{ + if (sem) { + sem->post(); + if (sem->try_wait()) { // if nobody else is waiting remove the semaphore resource + sem->post(); + boost::interprocess::named_semaphore::remove(determineSemaphoreName(mSnapshotCachePath, path).c_str()); } - logfile = mSnapshotCachePath + "/log"; - logStream = std::make_unique(logfile, ios_base::out | ios_base::app); - if (logStream->is_open()) { - *logStream.get() << "CCDB-access[" << getpid() << "] of " << mUniqueAgentID << " to " << path << " timestamp " << timestamp << " for load to memory\n"; + } +} + +bool CcdbApi::removeSemaphore(std::string const& semaname, bool remove) +{ + // removes a given named semaphore from the system + try { + boost::interprocess::named_semaphore semaphore(boost::interprocess::open_only, semaname.c_str()); + std::cout << "Found CCDB semaphore: " << semaname << "\n"; + if (remove) { + auto success = boost::interprocess::named_semaphore::remove(semaname.c_str()); + if (success) { + std::cout << "Removed CCDB semaphore: " << semaname << "\n"; + } + return success; } + return true; + } catch (std::exception const& e) { + // no EXISTING under this name semaphore found + // nothing to be done } + return false; +} - if (mInSnapshotMode) { // file must be there, otherwise a fatal will be produced - loadFileToMemory(dest, getSnapshotFile(mSnapshotTopPath, path), headers); +// helper function checking for leaking semaphores associated to CCDB cache files and removing them +// walks a local CCDB snapshot tree and checks +void CcdbApi::removeLeakingSemaphores(std::string const& snapshotdir, bool remove) +{ + namespace fs = std::filesystem; + std::string fileName{"snapshot.root"}; + try { + auto absolutesnapshotdir = fs::weakly_canonical(fs::absolute(snapshotdir)); + for (const auto& entry : fs::recursive_directory_iterator(absolutesnapshotdir)) { + if (entry.is_directory()) { + const fs::path& currentDir = fs::canonical(fs::absolute(entry.path())); + fs::path filePath = currentDir / fileName; + if (fs::exists(filePath) && fs::is_regular_file(filePath)) { + std::cout << "Directory with file '" << fileName << "': " << currentDir << std::endl; + + // we need to obtain the path relative to snapshotdir + auto pathtokens = o2::utils::Str::tokenize(currentDir, '/', true); + auto numtokens = pathtokens.size(); + if (numtokens < 3) { + // cannot be a CCDB path + continue; + } + // path are last 3 entries + std::string path = pathtokens[numtokens - 3] + "/" + pathtokens[numtokens - 2] + "/" + pathtokens[numtokens - 1]; + auto semaname = o2::ccdb::CcdbApi::determineSemaphoreName(absolutesnapshotdir, path); + removeSemaphore(semaname, remove); + } + } + } + } catch (std::exception const& e) { + LOG(info) << "Semaphore search had exception " << e.what(); + } +} + +void CcdbApi::getFromSnapshot(bool createSnapshot, std::string const& path, + long timestamp, std::map& headers, + std::string& snapshotpath, o2::pmr::vector& dest, int& fromSnapshot, std::string const& etag) const +{ + if (createSnapshot) { // create named semaphore + std::string logfile = mSnapshotCachePath + "/log"; + std::fstream logStream = std::fstream(logfile, ios_base::out | ios_base::app); + if (logStream.is_open()) { + logStream << "CCDB-access[" << getpid() << "] of " << mUniqueAgentID << " to " << path << " timestamp " << timestamp << " for load to memory\n"; + } + } + if (mInSnapshotMode) { // file must be there, otherwise a fatal will be produced; + if (etag.empty()) { + loadFileToMemory(dest, getSnapshotFile(mSnapshotTopPath, path), &headers); + } fromSnapshot = 1; - } else if (mPreferSnapshotCache && std::filesystem::exists(snapshotpath = getSnapshotFile(mSnapshotCachePath, path))) { + } else if (mPreferSnapshotCache && std::filesystem::exists(snapshotpath)) { // if file is available, use it, otherwise cache it below from the server. Do this only when etag is empty since otherwise the object was already fetched and cached if (etag.empty()) { - loadFileToMemory(dest, snapshotpath, headers); + loadFileToMemory(dest, snapshotpath, &headers); } fromSnapshot = 2; - } else { // look on the server - CURL* curl_handle = curl_easy_init(); - string fullUrl = getFullUrlForRetrieval(curl_handle, path, metadata, timestamp); - - initHeadersForRetrieve(curl_handle, timestamp, headers, etag, createdNotAfter, createdNotBefore); - - navigateURLsAndLoadFileToMemory(dest, curl_handle, fullUrl, headers); - - for (size_t hostIndex = 1; hostIndex < hostsPool.size() && isMemoryFileInvalid(dest); hostIndex++) { - fullUrl = getFullUrlForRetrieval(curl_handle, path, metadata, timestamp, hostIndex); - loadFileToMemory(dest, fullUrl, headers); // headers loaded from the file in case of the snapshot reading only - } - curl_easy_cleanup(curl_handle); } +} - if (dest.empty()) { - sem_release(); - return; // nothing was fetched: either cached value is good or error was produced - } - // !considerSnapshot means that the call was made by retrieve for snapshoting reasons - logReading(path, timestamp, headers, fmt::format("{}{}", considerSnapshot ? "load to memory" : "retrieve", fromSnapshot ? " from snapshot" : "")); +void CcdbApi::saveSnapshot(RequestContext& requestContext) const +{ + // Consider saving snapshot + if (!mSnapshotCachePath.empty() && !(mInSnapshotMode && mSnapshotTopPath == mSnapshotCachePath)) { // store in the snapshot only if the object was not read from the snapshot + auto semaphore_barrier = std::make_unique(mSnapshotCachePath, requestContext.path); - // are we asked to create a snapshot ? - if (createSnapshot && fromSnapshot != 2 && !(mInSnapshotMode && mSnapshotTopPath == mSnapshotCachePath)) { // store in the snapshot only if the object was not read from the snapshot - auto snapshotdir = getSnapshotDir(mSnapshotCachePath, path); - snapshotpath = getSnapshotFile(mSnapshotCachePath, path); + auto snapshotdir = getSnapshotDir(mSnapshotCachePath, requestContext.path); + std::string snapshotpath = getSnapshotFile(mSnapshotCachePath, requestContext.path); o2::utils::createDirectoriesIfAbsent(snapshotdir); - if (logStream->is_open()) { - *logStream.get() << "CCDB-access[" << getpid() << "] ... " << mUniqueAgentID << " downloading to snapshot " << snapshotpath << " from memory\n"; + std::fstream logStream; + if (logStream.is_open()) { + logStream << "CCDB-access[" << getpid() << "] ... " << mUniqueAgentID << " downloading to snapshot " << snapshotpath << " from memory\n"; } { // dump image to a file - LOGP(debug, "creating snapshot {} -> {}", path, snapshotpath); - CCDBQuery querysummary(path, metadata, timestamp); + LOGP(debug, "creating snapshot {} -> {}", requestContext.path, snapshotpath); + CCDBQuery querysummary(requestContext.path, requestContext.metadata, requestContext.timestamp); { std::ofstream objFile(snapshotpath, std::ios::out | std::ofstream::binary); - std::copy(dest.begin(), dest.end(), std::ostreambuf_iterator(objFile)); + std::copy(requestContext.dest.begin(), requestContext.dest.end(), std::ostreambuf_iterator(objFile)); } // now open the same file as root file and store metadata - std::lock_guard guard(gIOMutex); - auto oldlevel = gErrorIgnoreLevel; - gErrorIgnoreLevel = 6001; // ignoring error messages here (since we catch with IsZombie) - TFile snapshot(snapshotpath.c_str(), "UPDATE"); // the assumption is that the blob is a ROOT file - if (!snapshot.IsZombie()) { - snapshot.WriteObjectAny(&querysummary, TClass::GetClass(typeid(querysummary)), CCDBQUERY_ENTRY); - if (headers) { - snapshot.WriteObjectAny(headers, TClass::GetClass(typeid(metadata)), CCDBMETA_ENTRY); - } - } - snapshot.Close(); - gErrorIgnoreLevel = oldlevel; + updateMetaInformationInLocalFile(snapshotpath, &requestContext.headers, &querysummary); } } - sem_release(); } -// navigate sequence of URLs until TFile content is found; object is extracted and returned -void CcdbApi::navigateURLsAndLoadFileToMemory(o2::pmr::vector& dest, CURL* curl_handle, std::string const& url, std::map* headers) const +void CcdbApi::loadFileToMemory(std::vector& dest, std::string const& path, + std::map const& metadata, long timestamp, + std::map* headers, std::string const& etag, + const std::string& createdNotAfter, const std::string& createdNotBefore, bool considerSnapshot) const { - // a global internal data structure that can be filled with HTTP header information - // static --> to avoid frequent alloc/dealloc as optimization - // not sure if thread_local takes away that benefit - static thread_local std::multimap headerData; - - // let's see first of all if the url is something specific that curl cannot handle - if (url.find("alien:/", 0) != std::string::npos) { - return loadFileToMemory(dest, url, nullptr); // headers loaded from the file in case of the snapshot reading only + o2::pmr::vector destP; + destP.reserve(dest.size()); + loadFileToMemory(destP, path, metadata, timestamp, headers, etag, createdNotAfter, createdNotBefore, considerSnapshot); + dest.clear(); + dest.reserve(destP.size()); + for (const auto c : destP) { + dest.push_back(c); } - // otherwise make an HTTP/CURL request - bool errorflag = false; - auto signalError = [&chunk = dest, &errorflag]() { - chunk.clear(); - chunk.reserve(1); - errorflag = true; - }; - auto writeCallBack = [](void* contents, size_t size, size_t nmemb, void* chunkptr) { - o2::pmr::vector& chunk = *static_cast*>(chunkptr); - size_t realsize = size * nmemb; - try { - chunk.reserve(chunk.size() + realsize); - char* contC = (char*)contents; - chunk.insert(chunk.end(), contC, contC + realsize); - } catch (std::exception e) { - LOGP(info, "failed to expand by {} bytes chunk provided to CURL: {}", realsize, e.what()); - realsize = 0; +} + +void CcdbApi::loadFileToMemory(o2::pmr::vector& dest, std::string const& path, + std::map const& metadata, long timestamp, + std::map* headers, std::string const& etag, + const std::string& createdNotAfter, const std::string& createdNotBefore, bool considerSnapshot) const +{ + RequestContext requestContext(dest, metadata, *headers); + requestContext.path = path; + // std::map metadataCopy = metadata; // Create a copy because metadata will be passed as a pointer so it cannot be constant. The const in definition is for backwards compatability. + // requestContext.metadata = metadataCopy; + requestContext.timestamp = timestamp; + requestContext.etag = etag; + requestContext.createdNotAfter = createdNotAfter; + requestContext.createdNotBefore = createdNotBefore; + requestContext.considerSnapshot = considerSnapshot; + std::vector contexts = {requestContext}; + vectoredLoadFileToMemory(contexts); +} + +void CcdbApi::appendFlatHeader(o2::pmr::vector& dest, const std::map& headers) +{ + size_t hsize = getFlatHeaderSize(headers), cnt = dest.size(); + dest.resize(cnt + hsize); + auto addString = [&dest, &cnt](const std::string& s) { + for (char c : s) { + dest[cnt++] = c; } - return realsize; + dest[cnt++] = 0; }; - // specify URL to get - curl_easy_setopt(curl_handle, CURLOPT_URL, url.c_str()); - initCurlOptionsForRetrieve(curl_handle, (void*)&dest, writeCallBack, false); - curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, header_map_callback); - headerData.clear(); - curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, (void*)&headerData); - curlSetSSLOptions(curl_handle); + for (auto& h : headers) { + addString(h.first); + addString(h.second); + } + *reinterpret_cast(&dest[cnt]) = hsize; // store size + std::memcpy(&dest[cnt + sizeof(int)], FlatHeaderAnnot, sizeof(FlatHeaderAnnot)); // annotate the flattened headers map +} - auto res = curl_easy_perform(curl_handle); - long response_code = -1; - if (res == CURLE_OK && curl_easy_getinfo(curl_handle, CURLINFO_RESPONSE_CODE, &response_code) == CURLE_OK) { - if (headers) { - for (auto& p : headerData) { - (*headers)[p.first] = p.second; +void CcdbApi::navigateSourcesAndLoadFile(RequestContext& requestContext, int& fromSnapshot, size_t* requestCounter) const +{ + LOGP(debug, "loadFileToMemory {} ETag=[{}]", requestContext.path, requestContext.etag); + bool createSnapshot = requestContext.considerSnapshot && !mSnapshotCachePath.empty(); // create snaphot if absent + + std::string snapshotpath; + if (mInSnapshotMode || std::filesystem::exists(snapshotpath = getSnapshotFile(mSnapshotCachePath, requestContext.path))) { + auto semaphore_barrier = std::make_unique(mSnapshotCachePath, requestContext.path); + // if we are in snapshot mode we can simply open the file, unless the etag is non-empty: + // this would mean that the object was is already fetched and in this mode we don't to validity checks! + getFromSnapshot(createSnapshot, requestContext.path, requestContext.timestamp, requestContext.headers, snapshotpath, requestContext.dest, fromSnapshot, requestContext.etag); + } else { // look on the server + scheduleDownload(requestContext, requestCounter); + } +} + +void CcdbApi::vectoredLoadFileToMemory(std::vector& requestContexts) const +{ + std::vector fromSnapshots(requestContexts.size()); + size_t requestCounter = 0; + + // Get files from snapshots and schedule downloads + for (int i = 0; i < requestContexts.size(); i++) { + // navigateSourcesAndLoadFile either retrieves file from snapshot immediately, or schedules it to be downloaded when mDownloader->runLoop is ran at a later time + auto& requestContext = requestContexts.at(i); + navigateSourcesAndLoadFile(requestContext, fromSnapshots.at(i), &requestCounter); + } + + // Download the rest + while (requestCounter > 0) { + mDownloader->runLoop(0); + } + + // Save snapshots + for (int i = 0; i < requestContexts.size(); i++) { + auto& requestContext = requestContexts.at(i); + if (!requestContext.dest.empty()) { + logReading(requestContext.path, requestContext.timestamp, &requestContext.headers, + fmt::format("{}{}", requestContext.considerSnapshot ? "load to memory" : "retrieve", fromSnapshots.at(i) ? " from snapshot" : "")); + if (requestContext.considerSnapshot && fromSnapshots.at(i) != 2) { + saveSnapshot(requestContext); } } - if (200 <= response_code && response_code < 300) { - // good response and the content is directly provided and should have been dumped into "chunk" - } else if (response_code == 304) { - LOGP(debug, "Object exists but I am not serving it since it's already in your possession"); - } - // this is a more general redirection - else if (300 <= response_code && response_code < 400) { - // we try content locations in order of appearance until one succeeds - // 1st: The "Location" field - // 2nd: Possible "Content-Location" fields - Location field - // some locations are relative to the main server so we need to fix/complement them - auto complement_Location = [this](std::string const& loc) { - if (loc[0] == '/') { - // if it's just a path (noticed by trailing '/' we prepend the server url - return getURL() + loc; - } - return loc; - }; + } +} - std::vector locs; - auto iter = headerData.find("Location"); - if (iter != headerData.end()) { - locs.push_back(complement_Location(iter->second)); - } - // add alternative locations (not yet included) - auto iter2 = headerData.find("Content-Location"); - if (iter2 != headerData.end()) { - auto range = headerData.equal_range("Content-Location"); - for (auto it = range.first; it != range.second; ++it) { - if (std::find(locs.begin(), locs.end(), it->second) == locs.end()) { - locs.push_back(complement_Location(it->second)); - } - } - } - for (auto& l : locs) { - if (l.size() > 0) { - LOG(debug) << "Trying content location " << l; - navigateURLsAndLoadFileToMemory(dest, curl_handle, l, headers); - if (dest.size()) { /* or other success marker in future */ - break; - } - } - } - } else if (response_code == 404) { - LOG(error) << "Requested resource does not exist: " << url; - signalError(); +bool CcdbApi::loadLocalContentToMemory(o2::pmr::vector& dest, std::string& url) const +{ + if (url.find("alien:/", 0) != std::string::npos) { + std::map localHeaders; + loadFileToMemory(dest, url, &localHeaders, false); + auto it = localHeaders.find("Error"); + if (it != localHeaders.end() && it->second == "An error occurred during retrieval") { + return false; } else { - LOG(error) << "Error in fetching object " << url << ", curl response code:" << response_code; - signalError(); + return true; } - } else { - LOGP(alarm, "Curl request to {} failed with result {}, response code: {}", url, int(res), response_code); - signalError(); } - // indicate that an error occurred ---> used by caching layers (such as CCDBManager) - if (errorflag && headers) { - (*headers)["Error"] = "An error occurred during retrieval"; + if ((url.find("file:/", 0) != std::string::npos)) { + std::string path = url.substr(7); + if (std::filesystem::exists(path)) { + std::map localHeaders; + loadFileToMemory(dest, url, &localHeaders, o2::utils::Str::endsWith(path, ".root")); + auto it = localHeaders.find("Error"); + if (it != localHeaders.end() && it->second == "An error occurred during retrieval") { + return false; + } else { + return true; + } + } } - return; + return false; } -void CcdbApi::loadFileToMemory(o2::pmr::vector& dest, const std::string& path, std::map* localHeaders) const +void CcdbApi::loadFileToMemory(o2::pmr::vector& dest, const std::string& path, std::map* localHeaders, bool fetchLocalMetaData) const { // Read file to memory as vector. For special case of the locally cached file retriev metadata stored directly in the file constexpr size_t MaxCopySize = 0x1L << 25; @@ -1642,7 +2064,7 @@ void CcdbApi::loadFileToMemory(o2::pmr::vector& dest, const std::string& p totalread += nread; } while (nread == (long)MaxCopySize); - if (localHeaders) { + if (localHeaders && fetchLocalMetaData) { TMemFile memFile("name", const_cast(dest.data()), dest.size(), "READ"); auto storedmeta = (std::map*)extractFromTFile(memFile, TClass::GetClass("std::map"), CCDBMETA_ENTRY); if (storedmeta) { @@ -1652,6 +2074,9 @@ void CcdbApi::loadFileToMemory(o2::pmr::vector& dest, const std::string& p if ((isSnapshotMode() || mPreferSnapshotCache) && localHeaders->find("ETag") == localHeaders->end()) { // generate dummy ETag to profit from the caching (*localHeaders)["ETag"] = path; } + if (localHeaders->find("fileSize") == localHeaders->end()) { + (*localHeaders)["fileSize"] = fmt::format("{}", memFile.GetEND()); + } } return; } @@ -1698,4 +2123,74 @@ void CcdbApi::logReading(const std::string& path, long ts, const std::mapasynchSchedule(handle, requestCounter); +} + +CURLcode CcdbApi::CURL_perform(CURL* handle) const +{ + if (mIsCCDBDownloaderPreferred) { + return mDownloader->perform(handle); + } + CURLcode result; + for (int i = 1; i <= mCurlRetries && (result = curl_easy_perform(handle)) != CURLE_OK; i++) { + usleep(mCurlDelayRetries * i); + } + return result; +} + +/** + * Object, encapsulating a semaphore, regulating + * concurrent (multi-process) access to CCDB snapshot files. + */ +CCDBSemaphore::CCDBSemaphore(std::string const& snapshotpath, std::string const& path) +{ + LOG(debug) << "Entering semaphore barrier"; + mSemName = CcdbApi::determineSemaphoreName(snapshotpath, path); + try { + mSem = new boost::interprocess::named_semaphore(boost::interprocess::open_or_create_t{}, mSemName.c_str(), 1); + } catch (std::exception e) { + LOG(warn) << "Exception occurred during CCDB (cache) semaphore setup; Continuing without"; + mSem = nullptr; + } + // automatically wait + if (mSem) { + gSemaRegistry.add(this); + mSem->wait(); + } +} + +CCDBSemaphore::~CCDBSemaphore() +{ + LOG(debug) << "Ending semaphore barrier"; + if (mSem) { + mSem->post(); + if (mSem->try_wait()) { // if nobody else is waiting remove the semaphore resource + mSem->post(); + boost::interprocess::named_semaphore::remove(mSemName.c_str()); + } + gSemaRegistry.remove(this); + } +} + +SemaphoreRegistry::~SemaphoreRegistry() +{ + LOG(debug) << "Cleaning up semaphore registry with count " << mStore.size(); + for (auto& s : mStore) { + delete s; + mStore.erase(s); + } +} + +void SemaphoreRegistry::add(CCDBSemaphore const* ptr) +{ + mStore.insert(ptr); +} + +void SemaphoreRegistry::remove(CCDBSemaphore const* ptr) +{ + mStore.erase(ptr); +} + +} // namespace o2::ccdb diff --git a/CCDB/src/CleanCCDBSemaphores.cxx b/CCDB/src/CleanCCDBSemaphores.cxx new file mode 100644 index 0000000000000..f107cde21a051 --- /dev/null +++ b/CCDB/src/CleanCCDBSemaphores.cxx @@ -0,0 +1,62 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include "CCDB/CcdbApi.h" +#include +#include + +namespace bpo = boost::program_options; + +bool initOptionsAndParse(bpo::options_description& options, int argc, char* argv[], bpo::variables_map& vm) +{ + options.add_options()( + "cachepath,p", bpo::value()->default_value("ccdb"), "path to whole CCDB cache dir as a basis for semaphore search")( + "sema,s", bpo::value()->default_value(""), "Specific named semaphore to be remove")( + "help,h", "Produce help message."); + + try { + bpo::store(parse_command_line(argc, argv, options), vm); + // help + if (vm.count("help")) { + std::cout << options << std::endl; + return false; + } + bpo::notify(vm); + } catch (const bpo::error& e) { + std::cerr << e.what() << "\n\n"; + std::cerr << "Error parsing command line arguments; Available options:\n"; + + std::cerr << options << std::endl; + return false; + } + return true; +} + +// A simple tool to clean CCDB related semaphores +int main(int argc, char* argv[]) +{ + bpo::options_description options("Tool to find and remove leaking CCDB semaphore from the system"); + bpo::variables_map vm; + if (!initOptionsAndParse(options, argc, argv, vm)) { + return 1; + } + + std::string sema = vm["sema"].as(); + if (sema.size() > 0) { + if (o2::ccdb::CcdbApi::removeSemaphore(sema, true)) { + std::cout << "Successfully removed " << sema << "\n"; + } + } + + std::string path = vm["cachepath"].as(); + o2::ccdb::CcdbApi::removeLeakingSemaphores(path, true); + return 0; +} diff --git a/CCDB/src/DownloadCCDBFile.cxx b/CCDB/src/DownloadCCDBFile.cxx index 73da631751b11..2876e95257989 100644 --- a/CCDB/src/DownloadCCDBFile.cxx +++ b/CCDB/src/DownloadCCDBFile.cxx @@ -22,12 +22,14 @@ namespace bpo = boost::program_options; bool initOptionsAndParse(bpo::options_description& options, int argc, char* argv[], bpo::variables_map& vm) { options.add_options()( - "host", bpo::value()->default_value("ccdb-test.cern.ch:8080"), "CCDB server")( + "host", bpo::value()->default_value("alice-ccdb.cern.ch"), "CCDB server")( "path,p", bpo::value>()->multitoken(), "CCDB path (identifies the object) [or space separated list of paths for batch processing]")( "dest,d", bpo::value()->default_value("./"), "destination path")( "no-preserve-path", "Do not preserve path structure. If not set, the full path structure -- reflecting the '--path' argument will be put.")( "outfile,o", bpo::value()->default_value("snapshot.root"), "Name of output file. If set to \"\", the name will be determined from the uploaded content. (Will be the same in case of batch downloading multiple paths.)")( "timestamp,t", bpo::value()->default_value(-1), "timestamp in ms - default -1 = now")( + "created-not-before", bpo::value()->default_value(0), "CCDB created-not-before time (Time Machine)")( + "created-not-after", bpo::value()->default_value(3385078236000), "CCDB created-not-after time (Time Machine)")( "help,h", "Produce help message."); try { @@ -78,11 +80,13 @@ int main(int argc, char* argv[]) std::cout << "Querying host " << host << " for path(s) " << paths[0] << " ... and timestamp " << timestamp << "\n"; bool no_preserve_path = vm.count("no-preserve-path") == 0; auto filename = vm["outfile"].as(); + auto notBefore = vm["created-not-before"].as(); + auto notAfter = vm["created-not-after"].as(); bool success = true; for (auto& p : paths) { // could even multi-thread this - success |= api.retrieveBlob(p, dest, filter, timestamp, no_preserve_path, filename); + success |= api.retrieveBlob(p, dest, filter, timestamp, no_preserve_path, filename, std::to_string(notAfter), std::to_string(notBefore)); } return success ? 0 : 1; } diff --git a/CCDB/src/InspectCCDBFile.cxx b/CCDB/src/InspectCCDBFile.cxx index 46dc5d0706bab..975f2e00a0d49 100644 --- a/CCDB/src/InspectCCDBFile.cxx +++ b/CCDB/src/InspectCCDBFile.cxx @@ -15,47 +15,105 @@ #include "TFile.h" #include "TKey.h" #include +#include -// a simple tool to inspect/print metadata content of ROOT files containing CCDB entries +namespace bpo = boost::program_options; + +bool initOptionsAndParse(bpo::options_description& options, int argc, char* argv[], bpo::variables_map& vm) +{ + options.add_options()( + "files,f", bpo::value>()->multitoken(), "Space separated list of ROOT files holding (downloaded) CCDB object")( + "check-timestamp,t", bpo::value()->default_value(-1), "Checks that validity of objects is compatible with this timestamp. In millisecond")( + "help,h", "Produce help message."); + + try { + bpo::store(parse_command_line(argc, argv, options), vm); + // help + if (vm.count("help")) { + std::cout << options << std::endl; + return false; + } + bpo::notify(vm); + } catch (const bpo::error& e) { + std::cerr << e.what() << "\n\n"; + std::cerr << "Error parsing command line arguments; Available options:\n"; + + std::cerr << options << std::endl; + return false; + } + return true; +} + +// A simple tool to inspect/print metadata content of ROOT files containing CCDB entries // TODO: optionally print as JSON int main(int argc, char* argv[]) { - if (argc < 2) { - std::cerr << "Usage: " << argv[0] << " CCDBFile.root \n"; + bpo::options_description options("Tool to inspect meta-data content of downloaded CCDB objects. Allowed options:"); + bpo::variables_map vm; + if (!initOptionsAndParse(options, argc, argv, vm)) { + return 1; } - TFile file(argv[1]); - - // query the list of objects - auto keys = file.GetListOfKeys(); - if (keys) { - std::cout << "--- found the following objects -----\n"; - for (int i = 0; i < keys->GetEntries(); ++i) { - auto key = static_cast(keys->At(i)); - if (key) { - std::cout << key->GetName() << " of type " << key->GetClassName() << "\n"; + auto filenames = vm["files"].as>(); + auto timestamp2bchecked = vm["check-timestamp"].as(); + std::vector filesWrongValidity; // record files failing validity check + + for (auto& f : filenames) { + std::cout << "### Loading file : " << f << "\n"; + TFile file(f.c_str()); + + // query the list of objects + auto keys = file.GetListOfKeys(); + if (keys) { + std::cout << "--- found the following objects -----\n"; + for (int i = 0; i < keys->GetEntries(); ++i) { + auto key = static_cast(keys->At(i)); + if (key) { + std::cout << key->GetName() << " of type " << key->GetClassName() << "\n"; + } } + } else { + std::cout << "--- no objects found -----\n"; } - } else { - std::cout << "--- no objects found -----\n"; - } - auto queryinfo = o2::ccdb::CcdbApi::retrieveQueryInfo(file); - if (queryinfo) { - std::cout << "---found query info -----\n"; - queryinfo->print(); - } else { - std::cout << "--- no query information found ------\n"; - } + auto queryinfo = o2::ccdb::CcdbApi::retrieveQueryInfo(file); + if (queryinfo) { + std::cout << "---found query info -----\n"; + queryinfo->print(); + } else { + std::cout << "--- no query information found ------\n"; + } - auto meta = o2::ccdb::CcdbApi::retrieveMetaInfo(file); - if (meta) { - std::cout << "---found meta info -----\n"; - for (auto keyvalue : *meta) { - std::cout << keyvalue.first << " : " << keyvalue.second << "\n"; + auto meta = o2::ccdb::CcdbApi::retrieveMetaInfo(file); + if (meta) { + std::cout << "---found meta info -----\n"; + for (auto keyvalue : *meta) { + std::cout << keyvalue.first << " : " << keyvalue.second << "\n"; + } + if (timestamp2bchecked > 0) { + // retrieve Valid-From and Valid-To headers + try { + auto valid_from = std::stol((*meta)["Valid-From"]); + auto valid_to = std::stol((*meta)["Valid-Until"]); + if (!(valid_from <= timestamp2bchecked) && (timestamp2bchecked <= valid_to)) { + std::cerr << "### ERROR: failed validity check for timestamp " << timestamp2bchecked << " not in [" << valid_from << ":" << valid_to << "]\n"; + filesWrongValidity.push_back(f); + } + } catch (std::exception e) { + // no validity could be extracted; + filesWrongValidity.push_back(f); + } + } + } else { + std::cout << "--- no meta information found ---\n"; } - } else { - std::cout << "--- no meta information found ---\n"; } + if (filesWrongValidity.size() > 0) { + std::cerr << "### ERROR: Validity checks failed for:\n"; + for (auto& f : filesWrongValidity) { + std::cerr << "### " << f << "\n"; + } + return 1; + } return 0; } diff --git a/CCDB/src/UploadTool.cxx b/CCDB/src/UploadTool.cxx index 44b8d8e20bc7d..9aba417b4f4a9 100644 --- a/CCDB/src/UploadTool.cxx +++ b/CCDB/src/UploadTool.cxx @@ -147,33 +147,44 @@ int main(int argc, char* argv[]) meta[p.first] = p.second; } - TFile f(filename.c_str()); - auto key = f.GetKey(keyname.c_str()); - if (key) { - // get type of key - auto classname = key->GetClassName(); - auto tcl = TClass::GetClass(classname); - auto object = f.Get(keyname.c_str()); - if (tcl->InheritsFrom("TTree")) { - auto tree = static_cast(object); - tree->LoadBaskets(0x1L << 32); // make tree memory based - tree->SetDirectory(nullptr); - } - // convert classname to typeinfo - // typeinfo - auto ti = tcl->GetTypeInfo(); - - std::cout << " Uploading an object of type " << key->GetClassName() - << " to path " << path << " with timestamp validity from " << starttimestamp - << " to " << endtimestamp << "\n"; - - api.storeAsTFile_impl(object, *ti, path, meta, starttimestamp, endtimestamp); + if (filename == "headersOnly") { + auto ent = meta.find("Redirect"); + std::cout << " Uploading a headers-only object to path " << path << " with timestamp validity from " << starttimestamp << " to " << endtimestamp + << " Redirection to: " << ((ent != meta.end()) ? ent->second : std::string{"none"}) << "\n"; + api.storeAsBinaryFile(nullptr, 0, "ignored", "", path, meta, starttimestamp, endtimestamp); if (!api.isSnapshotMode() && meta.find("adjustableEOV") != meta.end() && meta.find("default") == meta.end()) { - o2::ccdb::CcdbObjectInfo oi(path, classname, filename, meta, starttimestamp, endtimestamp); + o2::ccdb::CcdbObjectInfo oi(path, "", "", meta, starttimestamp, endtimestamp); o2::ccdb::adjustOverriddenEOV(api, oi); } } else { - std::cerr << "Key " << keyname << " does not exist\n"; + TFile f(filename.c_str()); + auto key = f.GetKey(keyname.c_str()); + if (key) { + // get type of key + auto classname = key->GetClassName(); + auto tcl = TClass::GetClass(classname); + auto object = f.Get(keyname.c_str()); + if (tcl->InheritsFrom("TTree")) { + auto tree = static_cast(object); + tree->LoadBaskets(0x1L << 32); // make tree memory based + tree->SetDirectory(nullptr); + } + // convert classname to typeinfo + // typeinfo + auto ti = tcl->GetTypeInfo(); + + std::cout << " Uploading an object of type " << key->GetClassName() + << " to path " << path << " with timestamp validity from " << starttimestamp + << " to " << endtimestamp << "\n"; + + api.storeAsTFile_impl(object, *ti, path, meta, starttimestamp, endtimestamp); + if (!api.isSnapshotMode() && meta.find("adjustableEOV") != meta.end() && meta.find("default") == meta.end()) { + o2::ccdb::CcdbObjectInfo oi(path, classname, filename, meta, starttimestamp, endtimestamp); + o2::ccdb::adjustOverriddenEOV(api, oi); + } + } else { + std::cerr << "Key " << keyname << " does not exist\n"; + } } return 0; diff --git a/CCDB/test/testBasicCCDBManager.cxx b/CCDB/test/testBasicCCDBManager.cxx index 6cc954bfb44bd..6359bf2f5ccf4 100644 --- a/CCDB/test/testBasicCCDBManager.cxx +++ b/CCDB/test/testBasicCCDBManager.cxx @@ -26,18 +26,49 @@ using namespace o2::ccdb; +static std::string basePath; +std::string ccdbUrl = "http://ccdb-test.cern.ch:8080"; +bool hostReachable = false; + +/** + * Global fixture, ie general setup and teardown + */ +struct Fixture { + Fixture() + { + CcdbApi api; + api.init(ccdbUrl); + std::cout << "ccdb url: " << ccdbUrl << std::endl; + hostReachable = api.isHostReachable(); + std::cout << "Is host reachable ? --> " << hostReachable << std::endl; + char hostname[_POSIX_HOST_NAME_MAX]; + gethostname(hostname, _POSIX_HOST_NAME_MAX); + basePath = std::string("Test/") + hostname + "/pid" + getpid() + "/BasicCCDBManager/"; + std::cout << "Path we will use in this test suite : " + basePath << std::endl; + } + ~Fixture() + { + if (hostReachable) { + CcdbApi api; + api.init(ccdbUrl); + api.truncate(basePath + "*"); + std::cout << "Test data truncated (" << basePath << ")" << std::endl; + } + } +}; +BOOST_GLOBAL_FIXTURE(Fixture); + BOOST_AUTO_TEST_CASE(TestBasicCCDBManager) { CcdbApi api; - const std::string uri = "http://ccdb-test.cern.ch:8080"; - api.init(uri); + api.init(ccdbUrl); if (!api.isHostReachable()) { - LOG(warning) << "Host " << uri << " is not reacheable, abandoning the test"; + LOG(warning) << "Host " << ccdbUrl << " is not reacheable, abandoning the test"; return; } // - std::string pathA = "Test/CachingA"; - std::string pathB = "Test/CachingB"; + std::string pathA = basePath + "CachingA"; + std::string pathB = basePath + "CachingB"; std::string ccdbObjO = "testObjectO"; std::string ccdbObjN = "testObjectN"; std::map md; @@ -48,7 +79,7 @@ BOOST_AUTO_TEST_CASE(TestBasicCCDBManager) // test reading auto& cdb = o2::ccdb::BasicCCDBManager::instance(); - cdb.setURL(uri); + cdb.setURL(ccdbUrl); cdb.setTimestamp((start + stop) / 2); cdb.setCaching(true); diff --git a/CCDB/test/testCcdbApi.cxx b/CCDB/test/testCcdbApi.cxx index cac48ecff933c..1b6a5d6f0967a 100644 --- a/CCDB/test/testCcdbApi.cxx +++ b/CCDB/test/testCcdbApi.cxx @@ -14,6 +14,7 @@ /// \author Barthelemy von Haller /// +#define BOOST_BIND_GLOBAL_PLACEHOLDERS #define BOOST_TEST_MODULE CCDB #define BOOST_TEST_MAIN #define BOOST_TEST_DYN_LINK @@ -44,8 +45,8 @@ using namespace o2::ccdb; namespace utf = boost::unit_test; namespace tt = boost::test_tools; -static string ccdbUrl; -static string basePath; +static std::string ccdbUrl; +static std::string basePath; bool hostReachable = false; /** @@ -60,14 +61,18 @@ struct Fixture { cout << "ccdb url: " << ccdbUrl << endl; hostReachable = api.isHostReachable(); cout << "Is host reachable ? --> " << hostReachable << endl; - basePath = string("Test/pid") + getpid() + "/"; + char hostname[_POSIX_HOST_NAME_MAX]; + gethostname(hostname, _POSIX_HOST_NAME_MAX); + basePath = std::string("Test/TestCcdbApi/") + hostname + "/pid" + getpid() + "/"; + // Replace dashes by underscores to avoid problems in the creation of local directories + std::replace(basePath.begin(), basePath.end(), '-','_'); cout << "Path we will use in this test suite : " + basePath << endl; } ~Fixture() { if (hostReachable) { CcdbApi api; - map metadata; + std::map metadata; api.init(ccdbUrl); api.truncate(basePath + "*"); cout << "Test data truncated (" << basePath << ")" << endl; @@ -99,7 +104,7 @@ struct test_fixture { ~test_fixture() = default; CcdbApi api; - map metadata; + std::map metadata; }; BOOST_AUTO_TEST_CASE(storeTMemFile_test, *utf::precondition(if_reachable())) @@ -148,7 +153,7 @@ BOOST_AUTO_TEST_CASE(store_retrieve_TMemFile_templated_test, *utf::precondition( BOOST_CHECK(f.api.retrieveFromTFileAny(basePath + "CCDBPath", f.metadata) == nullptr); // try to get the headers back and to find the metadata - map md; + std::map md; path2 = f.api.retrieveFromTFileAny(basePath + "CCDBPath", f.metadata, -1, &md); BOOST_CHECK_EQUAL(md.count("Hello"), 1); BOOST_CHECK_EQUAL(md["Hello"], "World"); @@ -340,7 +345,7 @@ BOOST_AUTO_TEST_CASE(delete_test, *utf::precondition(if_reachable())) BOOST_CHECK(h2 == nullptr); } -void countItems(const string& s, int& countObjects, int& countSubfolders) +void countItems(const std::string& s, int& countObjects, int& countSubfolders) { countObjects = 0; countSubfolders = 0; @@ -363,7 +368,7 @@ BOOST_AUTO_TEST_CASE(list_test, *utf::precondition(if_reachable())) test_fixture f; // test non-empty top dir - string s = f.api.list("", "application/json"); // top dir + std::string s = f.api.list("", "application/json"); // top dir long nbLines = std::count(s.begin(), s.end(), '\n') + 1; BOOST_CHECK(nbLines > 5); @@ -377,8 +382,6 @@ BOOST_AUTO_TEST_CASE(list_test, *utf::precondition(if_reachable())) // more complex tree TH1F h1("object1", "object1", 100, 0, 99); - cout << "storing object 1 in Test" << endl; - f.api.storeAsTFile(&h1, "Test", f.metadata); cout << "storing object 2 in Test/Detector" << endl; f.api.storeAsTFile(&h1, basePath + "Detector", f.metadata); cout << "storing object 3 in Test/Detector" << endl; @@ -433,7 +436,7 @@ BOOST_AUTO_TEST_CASE(TestHeaderParsing) BOOST_AUTO_TEST_CASE(TestFetchingHeaders, *utf::precondition(if_reachable())) { // first store the object - string objectPath = basePath + "objectETag"; + std::string objectPath = basePath + "objectETag"; test_fixture f; TH1F h1("objectETag", "objectETag", 100, 0, 99); f.api.storeAsTFile(&h1, objectPath, f.metadata); @@ -442,7 +445,7 @@ BOOST_AUTO_TEST_CASE(TestFetchingHeaders, *utf::precondition(if_reachable())) std::string etag; std::vector headers; std::vector pfns; - string path = objectPath + "/" + std::to_string(getCurrentTimestamp()); + std::string path = objectPath + "/" + std::to_string(getCurrentTimestamp()); auto updated = CcdbApi::getCCDBEntryHeaders("http://ccdb-test.cern.ch:8080/" + path, etag, headers); BOOST_CHECK_EQUAL(updated, true); BOOST_REQUIRE(headers.size() != 0); @@ -459,7 +462,7 @@ BOOST_AUTO_TEST_CASE(TestRetrieveHeaders, *utf::precondition(if_reachable())) TH1F h1("object1", "object1", 100, 0, 99); cout << "storing object 1 in " << basePath << "Test" << endl; - map metadata; + std::map metadata; metadata["custom"] = "whatever"; f.api.storeAsTFile(&h1, basePath + "Test", metadata); @@ -495,7 +498,7 @@ BOOST_AUTO_TEST_CASE(TestUpdateMetadata, *utf::precondition(if_reachable())) // upload an object TH1F h1("object1", "object1", 100, 0, 99); cout << "storing object 1 in " << basePath << "Test" << endl; - map metadata; + std::map metadata; metadata["custom"] = "whatever"; metadata["id"] = "first"; f.api.storeAsTFile(&h1, basePath + "Test", metadata); @@ -504,10 +507,10 @@ BOOST_AUTO_TEST_CASE(TestUpdateMetadata, *utf::precondition(if_reachable())) std::map headers = f.api.retrieveHeaders(basePath + "Test", metadata); BOOST_CHECK(headers.count("custom") > 0); BOOST_CHECK(headers.at("custom") == "whatever"); - string firstID = headers.at("ETag"); + std::string firstID = headers.at("ETag"); firstID.erase(std::remove(firstID.begin(), firstID.end(), '"'), firstID.end()); - map newMetadata; + std::map newMetadata; newMetadata["custom"] = "somethingelse"; // update the metadata and check @@ -526,7 +529,7 @@ BOOST_AUTO_TEST_CASE(TestUpdateMetadata, *utf::precondition(if_reachable())) // get id cout << "get id" << endl; headers = f.api.retrieveHeaders(basePath + "Test", metadata); - string secondID = headers.at("ETag"); + std::string secondID = headers.at("ETag"); secondID.erase(std::remove(secondID.begin(), secondID.end(), '"'), secondID.end()); // update the metadata by id @@ -550,3 +553,47 @@ BOOST_AUTO_TEST_CASE(TestUpdateMetadata, *utf::precondition(if_reachable())) BOOST_CHECK(headers.count("custom") > 0); BOOST_CHECK(headers.at("custom") == "second"); } + +BOOST_AUTO_TEST_CASE(multi_host_test) +{ + CcdbApi api; + api.init("http://bogus-host.cern.ch,http://ccdb-test.cern.ch:8080"); + std::map metadata; + std::map headers; + o2::pmr::vector dst; + std::string url = "Analysis/ALICE3/Centrality"; + api.loadFileToMemory(dst, url, metadata, 1645780010602, &headers, "", "", "", true); + BOOST_CHECK(dst.size() != 0); +} + +BOOST_AUTO_TEST_CASE(vectored) +{ + CcdbApi api; + api.init("http://ccdb-test.cern.ch:8080"); + + int TEST_SAMPLE_SIZE = 5; + std::vector> dests(TEST_SAMPLE_SIZE); + std::vector> metadatas(TEST_SAMPLE_SIZE); + std::vector> headers(TEST_SAMPLE_SIZE); + + std::vector contexts; + for (int i = 0; i < TEST_SAMPLE_SIZE; i++) { + contexts.push_back(CcdbApi::RequestContext(dests.at(i), metadatas.at(i), headers.at(i))); + contexts.at(i).path = "Analysis/ALICE3/Centrality"; + contexts.at(i).timestamp = 1645780010602; + contexts.at(i).considerSnapshot = true; + } + + api.vectoredLoadFileToMemory(contexts); + + for (auto context : contexts) { + BOOST_CHECK(context.dest.size() != 0); + } +} + +BOOST_AUTO_TEST_CASE(empty_url) +{ + CcdbApi api; + string url = ""; + BOOST_CHECK_EXCEPTION(api.init(url), invalid_argument, [](std::invalid_argument const&) -> bool { return true; }); +} diff --git a/CCDB/test/testCcdbApiDownloader.cxx b/CCDB/test/testCcdbApiDownloader.cxx new file mode 100644 index 0000000000000..76686f5ee1c00 --- /dev/null +++ b/CCDB/test/testCcdbApiDownloader.cxx @@ -0,0 +1,379 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#define BOOST_TEST_MODULE CCDB +#define BOOST_TEST_MAIN +#define BOOST_TEST_DYN_LINK + +#include "CommonUtils/StringUtils.h" +#include "CCDB/CCDBTimeStampUtils.h" +#include +#include +#include +#include +#include // Sleep function to wait for asynch results +#include + +#include +#include +#include + +#include +#include "MemoryResources/MemoryResources.h" + +using namespace std; + +namespace o2 +{ +namespace ccdb +{ + +size_t CurlWrite_CallbackFunc_StdString2(void* contents, size_t size, size_t nmemb, std::string* s) +{ + size_t newLength = size * nmemb; + size_t oldLength = s->size(); + try { + s->resize(oldLength + newLength); + } catch (std::bad_alloc& e) { + LOG(error) << "memory error when getting data from CCDB"; + return 0; + } + + std::copy((char*)contents, (char*)contents + newLength, s->begin() + oldLength); + return size * nmemb; +} + +std::string uniqueAgentID() +{ + std::string host = boost::asio::ip::host_name(); + char const* jobID = getenv("ALIEN_PROC_ID"); + if (jobID) { + return fmt::format("{}-{}-{}-{}", host, getCurrentTimestamp() / 1000, o2::utils::Str::getRandomString(6), jobID); + } else { + return fmt::format("{}-{}-{}", host, getCurrentTimestamp() / 1000, o2::utils::Str::getRandomString(6)); + } +} + +CURL* createTestHandle(std::string* dst) +{ + CURL* handle = curl_easy_init(); + curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, CurlWrite_CallbackFunc_StdString2); + curl_easy_setopt(handle, CURLOPT_WRITEDATA, dst); + curl_easy_setopt(handle, CURLOPT_URL, "http://ccdb-test.cern.ch:8080/"); + auto userAgent = uniqueAgentID(); + curl_easy_setopt(handle, CURLOPT_USERAGENT, userAgent.c_str()); + return handle; +} + +namespace +{ +template > +size_t header_map_callback(char* buffer, size_t size, size_t nitems, void* userdata) +{ + auto* headers = static_cast(userdata); + auto header = std::string(buffer, size * nitems); + std::string::size_type index = header.find(':', 0); + if (index != std::string::npos) { + const auto key = boost::algorithm::trim_copy(header.substr(0, index)); + const auto value = boost::algorithm::trim_copy(header.substr(index + 1)); + headers->insert(std::make_pair(key, value)); + } + return size * nitems; +} +} // namespace + +size_t writeCallbackNoLambda(void* contents, size_t size, size_t nmemb, void* chunkptr) +{ + auto& ho = *static_cast(chunkptr); + auto& chunk = *ho.object; + size_t realsize = size * nmemb, sz = 0; + ho.counter++; + try { + if (chunk.capacity() < chunk.size() + realsize) { + auto cl = ho.header.find("Content-Length"); + if (cl != ho.header.end()) { + sz = std::max(chunk.size() + realsize, (size_t)std::stol(cl->second)); + } else { + sz = chunk.size() + realsize; + // LOGP(debug, "SIZE IS NOT IN HEADER, allocate {}", sz); + } + chunk.reserve(sz); + } + char* contC = (char*)contents; + chunk.insert(chunk.end(), contC, contC + realsize); + } catch (std::exception e) { + // LOGP(alarm, "failed to reserve {} bytes in CURL write callback (realsize = {}): {}", sz, realsize, e.what()); + realsize = 0; + } + return realsize; +} + +std::vector prepareAsyncHandles(size_t num, std::vector*>& dests) +{ + std::vector handles; + + for (int i = 0; i < num; i++) { + auto dest = new o2::pmr::vector(); + dests.push_back(dest); + CURL* curl_handle = curl_easy_init(); + handles.push_back(curl_handle); + + auto data = new DownloaderRequestData(); + data->hoPair.object = dest; + data->hosts.emplace_back("http://ccdb-test.cern.ch:8080"); + data->path = "Analysis/ALICE3/Centrality"; + data->timestamp = 1646729604010; + data->localContentCallback = nullptr; + + curl_easy_setopt(curl_handle, CURLOPT_URL, "http://ccdb-test.cern.ch:8080/Analysis/ALICE3/Centrality/1646729604010"); + curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, writeCallbackNoLambda); + curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, (void*)&(data->hoPair)); + + curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, header_map_callbackhoPair.header)>); + curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, (void*)&(data->hoPair.header)); + curl_easy_setopt(curl_handle, CURLOPT_PRIVATE, (void*)data); + } + return handles; +} + +BOOST_AUTO_TEST_CASE(asynch_schedule_test) +{ + int TRANSFERS = 5; + + if (curl_global_init(CURL_GLOBAL_ALL)) { + fprintf(stderr, "Could not init curl\n"); + return; + } + + CCDBDownloader downloader; + std::vector*> dests; + auto handles = prepareAsyncHandles(TRANSFERS, dests); + size_t transfersLeft = 0; + + for (auto handle : handles) { + downloader.asynchSchedule(handle, &transfersLeft); + } + + while (transfersLeft > 0) { + downloader.runLoop(false); + } + + for (int i = 0; i < TRANSFERS; i++) { + // I would claim that accessing the handles after they are complete + // is actually not supported by the current API, because it was + // previously relying on leaking the handles. Disabling the whole + // thing until we verify that's actually the case. + // + // long httpCode; + // curl_easy_getinfo(handles[i], CURLINFO_HTTP_CODE, &httpCode); + // BOOST_CHECK_EQUAL(httpCode, 200); + // BOOST_CHECK_NE(dests[i]->size(), 0); + // curl_easy_cleanup(handles[i]); + delete dests[i]; + } + curl_global_cleanup(); +} + +BOOST_AUTO_TEST_CASE(perform_test) +{ + if (curl_global_init(CURL_GLOBAL_ALL)) { + fprintf(stderr, "Could not init curl\n"); + return; + } + + CCDBDownloader downloader; + std::string dst = ""; + CURL* handle = createTestHandle(&dst); + + CURLcode curlCode = downloader.perform(handle); + + BOOST_CHECK_EQUAL(curlCode, CURLE_OK); + + long httpCode; + curl_easy_getinfo(handle, CURLINFO_HTTP_CODE, &httpCode); + BOOST_CHECK_EQUAL(httpCode, 200); + + curl_easy_cleanup(handle); + curl_global_cleanup(); +} + +BOOST_AUTO_TEST_CASE(blocking_batch_test) +{ + if (curl_global_init(CURL_GLOBAL_ALL)) { + fprintf(stderr, "Could not init curl\n"); + return; + } + + CCDBDownloader downloader; + std::vector handleVector; + std::vector destinations; + for (int i = 0; i < 100; i++) { + destinations.push_back(new std::string()); + handleVector.push_back(createTestHandle(destinations.back())); + } + + auto curlCodes = downloader.batchBlockingPerform(handleVector); + for (CURLcode code : curlCodes) { + BOOST_CHECK_EQUAL(code, CURLE_OK); + } + + for (CURL* handle : handleVector) { + long httpCode; + curl_easy_getinfo(handle, CURLINFO_HTTP_CODE, &httpCode); + BOOST_CHECK_EQUAL(httpCode, 200); + curl_easy_cleanup(handle); + } + + for (std::string* dst : destinations) { + delete dst; + } + + curl_global_cleanup(); +} + +BOOST_AUTO_TEST_CASE(test_with_break) +{ + if (curl_global_init(CURL_GLOBAL_ALL)) { + fprintf(stderr, "Could not init curl\n"); + return; + } + + CCDBDownloader downloader; + std::vector handleVector; + std::vector destinations; + for (int i = 0; i < 100; i++) { + destinations.push_back(new std::string()); + handleVector.push_back(createTestHandle(destinations.back())); + } + + auto curlCodes = downloader.batchBlockingPerform(handleVector); + + for (CURLcode code : curlCodes) { + BOOST_CHECK_EQUAL(code, CURLE_OK); + } + + for (CURL* handle : handleVector) { + long httpCode; + curl_easy_getinfo(handle, CURLINFO_HTTP_CODE, &httpCode); + BOOST_CHECK_EQUAL(httpCode, 200); + curl_easy_cleanup(handle); + } + + for (std::string* dst : destinations) { + delete dst; + } + + sleep(10); + + std::vector handleVector2; + std::vector destinations2; + for (int i = 0; i < 100; i++) { + destinations2.push_back(new std::string()); + handleVector2.push_back(createTestHandle(destinations2.back())); + } + + auto curlCodes2 = downloader.batchBlockingPerform(handleVector2); + for (CURLcode code : curlCodes2) { + BOOST_CHECK_EQUAL(code, CURLE_OK); + } + + for (CURL* handle : handleVector2) { + long httpCode; + curl_easy_getinfo(handle, CURLINFO_HTTP_CODE, &httpCode); + BOOST_CHECK_EQUAL(httpCode, 200); + curl_easy_cleanup(handle); + } + + for (std::string* dst : destinations2) { + delete dst; + } + + curl_global_cleanup(); +} + +void onUVClose(uv_handle_t* handle) +{ + if (handle != nullptr) { + free(handle); + } +} + +void closeAllHandles(uv_handle_t* handle, void* arg) +{ + if (!uv_is_closing(handle)) { + uv_close(handle, onUVClose); + } +} + +void testTimerCB(uv_timer_t* handle) +{ + // Mock function to be used by tested timer +} + +BOOST_AUTO_TEST_CASE(external_loop_test) +{ + // Prepare uv_loop to be provided to the downloader + auto uvLoop = new uv_loop_t(); + uv_loop_init(uvLoop); + + // Prepare test timer. It will be used to check whether the downloader affects external handles. + auto testTimer = (uv_timer_t*)malloc(sizeof(uv_timer_t)); + uv_timer_init(uvLoop, testTimer); + uv_timer_start(testTimer, testTimerCB, 10, 10); + + if (curl_global_init(CURL_GLOBAL_ALL)) { + fprintf(stderr, "Could not init curl\n"); + return; + } + + // Regular downloader test + auto downloader = new o2::ccdb::CCDBDownloader(uvLoop); + std::string dst = ""; + CURL* handle = createTestHandle(&dst); + + CURLcode curlCode = downloader->perform(handle); + + BOOST_CHECK_EQUAL(curlCode, CURLE_OK); + + long httpCode; + curl_easy_getinfo(handle, CURLINFO_HTTP_CODE, &httpCode); + BOOST_CHECK_EQUAL(httpCode, 200); + + curl_easy_cleanup(handle); + curl_global_cleanup(); + + // Check if test timer and external loop are still alive + BOOST_CHECK_NE(uv_is_active((uv_handle_t*)testTimer), 0); + BOOST_CHECK_NE(uv_loop_alive(uvLoop), 0); + + // Downloader must be closed before uv_loop. + // The reason for that are the uv_poll handles attached to the curl multi handle. + // The multi handle must be cleaned (via destuctor) before poll handles attached to them are removed (via walking and closing). + delete downloader; + while (uv_loop_alive(uvLoop) || uv_loop_close(uvLoop) == UV_EBUSY) { + uv_walk(uvLoop, closeAllHandles, nullptr); + uv_run(uvLoop, UV_RUN_ONCE); + } + delete uvLoop; +} + +BOOST_AUTO_TEST_CASE(trim_host_url_test) +{ + CCDBDownloader downloader; + BOOST_CHECK_EQUAL(downloader.trimHostUrl("http://localhost:8080"), "http://localhost:8080"); + BOOST_CHECK_EQUAL(downloader.trimHostUrl("http://localhost"), "http://localhost"); + BOOST_CHECK_EQUAL(downloader.trimHostUrl("http://localhost:8080/some/path"), "http://localhost:8080"); + BOOST_CHECK_EQUAL(downloader.trimHostUrl("http://localhost/some/path"), "http://localhost"); + BOOST_CHECK_EQUAL(downloader.trimHostUrl("http://localhost:8080/Task/Detector/1?HTTPOnly=true"), "http://localhost:8080"); +} + +} // namespace ccdb +} // namespace o2 diff --git a/CCDB/test/testCcdbApiHeaders.cxx b/CCDB/test/testCcdbApiHeaders.cxx new file mode 100644 index 0000000000000..bcfa2a5b44bc2 --- /dev/null +++ b/CCDB/test/testCcdbApiHeaders.cxx @@ -0,0 +1,413 @@ +// Copyright 2019-2025 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// +/// \file testCcdbApiHeaders.cxx +/// \brief Test BasicCCDBManager header/metadata information functionality with caching +/// \author martin.oines.eide@cern.ch + +#define BOOST_TEST_MODULE CCDB +#define BOOST_TEST_MAIN +#define BOOST_TEST_DYN_LINK + +#include +#include "CCDB/BasicCCDBManager.h" +#include "CCDB/CCDBTimeStampUtils.h" +#include "CCDB/CcdbApi.h" +#include + +static std::string basePath; +// std::string ccdbUrl = "http://localhost:8080"; +std::string ccdbUrl = "http://ccdb-test.cern.ch:8080"; +bool hostReachable = false; + +/** + * Global fixture, ie general setup and teardown + * Copied from testBasicCCDBManager.cxx + */ +struct Fixture { + Fixture() + { + auto& ccdbManager = o2::ccdb::BasicCCDBManager::instance(); + if (std::getenv("ALICEO2_CCDB_HOST")) { + ccdbUrl = std::string(std::getenv("ALICEO2_CCDB_HOST")); + } + ccdbManager.setURL(ccdbUrl); + hostReachable = ccdbManager.getCCDBAccessor().isHostReachable(); + char hostname[_POSIX_HOST_NAME_MAX]; + gethostname(hostname, _POSIX_HOST_NAME_MAX); + basePath = std::string("Users/m/meide/Tests/") + hostname + "/pid-" + getpid() + "/BasicCCDBManager/"; + + LOG(info) << "Path we will use in this test suite : " + basePath << std::endl; + LOG(info) << "ccdb url: " << ccdbUrl << std::endl; + LOG(info) << "Is host reachable ? --> " << hostReachable << std::endl; + } + ~Fixture() + { + if (hostReachable) { + o2::ccdb::BasicCCDBManager::instance().getCCDBAccessor().truncate(basePath + "*"); // This deletes the data after test is run, disable if you want to inspect the data + LOG(info) << "Test data truncated/deleted (" << basePath << ")" << std::endl; + } + } +}; +BOOST_GLOBAL_FIXTURE(Fixture); +/** + * Just an accessor to the hostReachable variable to be used to determine whether tests can be ran or not. + * Copied from testCcdbApi.cxx + */ +struct if_reachable { + boost::test_tools::assertion_result operator()(boost::unit_test::test_unit_id) + { + return hostReachable; + } +}; + +// Only compare known and stable keys (avoid volatile ones like Date) +static const std::set sStableKeys = { + "ETag", + "Valid-From", + "Valid-Until", + "Created", + "Last-Modified", + "Content-Disposition", + "Content-Location", + "path", + "partName", + "Content-MD5", + "Hello" // TODO find other headers to compare to +}; + +// Test that we get back the same header header keys as we put in (for stable keys) + +BOOST_AUTO_TEST_CASE(testCachedHeaders, *boost::unit_test::precondition(if_reachable())) +{ + /// ━━━━━━━ ARRANGE ━━━━━━━━━ + // First store objects to test with + auto& ccdbManager = o2::ccdb::BasicCCDBManager::instance(); + std::string pathA = basePath + "CachingA"; + std::string pathB = basePath + "CachingB"; + std::string pathC = basePath + "CachingC"; + std::string ccdbObjO = "testObjectO"; + std::string ccdbObjN = "testObjectN"; + std::string ccdbObjX = "testObjectX"; + std::map md = { + {"Hello", "World"}, + {"Key1", "Value1"}, + {"Key2", "Value2"}, + }; + long start = 1000, stop = 3000; + ccdbManager.getCCDBAccessor().storeAsTFileAny(&ccdbObjO, pathA, md, start, stop); + ccdbManager.getCCDBAccessor().storeAsTFileAny(&ccdbObjN, pathB, md, start, stop); + ccdbManager.getCCDBAccessor().storeAsTFileAny(&ccdbObjX, pathC, md, start, stop); + // initilize the BasicCCDBManager + ccdbManager.clearCache(); + ccdbManager.setCaching(true); // This is what we want to test. + + /// ━━━━━━━━━━━ ACT ━━━━━━━━━━━━ + // Plan: get one object, then another, then the first again and check the headers are the same + std::map headers1, headers2, headers3; + + auto* obj1 = ccdbManager.getForTimeStamp(pathA, (start + stop) / 2, &headers1); + auto* obj2 = ccdbManager.getForTimeStamp(pathB, (start + stop) / 2, &headers2); + auto* obj3 = ccdbManager.getForTimeStamp(pathA, (start + stop) / 2, &headers3); // Should lead to a cache hit! + + /// ━━━━━━━━━━━ ASSERT ━━━━━━━━━━━━ + /// Check that we got something + BOOST_REQUIRE(obj1 != nullptr); + BOOST_REQUIRE(obj2 != nullptr); + BOOST_REQUIRE(obj3 != nullptr); + + LOG(debug) << "obj1: " << *obj1; + LOG(debug) << "obj2: " << *obj2; + LOG(debug) << "obj3: " << *obj3; + + // Sanity check + /// Check that the objects are correct + BOOST_TEST(*obj1 == ccdbObjO); + BOOST_TEST(*obj3 == ccdbObjO); + BOOST_TEST(obj3 == obj1); // should be the same object in memory since it is cached + + BOOST_TEST(obj2 != obj1); + + (*obj1) = "ModifiedObject"; + BOOST_TEST(*obj1 == "ModifiedObject"); + BOOST_TEST(*obj3 == "ModifiedObject"); // obj3 and obj1 are the same object in memory + + // Check that the headers are the same for the two retrievals of the same object + BOOST_REQUIRE(headers1.size() != 0); + BOOST_REQUIRE(headers3.size() != 0); + + LOG(debug) << "Headers1 size: " << headers1.size(); + for (const auto& h : headers1) { + LOG(debug) << " " << h.first << " -> " << h.second; + } + LOG(debug) << "Headers3 size: " << headers3.size(); + for (const auto& h : headers3) { + LOG(debug) << " " << h.first << " -> " << h.second; + } + + for (const auto& stableKey : sStableKeys) { + LOG(info) << "Checking key: " << stableKey; + + BOOST_REQUIRE(headers1.count(stableKey) > 0); + BOOST_REQUIRE(headers3.count(stableKey) > 0); + BOOST_TEST(headers1.at(stableKey) == headers3.at(stableKey)); + } + BOOST_TEST(headers1 != headers2, "The headers for different objects should be different"); + + // Test that we can change the map and the two headers are not affected + headers1["NewKey"] = "NewValue"; + headers3["NewKey"] = "DifferentValue"; + BOOST_TEST(headers1["NewKey"] != headers3["NewKey"]); // This tests that we have a deep copy of the headers +} + +BOOST_AUTO_TEST_CASE(testNonCachedHeaders, *boost::unit_test::precondition(if_reachable())) +{ + /// ━━━━━━━ ARRANGE ━━━━━━━━━ + // First store objects to test with + auto& ccdbManager = o2::ccdb::BasicCCDBManager::instance(); + std::string pathA = basePath + "NonCachingA"; + std::string pathB = basePath + "NonCachingB"; + std::string ccdbObjO = "testObjectO"; + std::string ccdbObjN = "testObjectN"; + std::map md = { + {"Hello", "World"}, + {"Key1", "Value1"}, + {"Key2", "Value2"}, + }; + long start = 1000, stop = 2000; + ccdbManager.getCCDBAccessor().storeAsTFileAny(&ccdbObjO, pathA, md, start, stop); + ccdbManager.getCCDBAccessor().storeAsTFileAny(&ccdbObjN, pathB, md, start, stop); + // initilize the BasicCCDBManager + ccdbManager.clearCache(); + ccdbManager.setCaching(false); // This is what we want to test, no caching + + /// ━━━━━━━━━━━ ACT ━━━━━━━━━━━━ + // Plan: get one object, then another, then the first again. Then check that the contents is the same but not the object in memory + std::map headers1, headers2, headers3; + + auto* obj1 = ccdbManager.getForTimeStamp(pathA, (start + stop) / 2, &headers1); + auto* obj2 = ccdbManager.getForTimeStamp(pathB, (start + stop) / 2, &headers2); + auto* obj3 = ccdbManager.getForTimeStamp(pathA, (start + stop) / 2, &headers3); // Should not be cached since explicitly disabled + + ccdbManager.setCaching(true); // Restore default state + /// ━━━━━━━━━━━ ASSERT ━━━━━━━━━━━ + /// Check that we got something + BOOST_REQUIRE(obj1 != nullptr); + BOOST_REQUIRE(obj2 != nullptr); + BOOST_REQUIRE(obj3 != nullptr); + + LOG(debug) << "obj1: " << *obj1; + LOG(debug) << "obj2: " << *obj2; + LOG(debug) << "obj3: " << *obj3; + + // Sanity check + /// Check that the objects are correct + BOOST_TEST(*obj1 == ccdbObjO); + BOOST_TEST(*obj3 == ccdbObjO); + BOOST_TEST(obj2 != obj1); + BOOST_TEST(obj3 != obj1); // should NOT be the same object in memory + (*obj1) = "ModifiedObject"; + BOOST_TEST(*obj1 == "ModifiedObject"); + BOOST_TEST(*obj3 != "ModifiedObject"); // obj3 and obj1 are NOT the same object in memory + + BOOST_TEST(headers1.size() == headers3.size()); + + // Remove the date header since it may be different even for the same object since we might have asked in different seconds + headers1.erase("Date"); + headers3.erase("Date"); + BOOST_TEST(headers1 == headers3, "The headers for the same object should be the same even if not cached"); + + BOOST_TEST(headers1 != headers2, "The headers for different objects should be different"); + BOOST_TEST(headers1.size() != 0); + BOOST_TEST(headers3.size() != 0); + BOOST_TEST(headers2.size() != 0); + BOOST_TEST(headers1 != headers2, "The headers for different objects should be different"); + + // cleanup + delete obj1; + delete obj2; + delete obj3; +} + +BOOST_AUTO_TEST_CASE(CacheFirstRetrievalAndHeadersPersistence, *boost::unit_test::precondition(if_reachable())) +{ + /// ━━━━━━━ ARRANGE ━━━━━━━━━ + auto& mgr = o2::ccdb::BasicCCDBManager::instance(); + // Prepare two validity slots for same path to test ETag change later + std::string path = basePath + "ObjA"; + std::string objV1 = "ObjectVersion1"; + std::string objV2 = "ObjectVersion2"; + std::map meta1{ + {"UserKey1", "UValue1"}, + {"UserKey2", "UValue2"}}; + long v1start = 10'000; + long v1stop = 20'000; + long v2start = v1stop; // contiguous slot + long v2stop = v2start + (v1stop - v1start); + long mid1 = (v1start + v1stop) / 2; + // Store 2 versions + mgr.getCCDBAccessor().storeAsTFileAny(&objV1, path, meta1, v1start, v1stop); + mgr.getCCDBAccessor().storeAsTFileAny(&objV2, path, meta1, v2start, v2stop); + + mgr.clearCache(); + mgr.setCaching(true); + mgr.setFatalWhenNull(true); + mgr.setTimestamp(mid1); + + /// ━━━━━━━ACT━━━━━━━━━ + std::map headers1, headers2, headers4, headers5; + + // 1) First retrieval WITH headers inside 1st slot + auto* p1 = mgr.getForTimeStamp(path, mid1, &headers1); + size_t fetchedSizeAfterFirst = mgr.getFetchedSize(); + // 2) Second retrieval (cache hit) + auto* p2 = mgr.getForTimeStamp(path, mid1, &headers2); + size_t fetchedSizeAfterSecond = mgr.getFetchedSize(); + // 3) Third retrieval (cache hit) WITHOUT passing headers + auto* p3 = mgr.getForTimeStamp(path, mid1); + // 4) Fourth retrieval with headers again -> should still produce same headers + auto* p4 = mgr.getForTimeStamp(path, mid1, &headers4); + // 5) Fifth retrieval with headers again to check persistence + auto* p5 = mgr.getForTimeStamp(path, mid1, &headers5); + + mgr.setFatalWhenNull(false); // restore default + + /// ━━━━━━━ASSERT━━━━━━━━━ + + BOOST_TEST(p1 != nullptr); + BOOST_TEST(*p1 == objV1); + + BOOST_TEST(headers1.count("UserKey1") == 1); + BOOST_TEST(headers1.count("UserKey2") == 1); + BOOST_TEST(headers1["UserKey1"] == "UValue1"); + BOOST_TEST(headers1["UserKey2"] == "UValue2"); + BOOST_TEST(headers1.count("Valid-From") == 1); + BOOST_TEST(headers1.count("Valid-Until") == 1); + BOOST_TEST(headers1.count("ETag") == 1); + + /* Need to manually amend the headers1 to have cache valid until for comparison sake, + * the header is not set in the first request. + * It is only set if the internal cache of CCDB has seen this object before, apperently. + * This will never happen in this test since it was just created and not asked for before. + */ + headers1["Cache-Valid-Until"] = std::to_string(v1stop); + + /* In rare cases the header date might be different, if the second has ticked over between the requests + */ + headers1.erase("Date"); + headers2.erase("Date"); + headers4.erase("Date"); + headers5.erase("Date"); + + BOOST_TEST(p2 == p1); // same pointer for cached scenario + BOOST_TEST(headers2 == headers1); // identical header map + BOOST_TEST(fetchedSizeAfterSecond == fetchedSizeAfterFirst); // no new fetch + + BOOST_TEST(p3 == p1); + + BOOST_TEST(p4 == p1); + BOOST_TEST(headers4 == headers1); + + // Mutate the returned header map locally and ensure it does not corrupt internal cache + headers4["UserKey1"] = "Tampered"; + BOOST_TEST(p5 == p1); + BOOST_TEST(headers5["UserKey1"] == "UValue1"); // internal unchanged +} + +BOOST_AUTO_TEST_CASE(FailedFetchDoesNotGiveMetadata, *boost::unit_test::precondition(if_reachable())) +{ + + /// ━━━━━━━ ARRANGE ━━━━━━━━━ + auto& mgr = o2::ccdb::BasicCCDBManager::instance(); + std::string path = basePath + "FailThenRecover"; + std::string content = "ContentX"; + std::map meta{{"Alpha", "Beta"}}; + long s = 300'000, e = 310'000; + mgr.getCCDBAccessor().storeAsTFileAny(&content, path, meta, s, e); + mgr.clearCache(); + mgr.setCaching(true); + mgr.setFatalWhenNull(false); + + /// ━━━━━━━ ACT ━━━━━━━━━ + // Intentionally pick a timestamp outside validity to fail first + long badTS = s - 1000; + long goodTS = (s + e) / 2; + std::map hFail, hGood; + auto* badObj = mgr.getForTimeStamp(path, badTS, &hFail); + auto* goodObj = mgr.getForTimeStamp(path, goodTS, &hGood); + + /// ━━━━━━━ ASSERT ━━━━━━━━━ + BOOST_TEST(!hFail.empty()); // Should have some headers + BOOST_TEST(hFail["Alpha"] != "Beta"); // But not the metadata + BOOST_TEST(hGood.count("Alpha") == 1); + BOOST_TEST(hGood["Alpha"] == "Beta"); + + mgr.setFatalWhenNull(true); +} + +BOOST_AUTO_TEST_CASE(FirstCallWithoutHeadersThenWithHeaders, *boost::unit_test::precondition(if_reachable())) +{ + + auto& mgr = o2::ccdb::BasicCCDBManager::instance(); + std::string path = basePath + "LateHeaders"; + std::string body = "Late"; + std::map meta{{"LateKey", "LateVal"}}; + long s = 400'000, e = 410'000; + mgr.getCCDBAccessor().storeAsTFileAny(&body, path, meta, s, e); + + mgr.clearCache(); + mgr.setCaching(true); + long ts = (s + e) / 2; + + // 1) First call with nullptr headers + auto* first = mgr.getForTimeStamp(path, ts); + BOOST_TEST(first != nullptr); + BOOST_TEST(*first == body); + + // 2) Second call asking for headers - should return the full set + std::map h2; + auto* second = mgr.getForTimeStamp(path, ts, &h2); + BOOST_TEST(second == first); + BOOST_TEST(h2.count("LateKey") == 1); + BOOST_TEST(h2["LateKey"] == "LateVal"); + BOOST_TEST(h2.count("Valid-From") == 1); + BOOST_TEST(h2.count("Valid-Until") == 1); +} + +BOOST_AUTO_TEST_CASE(HeadersAreStableAcrossMultipleHits, *boost::unit_test::precondition(if_reachable())) +{ + + auto& mgr = o2::ccdb::BasicCCDBManager::instance(); + std::string path = basePath + "StableHeaders"; + std::string body = "Stable"; + std::map meta{{"HK", "HV"}}; + long s = 500'000, e = 510'000; + mgr.getCCDBAccessor().storeAsTFileAny(&body, path, meta, s, e); + + mgr.clearCache(); + mgr.setCaching(true); + long ts = (s + e) / 2; + + std::map h1; + auto* o1 = mgr.getForTimeStamp(path, ts, &h1); + BOOST_TEST(o1 != nullptr); + BOOST_TEST(h1.count("HK") == 1); + + std::string etag = h1["ETag"]; + for (int i = 0; i < 15; ++i) { + std::map hi; + auto* oi = mgr.getForTimeStamp(path, ts, &hi); + BOOST_TEST(oi == o1); + BOOST_TEST(hi.count("HK") == 1); + BOOST_TEST(hi["ETag"] == etag); + } +} diff --git a/CCDB/test/testCcdbApiMultipleUrls.cxx b/CCDB/test/testCcdbApiMultipleUrls.cxx index 331d0553c3aec..07ab0ddcb4dcf 100644 --- a/CCDB/test/testCcdbApiMultipleUrls.cxx +++ b/CCDB/test/testCcdbApiMultipleUrls.cxx @@ -24,8 +24,8 @@ using namespace o2::ccdb; namespace utf = boost::unit_test; namespace tt = boost::test_tools; -static string ccdbUrl; -static string basePath; +static std::string ccdbUrl; +static std::string basePath; bool hostReachable = false; /** @@ -40,14 +40,14 @@ struct Fixture { cout << "ccdb url: " << ccdbUrl << endl; hostReachable = api.isHostReachable(); cout << "Is host reachable ? --> " << hostReachable << endl; - basePath = string("Test/pid") + getpid() + "/"; + basePath = std::string("Test/pid") + getpid() + "/"; cout << "Path we will use in this test suite : " + basePath << endl; } ~Fixture() { if (hostReachable) { CcdbApi api; - map metadata; + std::map metadata; api.init(ccdbUrl); api.truncate(basePath + "*"); cout << "Test data truncated (" << basePath << ")" << endl; @@ -79,7 +79,7 @@ struct test_fixture { ~test_fixture() = default; CcdbApi api; - map metadata; + std::map metadata; }; BOOST_AUTO_TEST_CASE(storeAndRetrieve, *utf::precondition(if_reachable())) diff --git a/CCDB/test/testCcdbApi_ConfigParam.cxx b/CCDB/test/testCcdbApi_ConfigParam.cxx index 57335505cfab6..568669d05978f 100644 --- a/CCDB/test/testCcdbApi_ConfigParam.cxx +++ b/CCDB/test/testCcdbApi_ConfigParam.cxx @@ -17,6 +17,7 @@ #define BOOST_TEST_MODULE CCDB #define BOOST_TEST_MAIN #define BOOST_TEST_DYN_LINK +#define BOOST_BIND_GLOBAL_PLACEHOLDERS #include "CCDB/CcdbApi.h" #include "CommonUtils/ConfigurableParam.h" @@ -46,8 +47,8 @@ using namespace o2::ccdb; namespace utf = boost::unit_test; namespace tt = boost::test_tools; -static string ccdbUrl; -static string basePath; +static std::string ccdbUrl; +static std::string basePath; bool hostReachable = false; /** @@ -61,15 +62,16 @@ struct Fixture { api.init(ccdbUrl); cout << "ccdb url: " << ccdbUrl << endl; hostReachable = api.isHostReachable(); - cout << "Is host reachable ? --> " << hostReachable << endl; - basePath = string("Test/pid") + getpid() + "/"; + char hostname[_POSIX_HOST_NAME_MAX]; + gethostname(hostname, _POSIX_HOST_NAME_MAX); + basePath = std::string("Test/") + hostname + "/pid" + getpid() + "/"; cout << "Path we will use in this test suite : " + basePath << endl; } ~Fixture() { if (hostReachable) { CcdbApi api; - map metadata; + std::map metadata; api.init(ccdbUrl); api.truncate(basePath + "*"); cout << "Test data truncated (" << basePath << ")" << endl; @@ -101,7 +103,7 @@ struct test_fixture { ~test_fixture() = default; CcdbApi api; - map metadata; + std::map metadata; }; BOOST_AUTO_TEST_CASE(testConfigParamRetrieval, *utf::precondition(if_reachable())) @@ -133,4 +135,4 @@ BOOST_AUTO_TEST_CASE(testConfigParamRetrieval, *utf::precondition(if_reachable() BOOST_CHECK(p1.getMemberProvenance("dbscanDeltaT") == o2::conf::ConfigurableParam::EParamProvenance::kRT); BOOST_CHECK(p1.getMemberProvenance("useMeanVertexConstraint") == o2::conf::ConfigurableParam::EParamProvenance::kCCDB); BOOST_CHECK(object == &p1); -} \ No newline at end of file +} diff --git a/CCDB/test/testCcdbApi_alien.cxx b/CCDB/test/testCcdbApi_alien.cxx index f11f579346524..c50f83466fe06 100644 --- a/CCDB/test/testCcdbApi_alien.cxx +++ b/CCDB/test/testCcdbApi_alien.cxx @@ -29,7 +29,7 @@ using namespace o2::ccdb; namespace utf = boost::unit_test; namespace tt = boost::test_tools; -static string ccdbUrl; +static std::string ccdbUrl; bool hostReachable = false; /** @@ -71,7 +71,7 @@ struct test_fixture { ~test_fixture() = default; CcdbApi api; - map metadata; + std::map metadata; }; // handle the case where the object comes from alien and redirect does not work with curl diff --git a/CMakeLists.txt b/CMakeLists.txt index cd774e6272769..adecffc0f4dbf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,7 +11,7 @@ # Preamble -cmake_minimum_required(VERSION 3.19 FATAL_ERROR) +cmake_minimum_required(VERSION 3.27.1 FATAL_ERROR) # it's important to specify accurately the list of languages. for instance C and # C++ as we _do_ have some C files to compile explicitely as C (e.g. gl3w.c) @@ -25,6 +25,7 @@ include(CTest) set(ALIGPU_BUILD_TYPE "O2") set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake) +string(TOUPPER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_UPPER) set_property(GLOBAL PROPERTY REPORT_UNDEFINED_PROPERTIES) cmake_host_system_information(RESULT _totalmem QUERY TOTAL_PHYSICAL_MEMORY) @@ -37,14 +38,9 @@ set_property(GLOBAL PROPERTY JOB_POOLS analysis=${ANALYSIS_COMPILE_POOL}) include(O2BuildSanityChecks) o2_build_sanity_checks() -set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED TRUE) -find_package(ONNXRuntime::ONNXRuntime CONFIG) -if (ONNXRuntime::ONNXRuntime_FOUND) - add_definitions(-DZDC_FASTSIM_ONNX) -endif() - include(O2CheckCXXFeatures) o2_check_cxx_features() @@ -90,9 +86,10 @@ include(O2AddTestRootMacro) include(O2ReportNonTestedMacros) include(O2TargetRootDictionary) include(O2DataFile) -include(O2TargetManPage) include(O2AddWorkflow) include(O2SetROOTPCMDependencies) +include(O2AddHipifiedExecutable) +include(O2AddHipifiedLibrary) # Main targets of the project in various subdirectories. Order matters. add_subdirectory(Common) @@ -119,10 +116,6 @@ endif() add_subdirectory(config) -add_custom_target(man ALL) -o2_target_man_page(man NAME o2) -o2_target_man_page(man NAME FairMQDevice) - # Testing and packaging only needed if we are the top level directory if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) # Documentation diff --git a/CODEOWNERS b/CODEOWNERS index 5721d79be9997..26021d458ad76 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -28,26 +28,28 @@ /DataFormats/Detectors/Common @shahor02 /DataFormats/Detectors/CPV @peressounko @kharlov /DataFormats/Detectors/CTP @lietava -/DataFormats/Detectors/EMCAL @mfasDa @jokonig -/DataFormats/Detectors/FIT @AllaMaevskaya @jotwinow @mslupeck +/DataFormats/Detectors/EMCAL @nstrangm @jokonig +/DataFormats/Detectors/FIT @jotwinow @afurs @andreasmolander @sahilupadhyaya92 +/DataFormats/Detectors/FOCAL @maxrauch @iarsene @matthiasrichter /DataFormats/Detectors/GlobalTracking @shahor02 /DataFormats/Detectors/GlobalTrackingWorkflow @shahor02 /DataFormats/Detectors/HMPID @gvolpe79 -/DataFormats/Detectors/ITSMFT @iouribelikov @bovulpes @rpezzi -/DataFormats/Detectors/MUON @AliceO2Group/muon-experts +/DataFormats/Detectors/ITSMFT @fprino @mcoquet642 @shahor02 +/DataFormats/Detectors/MUON @AliceO2Group/muon-experts @shahor02 /DataFormats/Detectors/PHOS @peressounko @kharlov /DataFormats/Detectors/Passive @sawenzel /DataFormats/Detectors/TOF @noferini /DataFormats/Detectors/TPC @davidrohr @wiechula @shahor02 -/DataFormats/Detectors/TRD @jolopezl @bazinski @tdietel @martenole -/DataFormats/Detectors/Upgrades @qgp @marcovanleeuwen @mconcas +/DataFormats/Detectors/TRD @f3sch @bazinski @wille10 +/DataFormats/Detectors/Upgrades @mconcas +/DataFormats/Detectors/Upgrades/ITS3 @fgrosa @arossi81 /DataFormats/Detectors/ZDC @coppedis #/DataFormats/Headers #/DataFormats/Legacy #/DataFormats/MemoryResources /DataFormats/Parameters @shahor02 -/DataFormats/QualityControl @knopers8 @Barthelemy @chiarazampolli +/DataFormats/QualityControl @knopers8 @Barthelemy @justonedev1 @chiarazampolli /DataFormats/Reconstruction @shahor02 #/DataFormats/TimeFrame /DataFormats/common @shahor02 @@ -56,23 +58,27 @@ /Detectors/Base @sawenzel @shahor02 /Detectors/Calibration @chiarazampolli @shahor02 /Detectors/CPV @peressounko @kharlov -/Detectors/EMCAL @mfasDa @jokonig -/Detectors/FIT @AllaMaevskaya @jotwinow @mslupeck +/Detectors/EMCAL @nstrangm @jokonig +/Detectors/FIT @jotwinow @afurs @andreasmolander @sahilupadhyaya92 +/Detectors/FOCAL @maxrauch @iarsene @matthiasrichter /Detectors/Geometry @sawenzel @shahor02 /Detectors/GlobalTracking @shahor02 /Detectors/GlobalTrackingWorkflow @shahor02 /Detectors/HMPID @gvolpe79 -/Detectors/ITSMFT @iouribelikov @bovulpes @rpezzi @mconcas -/Detectors/MUON @AliceO2Group/muon-experts +/Detectors/ITSMFT @fprino @mcoquet642 @mconcas @shahor02 +/Detectors/MUON @AliceO2Group/muon-experts @shahor02 /Detectors/PHOS @peressounko @kharlov /Detectors/Passive @sawenzel /Detectors/TOF @noferini /Detectors/TPC @davidrohr @wiechula @shahor02 -/Detectors/TRD @jolopezl @bazinski @tdietel @martenole -/Detectors/Upgrades @qgp @marcovanleeuwen @mconcas +/Detectors/TRD @f3sch @bazinski @wille10 +/Detectors/Upgrades @mconcas +/Detectors/Upgrades/ALICE3 @mconcas @njacazio +/Detectors/Upgrades/ITS3 @fgrosa @arossi81 @mconcas @f3sch /Detectors/ZDC @coppedis /Detectors/CTF @shahor02 /Detectors/Raw @shahor02 +/Detectors/StrangenessTracking @mconcas @mpuccio @fmazzasc /EventVisualisation @jmyrcha #/EventVisualisation/Base @@ -105,19 +111,19 @@ /GPU @davidrohr #/GPU/Common #/GPU/GPUTracking -/GPU/GPUTracking/TRDTracking @davidrohr @martenole +/GPU/GPUTracking/TRDTracking @davidrohr /GPU/TPCFastTransformation @davidrohr @sgorbuno /GPU/TPCSpaceChargeBase @davidrohr @ehellbar -/Generators @sawenzel +/Generators @sawenzel @jackal1-66 /Steer @sawenzel @shahor02 /Testing /Utilities @AliceO2Group/framework-admins -/Utilities/Mergers @Barthelemy @knopers8 -/Utilities/DataSampling @Barthelemy @knopers8 +/Utilities/Mergers @Barthelemy @knopers8 @justonedev1 +/Utilities/DataSampling @Barthelemy @knopers8 @justonedev1 #/Utilities/DataCompression #/Utilities/DataFlow #/Utilities/MCStepLogger diff --git a/Common/CMakeLists.txt b/Common/CMakeLists.txt index bfb4a24ff3bc9..0b92758e45f43 100644 --- a/Common/CMakeLists.txt +++ b/Common/CMakeLists.txt @@ -15,5 +15,7 @@ add_subdirectory(Field) add_subdirectory(Types) add_subdirectory(Utils) add_subdirectory(SimConfig) +add_subdirectory(DCAFitter) +add_subdirectory(ML) o2_data_file(COPY maps DESTINATION Common) diff --git a/Common/Constants/include/CommonConstants/MathConstants.h b/Common/Constants/include/CommonConstants/MathConstants.h index 03368bb6f8bf9..9ef3b4dba5ae0 100644 --- a/Common/Constants/include/CommonConstants/MathConstants.h +++ b/Common/Constants/include/CommonConstants/MathConstants.h @@ -22,8 +22,9 @@ namespace constants { namespace math { -constexpr float Almost0 = 1.17549e-38; -constexpr float Almost1 = 1.f - Almost0; +constexpr float Almost0 = 0x1.0p-126f; // smallest non-denormal float +constexpr float Epsilon = 0x0.000002p0f; // smallest float such that 1 != 1 + Epsilon +constexpr float Almost1 = 1.f - 1.0e-6f; constexpr float VeryBig = 1.f / Almost0; constexpr float PI = 3.14159274101257324e+00f; diff --git a/Common/Constants/include/CommonConstants/PhysicsConstants.h b/Common/Constants/include/CommonConstants/PhysicsConstants.h index d3342f1df6f64..46aeff98d6033 100644 --- a/Common/Constants/include/CommonConstants/PhysicsConstants.h +++ b/Common/Constants/include/CommonConstants/PhysicsConstants.h @@ -12,39 +12,232 @@ /// \file PhysicsConstants.h /// \brief Header to collect physics constants /// \author ruben.shahoyan@cern.ch +/// \author Vít Kučera , Inha University +/// \note Use the make_pdg_header.py script to generate the enums and mass declarations. #ifndef ALICEO2_PHYSICSCONSTANTS_H_ #define ALICEO2_PHYSICSCONSTANTS_H_ -namespace o2 -{ -namespace constants -{ -namespace physics +namespace o2::constants::physics { // particles masses -constexpr float MassPhoton = 0.0; -constexpr float MassElectron = 0.000511; -constexpr float MassMuon = 0.105658; -constexpr float MassPionCharged = 0.139570; -constexpr float MassPionNeutral = 0.134976; -constexpr float MassKaonCharged = 0.493677; -constexpr float MassKaonNeutral = 0.497648; -constexpr float MassProton = 0.938272; -constexpr float MassLambda = 1.115683; -constexpr float MassDeuteron = 1.8756129; -constexpr float MassTriton = 2.8089211; -constexpr float MassHelium3 = 2.8083916; -constexpr float MassAlpha = 3.7273794; -constexpr float MassHyperTriton = 2.992; -constexpr float MassHyperhydrog4 = 3.931; -constexpr float MassXiMinus = 1.32171; -constexpr float MassOmegaMinus = 1.67245; - -constexpr float LightSpeedCm2S = 299792458.e2; // C in cm/s -constexpr float LightSpeedCm2NS = LightSpeedCm2S * 1e-9; // C in cm/ns -} // namespace physics -} // namespace constants -} // namespace o2 + +// BEGINNING OF THE GENERATED BLOCK. +// DO NOT EDIT THIS BLOCK DIRECTLY! +// It has been generated by the make_pdg_header.py script. +// For modifications, edit the script and generate this block again. + +/// \brief Declarations of named PDG codes of particles missing in ROOT PDG_t +/// \note Follow kCamelCase naming convention +/// \link https://root.cern/doc/master/TPDGCode_8h.html +enum Pdg { + kEta = 221, + kOmega = 223, + kEtaPrime = 331, + kB0 = 511, + kB0Bar = -511, + kBPlus = 521, + kBCPlus = 541, + kBS = 531, + kBSBar = -531, + kD0 = 421, + kD0Bar = -421, + kD0StarPlus = 10411, + kD0Star0 = 10421, + kD1Plus = 20413, + kD10 = 20423, + kD2StarPlus = 415, + kD2Star0 = 425, + kDMinus = -411, + kDPlus = 411, + kDS = 431, + kDSBar = -431, + kDSStar = 433, + kDS1 = 10433, + kDS1Star2700 = 30433, + kDS1Star2860 = 40433, + kDS2Star = 435, + kDS3Star2860 = 437, + kDStar = 413, + kDStar0 = 423, + kChiC1 = 20443, + kJPsi = 443, + kLambdaB0 = 5122, + kLambdaCPlus = 4122, + kOmegaC0 = 4332, + kK0Star892 = 313, + kKPlusStar892 = 323, + kPhi = 333, + kSigmaC0 = 4112, + kSigmaCPlusPlus = 4222, + kSigmaCStar0 = 4114, + kSigmaCStarPlusPlus = 4224, + kX3872 = 9920443, + kXi0 = 3322, + kXiB0 = 5232, + kXiCCPlusPlus = 4422, + kXiCPlus = 4232, + kXiC0 = 4132, + kXiC3055Plus = 4325, + kXiC3080Plus = 4326, + kXiC3055_0 = 4315, + kXiC3080_0 = 4316, + kDeuteron = 1000010020, + kTriton = 1000010030, + kHelium3 = 1000020030, + kAlpha = 1000020040, + kLithium4 = 1000030040, + kHyperTriton = 1010010030, + kHyperHydrogen4 = 1010010040, + kHyperHelium4 = 1010020040, + kHyperHelium5 = 1010020050, + kHyperHelium4Sigma = 1110020040, + kLambda1520_Py = 102134 +}; + +/// \brief Declarations of masses for additional particles +constexpr double MassEta = 0.547862; +constexpr double MassOmega = 0.78266; +constexpr double MassEtaPrime = 0.95778; +constexpr double MassB0 = 5.27966; +constexpr double MassB0Bar = 5.27966; +constexpr double MassBPlus = 5.27934; +constexpr double MassBCPlus = 6.27447; +constexpr double MassBS = 5.36692; +constexpr double MassBSBar = 5.36692; +constexpr double MassD0 = 1.86484; +constexpr double MassD0Bar = 1.86484; +constexpr double MassD0StarPlus = 2.272; +constexpr double MassD0Star0 = 2.343; +constexpr double MassD1Plus = 2.372; +constexpr double MassD10 = 2.412; +constexpr double MassD2StarPlus = 2.4601; +constexpr double MassD2Star0 = 2.4611; +constexpr double MassDMinus = 1.86966; +constexpr double MassDPlus = 1.86966; +constexpr double MassDS = 1.96835; +constexpr double MassDSBar = 1.96835; +constexpr double MassDSStar = 2.1122; +constexpr double MassDS1 = 2.53511; +constexpr double MassDS1Star2700 = 2.714; +constexpr double MassDS1Star2860 = 2.859; +constexpr double MassDS2Star = 2.5691; +constexpr double MassDS3Star2860 = 2.86; +constexpr double MassDStar = 2.01026; +constexpr double MassDStar0 = 2.00685; +constexpr double MassChiC1 = 3.51067; +constexpr double MassJPsi = 3.0969; +constexpr double MassLambdaB0 = 5.6196; +constexpr double MassLambdaCPlus = 2.28646; +constexpr double MassOmegaC0 = 2.6952; +constexpr double MassK0Star892 = 0.89555; +constexpr double MassKPlusStar892 = 0.89167; +constexpr double MassPhi = 1.019461; +constexpr double MassSigmaC0 = 2.45375; +constexpr double MassSigmaCPlusPlus = 2.45397; +constexpr double MassSigmaCStar0 = 2.51848; +constexpr double MassSigmaCStarPlusPlus = 2.51841; +constexpr double MassX3872 = 3.87165; +constexpr double MassXi0 = 1.31486; +constexpr double MassXiB0 = 5.7919; +constexpr double MassXiCCPlusPlus = 3.62155; +constexpr double MassXiCPlus = 2.46771; +constexpr double MassXiC0 = 2.47044; +constexpr double MassXiC3055Plus = 3.0559; +constexpr double MassXiC3080Plus = 3.0772; +constexpr double MassXiC3055_0 = 3.059; +constexpr double MassXiC3080_0 = 3.0799; +constexpr double MassDeuteron = 1.87561294257; +constexpr double MassTriton = 2.80892113298; +constexpr double MassHelium3 = 2.80839160743; +constexpr double MassAlpha = 3.7273794066; +constexpr double MassLithium4 = 3.7513; +constexpr double MassHyperTriton = 2.991134; +constexpr double MassHyperHydrogen4 = 3.922434; +constexpr double MassHyperHelium4 = 3.921728; +constexpr double MassHyperHelium5 = 4.839961; +constexpr double MassHyperHelium4Sigma = 3.995; +constexpr double MassLambda1520_Py = 1.5195; + +/// \brief Declarations of masses for particles in ROOT PDG_t +constexpr double MassDown = 0.00467; +constexpr double MassDownBar = 0.00467; +constexpr double MassUp = 0.00216; +constexpr double MassUpBar = 0.00216; +constexpr double MassStrange = 0.0934; +constexpr double MassStrangeBar = 0.0934; +constexpr double MassCharm = 1.27; +constexpr double MassCharmBar = 1.27; +constexpr double MassBottom = 4.18; +constexpr double MassBottomBar = 4.18; +constexpr double MassTop = 172.5; +constexpr double MassTopBar = 172.5; +constexpr double MassGluon = 0.0; +constexpr double MassElectron = 0.000510999; +constexpr double MassPositron = 0.000510999; +constexpr double MassNuE = 0.0; +constexpr double MassNuEBar = 0.0; +constexpr double MassMuonMinus = 0.1056584; +constexpr double MassMuonPlus = 0.1056584; +constexpr double MassNuMu = 0.0; +constexpr double MassNuMuBar = 0.0; +constexpr double MassTauMinus = 1.77686; +constexpr double MassTauPlus = 1.77686; +constexpr double MassNuTau = 0.0; +constexpr double MassNuTauBar = 0.0; +constexpr double MassGamma = 0.0; +constexpr double MassZ0 = 91.1876; +constexpr double MassWPlus = 80.377; +constexpr double MassWMinus = 80.377; +constexpr double MassPi0 = 0.1349768; +constexpr double MassK0Long = 0.497611; +constexpr double MassPiPlus = 0.1395704; +constexpr double MassPiMinus = 0.1395704; +constexpr double MassProton = 0.9382721; +constexpr double MassProtonBar = 0.9382721; +constexpr double MassNeutron = 0.9395654; +constexpr double MassNeutronBar = 0.9395654; +constexpr double MassK0Short = 0.497611; +constexpr double MassK0 = 0.497611; +constexpr double MassK0Bar = 0.497611; +constexpr double MassKPlus = 0.493677; +constexpr double MassKMinus = 0.493677; +constexpr double MassLambda0 = 1.115683; +constexpr double MassLambda0Bar = 1.115683; +constexpr double MassLambda1520 = 1.519; +constexpr double MassSigmaMinus = 1.197449; +constexpr double MassSigmaBarPlus = 1.197449; +constexpr double MassSigmaPlus = 1.18937; +constexpr double MassSigmaBarMinus = 1.18937; +constexpr double MassSigma0 = 1.192642; +constexpr double MassSigma0Bar = 1.192642; +constexpr double MassXiMinus = 1.32171; +constexpr double MassXiPlusBar = 1.32171; +constexpr double MassOmegaMinus = 1.67245; +constexpr double MassOmegaPlusBar = 1.67245; + +// END OF THE GENERATED BLOCK + +// legacy names +constexpr double MassPhoton = MassGamma; +constexpr double MassMuon = MassMuonMinus; +constexpr double MassPionCharged = MassPiPlus; +constexpr double MassPionNeutral = MassPi0; +constexpr double MassKaonCharged = MassKPlus; +constexpr double MassKaonNeutral = MassK0; +constexpr double MassLambda = MassLambda0; +constexpr double MassHyperhydrog4 = MassHyperHydrogen4; +constexpr double MassHyperhelium4 = MassHyperHelium4; +constexpr double MassHyperhelium4sigma = MassHyperHelium4Sigma; + +// Light speed +constexpr float LightSpeedCm2S = 299792458.e2; // C in cm/s +constexpr float LightSpeedCm2NS = LightSpeedCm2S * 1e-9; // C in cm/ns +constexpr float LightSpeedCm2PS = LightSpeedCm2S * 1e-12; // C in cm/ps + +// Light speed inverse +constexpr float invLightSpeedCm2PS = 1. / LightSpeedCm2PS; // 1/C in ps/cm + +} // namespace o2::constants::physics #endif diff --git a/Common/Constants/include/CommonConstants/ZDCConstants.h b/Common/Constants/include/CommonConstants/ZDCConstants.h new file mode 100644 index 0000000000000..a07de17abff65 --- /dev/null +++ b/Common/Constants/include/CommonConstants/ZDCConstants.h @@ -0,0 +1,63 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// @file ZDCConstants.h +/// @brief Some ZDC constants shared between O2 and O2Physics + +/// @author pietro.cortese@cern.ch + +#ifndef ALICEO2_ZDCCONSTANTS_H_ +#define ALICEO2_ZDCCONSTANTS_H_ + +namespace o2 +{ +namespace zdc +{ + +//< map detector/tower to continuous channel Id +constexpr int IdDummy = -1; +constexpr int IdVoid = -2; + +constexpr int IdZNAC = 0; +constexpr int IdZNA1 = 1; +constexpr int IdZNA2 = 2; +constexpr int IdZNA3 = 3; +constexpr int IdZNA4 = 4; +constexpr int IdZNASum = 5; +// +constexpr int IdZPAC = 6; +constexpr int IdZPA1 = 7; +constexpr int IdZPA2 = 8; +constexpr int IdZPA3 = 9; +constexpr int IdZPA4 = 10; +constexpr int IdZPASum = 11; +// +constexpr int IdZEM1 = 12; +constexpr int IdZEM2 = 13; +// +constexpr int IdZNCC = 14; +constexpr int IdZNC1 = 15; +constexpr int IdZNC2 = 16; +constexpr int IdZNC3 = 17; +constexpr int IdZNC4 = 18; +constexpr int IdZNCSum = 19; +// +constexpr int IdZPCC = 20; +constexpr int IdZPC1 = 21; +constexpr int IdZPC2 = 22; +constexpr int IdZPC3 = 23; +constexpr int IdZPC4 = 24; +constexpr int IdZPCSum = 25; + +} // namespace zdc +} // namespace o2 + +#endif diff --git a/Common/Constants/include/CommonConstants/make_pdg_header.py b/Common/Constants/include/CommonConstants/make_pdg_header.py new file mode 100755 index 0000000000000..f83c44bb401db --- /dev/null +++ b/Common/Constants/include/CommonConstants/make_pdg_header.py @@ -0,0 +1,271 @@ +#!/usr/bin/env python3 + +# Copyright 2019-2020 CERN and copyright holders of ALICE O2. +# See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +# All rights not expressly granted are reserved. +# +# This software is distributed under the terms of the GNU General Public +# License v3 (GPL Version 3), copied verbatim in the file "COPYING". +# +# In applying this license CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization +# or submit itself to any jurisdiction. + +"""! +@brief Generates and updates the body of a C++ header with PDG codes and particle masses. +@author Vít Kučera , Inha University +@date 2023-09-21 +""" + +import os +from ctypes import c_bool +from enum import Enum + +try: + import ROOT # pylint: disable=import-error + from ROOT import o2 +except (ModuleNotFoundError, ImportError) as exc: + raise OSError("O2 environment is not loaded.") from exc + + +# Enum of PDG_t particles +class PdgROOT(Enum): + kDown = ROOT.kDown + kDownBar = ROOT.kDownBar + kUp = ROOT.kUp + kUpBar = ROOT.kUpBar + kStrange = ROOT.kStrange + kStrangeBar = ROOT.kStrangeBar + kCharm = ROOT.kCharm + kCharmBar = ROOT.kCharmBar + kBottom = ROOT.kBottom + kBottomBar = ROOT.kBottomBar + kTop = ROOT.kTop + kTopBar = ROOT.kTopBar + kGluon = ROOT.kGluon + kElectron = ROOT.kElectron + kPositron = ROOT.kPositron + kNuE = ROOT.kNuE + kNuEBar = ROOT.kNuEBar + kMuonMinus = ROOT.kMuonMinus + kMuonPlus = ROOT.kMuonPlus + kNuMu = ROOT.kNuMu + kNuMuBar = ROOT.kNuMuBar + kTauMinus = ROOT.kTauMinus + kTauPlus = ROOT.kTauPlus + kNuTau = ROOT.kNuTau + kNuTauBar = ROOT.kNuTauBar + kGamma = ROOT.kGamma + kZ0 = ROOT.kZ0 + kWPlus = ROOT.kWPlus + kWMinus = ROOT.kWMinus + kPi0 = ROOT.kPi0 + kK0Long = ROOT.kK0Long + kPiPlus = ROOT.kPiPlus + kPiMinus = ROOT.kPiMinus + kProton = ROOT.kProton + kProtonBar = ROOT.kProtonBar + kNeutron = ROOT.kNeutron + kNeutronBar = ROOT.kNeutronBar + kK0Short = ROOT.kK0Short + kK0 = ROOT.kK0 + kK0Bar = ROOT.kK0Bar + kKPlus = ROOT.kKPlus + kKMinus = ROOT.kKMinus + kLambda0 = ROOT.kLambda0 + kLambda0Bar = ROOT.kLambda0Bar + kLambda1520 = ROOT.kLambda1520 + kSigmaMinus = ROOT.kSigmaMinus + kSigmaBarPlus = ROOT.kSigmaBarPlus + kSigmaPlus = ROOT.kSigmaPlus + kSigmaBarMinus = ROOT.kSigmaBarMinus + kSigma0 = ROOT.kSigma0 + kSigma0Bar = ROOT.kSigma0Bar + kXiMinus = ROOT.kXiMinus + kXiPlusBar = ROOT.kXiPlusBar + kOmegaMinus = ROOT.kOmegaMinus + kOmegaPlusBar = ROOT.kOmegaPlusBar + + +# Enum of additional particles +class Pdg(Enum): + kEta = 221 + kOmega = 223 + kEtaPrime = 331 + kB0 = 511 + kB0Bar = -511 + kBPlus = 521 + kBCPlus = 541 + kBS = 531 + kBSBar = -531 + kD0 = 421 + kD0Bar = -421 + kD0StarPlus = 10411 + kD0Star0 = 10421 + kD1Plus = 20413 + kD10 = 20423 + kD2StarPlus = 415 + kD2Star0 = 425 + kDMinus = -411 + kDPlus = 411 + kDS = 431 + kDSBar = -431 + kDSStar = 433 + kDS1 = 10433 + kDS1Star2700 = 30433 + kDS1Star2860 = 40433 + kDS2Star = 435 + kDS3Star2860 = 437 + kDStar = 413 + kDStar0 = 423 + kChiC1 = 20443 + kJPsi = 443 + kLambdaB0 = 5122 + kLambdaCPlus = 4122 + kOmegaC0 = 4332 + kK0Star892 = 313 + kKPlusStar892 = 323 + kPhi = 333 + kSigmaC0 = 4112 + kSigmaCPlusPlus = 4222 + kSigmaCStar0 = 4114 + kSigmaCStarPlusPlus = 4224 + kX3872 = 9920443 + kXi0 = 3322 + kXiB0 = 5232 + kXiCCPlusPlus = 4422 + kXiCPlus = 4232 + kXiC0 = 4132 + kXiC3055Plus = 4325 + kXiC3080Plus = 4326 + kXiC3055_0 = 4315 + kXiC3080_0 = 4316 + kDeuteron = 1000010020 + kTriton = 1000010030 + kHelium3 = 1000020030 + kAlpha = 1000020040 + kLithium4 = 1000030040 + kHyperTriton = 1010010030 + kHyperHydrogen4 = 1010010040 + kHyperHelium4 = 1010020040 + kHyperHelium5 = 1010020050 + kHyperHelium4Sigma = 1110020040 + kLambda1520_Py = 102134 # PYTHIA code different from PDG + + +dbPdg = o2.O2DatabasePDG + + +def mass(code): + """Returns particle mass from o2::O2DatabasePDG.""" + # Missing particles should be added in O2DatabasePDG.h. + success = c_bool(True) + return dbPdg.Mass(code, success) + + +def declare_mass(pdg, mass_type="double") -> str: + """Returns a C++ declaration of a particle mass constant.""" + return f"constexpr {mass_type} Mass{pdg.name[1:]} = {mass(pdg.value)};" + + +def main(): + """Main function""" + + path_header = "PhysicsConstants.h" + name_script = os.path.basename(__file__) + + # Comment at the beginning of the output + block_begin = "// BEGINNING OF THE GENERATED BLOCK." + # Comment at the end of the output + block_end = "// END OF THE GENERATED BLOCK" + # Preamble with instructions + block_preamble = ( + "// DO NOT EDIT THIS BLOCK DIRECTLY!" + f"\n// It has been generated by the {name_script} script." + "\n// For modifications, edit the script and generate this block again." + ) + # Start of enum declarations of additional particles + enum_o2_head = ( + "/// \\brief Declarations of named PDG codes of particles missing in ROOT PDG_t" + "\n/// \\note Follow kCamelCase naming convention" + "\n/// \\link https://root.cern/doc/master/TPDGCode_8h.html" + "\nenum Pdg {" + ) + # End of enum declarations of additional particles + enum_o2_foot = "};" + # Documentation string for mass declarations of additional particles + mass_o2_head = "/// \\brief Declarations of masses for additional particles" + # Documentation string for mass declarations of PDG_t particles + mass_root_head = "/// \\brief Declarations of masses for particles in ROOT PDG_t" + + # Get header content before and after the generated block. + print(f'File "{path_header}" will be updated.') + try: + with open(path_header, encoding="utf-8") as file: + content_old = file.readlines() + except OSError as exc: + raise OSError(f'Failed to open file "{path_header}".') from exc + lines_header_before: list[str] = [] + lines_header_after: list[str] = [] + got_block_begin = False + got_block_end = False + for line in content_old: + line = line.strip() + if line == block_begin: + got_block_begin = True + if not got_block_begin: + lines_header_before.append(line) + if got_block_end: + lines_header_after.append(line) + if line == block_end: + got_block_end = True + if not got_block_begin: + raise ValueError("Did not find the beginning of the block.") + if not got_block_end: + raise ValueError("Did not find the end of the block.") + + # Additional particles + lines_enum_o2: list[str] = [enum_o2_head] + lines_mass_o2: list[str] = [mass_o2_head] + for pdg_o2 in Pdg: + lines_enum_o2.append(f" {pdg_o2.name} = {pdg_o2.value},") + lines_mass_o2.append(declare_mass(pdg_o2)) + lines_enum_o2[-1] = lines_enum_o2[-1][:-1] # Remove the last comma. + lines_enum_o2.append(enum_o2_foot) + + # PDG_t particles + lines_mass_root: list[str] = [mass_root_head] + for pdg_root in PdgROOT: + lines_mass_root.append(declare_mass(pdg_root)) + + # Header body + content_new = "\n".join( + ( + *lines_header_before, + block_begin, + block_preamble, + "", + *lines_enum_o2, + "", + *lines_mass_o2, + "", + *lines_mass_root, + "", + block_end, + *lines_header_after, + "", + ) + ) + # print(content_new) + + # Overwrite the input file. + try: + with open(path_header, "w", encoding="utf-8") as file: + file.write(content_new) + print(f'File "{path_header}" has been overwritten.') + except OSError as exc: + raise OSError(f'Failed to write to file "{path_header}".') from exc + + +if __name__ == "__main__": + main() diff --git a/Common/DCAFitter/CMakeLists.txt b/Common/DCAFitter/CMakeLists.txt new file mode 100644 index 0000000000000..c0b2d0dca1026 --- /dev/null +++ b/Common/DCAFitter/CMakeLists.txt @@ -0,0 +1,43 @@ +# Copyright 2019-2020 CERN and copyright holders of ALICE O2. +# See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +# All rights not expressly granted are reserved. +# +# This software is distributed under the terms of the GNU General Public +# License v3 (GPL Version 3), copied verbatim in the file "COPYING". +# +# In applying this license CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization +# or submit itself to any jurisdiction. + +# add_compile_options(-O0 -g -fPIC) + +o2_add_library(DCAFitter + TARGETVARNAME targetName + SOURCES src/DCAFitterN.cxx + src/FwdDCAFitterN.cxx + PUBLIC_LINK_LIBRARIES ROOT::Core + O2::CommonUtils + O2::ReconstructionDataFormats + O2::DataFormatsParameters + O2::DetectorsBase) + +o2_target_root_dictionary(DCAFitter + HEADERS include/DCAFitter/DCAFitterN.h + include/DCAFitter/FwdDCAFitterN.h) + +if (OpenMP_CXX_FOUND) + target_compile_definitions(${targetName} PRIVATE WITH_OPENMP) + target_link_libraries(${targetName} PRIVATE OpenMP::OpenMP_CXX) +endif() + + +o2_add_test( + DCAFitterN + SOURCES test/testDCAFitterN.cxx + COMPONENT_NAME DCAFitter + PUBLIC_LINK_LIBRARIES O2::DCAFitter ROOT::Core ROOT::Physics + LABELS vertexing + ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage + VMCWORKDIR=${CMAKE_BINARY_DIR}/stage/${CMAKE_INSTALL_DATADIR}) + +add_subdirectory(GPU) diff --git a/Common/DCAFitter/GPU/CMakeLists.txt b/Common/DCAFitter/GPU/CMakeLists.txt new file mode 100644 index 0000000000000..faf15f8aab3df --- /dev/null +++ b/Common/DCAFitter/GPU/CMakeLists.txt @@ -0,0 +1,18 @@ +# Copyright 2019-2020 CERN and copyright holders of ALICE O2. +# See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +# All rights not expressly granted are reserved. +# +# This software is distributed under the terms of the GNU General Public +# License v3 (GPL Version 3), copied verbatim in the file "COPYING". +# +# In applying this license CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization +# or submit itself to any jurisdiction. + +if (CUDA_ENABLED) +add_subdirectory(cuda) +endif() + +if (HIP_ENABLED) +add_subdirectory(hip) +endif() \ No newline at end of file diff --git a/Common/DCAFitter/GPU/DeviceInterface/GPUInterface.h b/Common/DCAFitter/GPU/DeviceInterface/GPUInterface.h new file mode 100644 index 0000000000000..3aa5ead805acd --- /dev/null +++ b/Common/DCAFitter/GPU/DeviceInterface/GPUInterface.h @@ -0,0 +1,71 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \brief Helper interface to the GPU device, meant to be compatible with manual allocation/streams and GPUReconstruction ones. +/// \author matteo.concas@cern.ch + +#ifndef DCAFITTER_GPU_INTERFACE +#define DCAFITTER_GPU_INTERFACE + +#include +#include +#include + +namespace o2 +{ +namespace vertexing +{ +namespace device +{ + +#if !defined(__HIPCC__) && !defined(__CUDACC__) +typedef struct _dummyStream { +} Stream; +#else +#ifdef __HIPCC__ +typedef hipStream_t Stream; +#else +typedef cudaStream_t Stream; +#endif +#endif + +class GPUInterface +{ + public: + GPUInterface(GPUInterface& other) = delete; + void operator=(const GPUInterface&) = delete; + + static GPUInterface* Instance(); + + // APIs + void registerBuffer(void*, size_t); + void unregisterBuffer(void* addr); + void allocDevice(void**, size_t); + void freeDevice(void*); + Stream& getStream(unsigned short N = 0); + Stream& getNextStream(); + + protected: + GPUInterface(size_t N = 1); + ~GPUInterface(); + + void resize(size_t); + + std::atomic mLastUsedStream{0}; + static GPUInterface* sGPUInterface; + std::vector mPool{}; + std::vector mStreams{}; +}; + +} // namespace device +} // namespace vertexing +} // namespace o2 +#endif diff --git a/Common/DCAFitter/GPU/cuda/CMakeLists.txt b/Common/DCAFitter/GPU/cuda/CMakeLists.txt new file mode 100644 index 0000000000000..6b89207279fe0 --- /dev/null +++ b/Common/DCAFitter/GPU/cuda/CMakeLists.txt @@ -0,0 +1,35 @@ +# Copyright 2019-2020 CERN and copyright holders of ALICE O2. +# See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +# All rights not expressly granted are reserved. +# +# This software is distributed under the terms of the GNU General Public +# License v3 (GPL Version 3), copied verbatim in the file "COPYING". +# +# In applying this license CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization +# or submit itself to any jurisdiction. + +o2_add_library(DCAFitterCUDA + TARGETVARNAME targetName + SOURCES DCAFitterN.cu + GPUInterface.cu + PUBLIC_INCLUDE_DIRECTORIES ../../include + PUBLIC_INCLUDE_DIRECTORIES ../ + PUBLIC_LINK_LIBRARIES O2::MathUtils + O2::ReconstructionDataFormats + O2::DetectorsBase + PRIVATE_LINK_LIBRARIES O2::GPUTrackingCUDAExternalProvider) +set_property(TARGET ${targetName} PROPERTY CUDA_SEPARABLE_COMPILATION ON) +# add_compile_options(-lineinfo) + +#o2_add_test(DCAFitterNCUDA +# SOURCES test/testDCAFitterNGPU.cxx +# PUBLIC_LINK_LIBRARIES O2::ReconstructionDataFormats +# O2::DCAFitterCUDA +# O2::DCAFitter +# ROOT::Core +# ROOT::Physics +# COMPONENT_NAME gpu +# LABELS vertexing +# ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage +# VMCWORKDIR=${CMAKE_BINARY_DIR}/stage/${CMAKE_INSTALL_DATADIR}) diff --git a/Common/DCAFitter/GPU/cuda/DCAFitterN.cu b/Common/DCAFitter/GPU/cuda/DCAFitterN.cu new file mode 100644 index 0000000000000..ed7e8acba5397 --- /dev/null +++ b/Common/DCAFitter/GPU/cuda/DCAFitterN.cu @@ -0,0 +1,280 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifdef __HIPCC__ +#include "hip/hip_runtime.h" +#else +#include +#endif + +#include + +#include "GPUCommonDef.h" +#include "DCAFitter/DCAFitterN.h" +#include "DeviceInterface/GPUInterface.h" + +#define gpuCheckError(x) \ + { \ + gpuAssert((x), __FILE__, __LINE__); \ + } +inline void gpuAssert(cudaError_t code, const char* file, int line, bool abort = true) +{ + if (code != cudaSuccess) { + std::cout << "GPUassert: " << cudaGetErrorString(code) << " " << file << " " << line << std::endl; + if (abort) { + throw std::runtime_error("GPU assert failed."); + } + } +} +namespace o2::vertexing::device +{ +namespace kernel +{ +GPUg() void warmUpGpuKernel() +{ + unsigned int tid = blockIdx.x * blockDim.x + threadIdx.x; + float ia, ib; + ia = ib = 0.0f; + ib += ia + tid; +} + +template +GPUg() void printKernel(Fitter* fitter) +{ + if (threadIdx.x == 0) { + printf(" =============== GPU DCA Fitter %d prongs =================\n", Fitter::getNProngs()); + fitter->print(); + printf(" =========================================================\n"); + } +} + +template +GPUg() void initFitters(Fitter* fitters, unsigned int off, unsigned int N) +{ + for (auto iThread{blockIdx.x * blockDim.x + threadIdx.x + 1}; iThread < N; iThread += blockDim.x * gridDim.x) { + fitters[iThread + off] = fitters[off]; + } +} + +template +GPUg() void processKernel(Fitter* fitter, int* res, Tr*... tracks) +{ + *res = fitter->process(*tracks...); +} + +template +GPUg() void processBatchKernel(Fitter* fitters, int* results, unsigned int off, unsigned int N, Tr*... tracks) +{ + for (auto iThread{blockIdx.x * blockDim.x + threadIdx.x}; iThread < N; iThread += blockDim.x * gridDim.x) { + results[iThread + off] = fitters[iThread + off].process(tracks[iThread + off]...); + } +} + +} // namespace kernel + +/// CPU handlers +template +void print(const int nBlocks, + const int nThreads, + Fitter& fitter) +{ + Fitter* fitter_device; + gpuCheckError(cudaMalloc(reinterpret_cast(&fitter_device), sizeof(Fitter))); + gpuCheckError(cudaMemcpy(fitter_device, &fitter, sizeof(Fitter), cudaMemcpyHostToDevice)); + + kernel::printKernel<<>>(fitter_device); + + gpuCheckError(cudaPeekAtLastError()); + gpuCheckError(cudaDeviceSynchronize()); +} + +template +int process(const int nBlocks, + const int nThreads, + Fitter& fitter, + Tr&... args) +{ + Fitter* fitter_device; + std::array tracks_device; + int result, *result_device; + + gpuCheckError(cudaMalloc(reinterpret_cast(&fitter_device), sizeof(Fitter))); + gpuCheckError(cudaMalloc(reinterpret_cast(&result_device), sizeof(int))); + + int iArg{0}; + ([&] { + gpuCheckError(cudaMalloc(reinterpret_cast(&(tracks_device[iArg])), sizeof(o2::track::TrackParCov))); + gpuCheckError(cudaMemcpy(tracks_device[iArg], &args, sizeof(o2::track::TrackParCov), cudaMemcpyHostToDevice)); + ++iArg; + }(), + ...); + + gpuCheckError(cudaMemcpy(fitter_device, &fitter, sizeof(Fitter), cudaMemcpyHostToDevice)); + + std::apply([&](auto&&... args) { kernel::processKernel<<>>(fitter_device, result_device, args...); }, tracks_device); + + gpuCheckError(cudaPeekAtLastError()); + gpuCheckError(cudaDeviceSynchronize()); + + gpuCheckError(cudaMemcpy(&result, result_device, sizeof(int), cudaMemcpyDeviceToHost)); + gpuCheckError(cudaMemcpy(&fitter, fitter_device, sizeof(Fitter), cudaMemcpyDeviceToHost)); + iArg = 0; + ([&] { + gpuCheckError(cudaMemcpy(&args, tracks_device[iArg], sizeof(o2::track::TrackParCov), cudaMemcpyDeviceToHost)); + gpuCheckError(cudaFree(tracks_device[iArg])); + ++iArg; + }(), + ...); + + gpuCheckError(cudaFree(fitter_device)); + gpuCheckError(cudaFree(result_device)); + + return result; +} + +template +void processBulk(const int nBlocks, + const int nThreads, + const int nBatches, + std::vector& fitters, + std::vector& results, + std::vector&... args) +{ + auto* gpuInterface = GPUInterface::Instance(); + kernel::warmUpGpuKernel<<<1, 1, 0, gpuInterface->getNextStream()>>>(); + + // Benchmarking events + std::vector ioUp(nBatches), ioDown(nBatches), kerElapsed(nBatches); + std::vector startIOUp(nBatches), endIOUp(nBatches), startIODown(nBatches), endIODown(nBatches), startKer(nBatches), endKer(nBatches); + for (int iBatch{0}; iBatch < nBatches; ++iBatch) { + gpuCheckError(cudaEventCreate(&startIOUp[iBatch])); + gpuCheckError(cudaEventCreate(&endIOUp[iBatch])); + gpuCheckError(cudaEventCreate(&startIODown[iBatch])); + gpuCheckError(cudaEventCreate(&endIODown[iBatch])); + gpuCheckError(cudaEventCreate(&startKer[iBatch])); + gpuCheckError(cudaEventCreate(&endKer[iBatch])); + } + + // Tracks + std::array tracks_device; + int iArg{0}; + ([&] { + gpuInterface->registerBuffer(reinterpret_cast(args.data()), sizeof(Tr) * args.size()); + gpuInterface->allocDevice(reinterpret_cast(&(tracks_device[iArg])), sizeof(Tr) * args.size()); + ++iArg; + }(), + ...); + + // Fitters + gpuInterface->registerBuffer(reinterpret_cast(fitters.data()), sizeof(Fitter) * fitters.size()); + Fitter* fitters_device; + gpuInterface->allocDevice(reinterpret_cast(&fitters_device), sizeof(Fitter) * fitters.size()); + + // Results + gpuInterface->registerBuffer(reinterpret_cast(results.data()), sizeof(int) * fitters.size()); + int* results_device; + gpuInterface->allocDevice(reinterpret_cast(&results_device), sizeof(int) * fitters.size()); + + // R.R. Computation + int totalSize = fitters.size(); + int batchSize = totalSize / nBatches; + int remainder = totalSize % nBatches; + + for (int iBatch{0}; iBatch < nBatches; ++iBatch) { + auto& stream = gpuInterface->getNextStream(); + auto offset = iBatch * batchSize + std::min(iBatch, remainder); + auto nFits = batchSize + (iBatch < remainder ? 1 : 0); + + gpuCheckError(cudaEventRecord(startIOUp[iBatch], stream)); + gpuCheckError(cudaMemcpyAsync(fitters_device + offset, fitters.data() + offset, sizeof(Fitter) /* * nFits */, cudaMemcpyHostToDevice, stream)); // copying just the first element of the buffer + iArg = 0; + ([&] { + gpuCheckError(cudaMemcpyAsync(tracks_device[iArg] + offset, args.data() + offset, sizeof(Tr) * nFits, cudaMemcpyHostToDevice, stream)); + ++iArg; + }(), + ...); + gpuCheckError(cudaEventRecord(endIOUp[iBatch], stream)); + + gpuCheckError(cudaEventRecord(startKer[iBatch], stream)); + kernel::initFitters<<>>(fitters_device, offset, nFits); + std::apply([&](auto&&... args) { kernel::processBatchKernel<<>>(fitters_device, results_device, offset, nFits, args...); }, tracks_device); + gpuCheckError(cudaEventRecord(endKer[iBatch], stream)); + + gpuCheckError(cudaPeekAtLastError()); + iArg = 0; + gpuCheckError(cudaEventRecord(startIODown[iBatch], stream)); + ([&] { + gpuCheckError(cudaMemcpyAsync(args.data() + offset, tracks_device[iArg] + offset, sizeof(Tr) * nFits, cudaMemcpyDeviceToHost, stream)); + ++iArg; + }(), + ...); + + gpuCheckError(cudaMemcpyAsync(fitters.data() + offset, fitters_device + offset, sizeof(Fitter) * nFits, cudaMemcpyDeviceToHost, stream)); + gpuCheckError(cudaMemcpyAsync(results.data() + offset, results_device + offset, sizeof(int) * nFits, cudaMemcpyDeviceToHost, stream)); + gpuCheckError(cudaEventRecord(endIODown[iBatch], stream)); + } + + ([&] { gpuInterface->unregisterBuffer(args.data()); }(), ...); + + for (auto* tracksD : tracks_device) { + gpuInterface->freeDevice(tracksD); + } + + gpuInterface->freeDevice(fitters_device); + gpuInterface->freeDevice(results_device); + gpuInterface->unregisterBuffer(fitters.data()); + gpuInterface->unregisterBuffer(results.data()); + + // Do benchmarks + gpuCheckError(cudaDeviceSynchronize()); + for (int iBatch{0}; iBatch < nBatches; ++iBatch) { + gpuCheckError(cudaEventElapsedTime(&ioUp[iBatch], startIOUp[iBatch], endIOUp[iBatch])); + gpuCheckError(cudaEventElapsedTime(&kerElapsed[iBatch], startKer[iBatch], endKer[iBatch])); + gpuCheckError(cudaEventElapsedTime(&ioDown[iBatch], startIODown[iBatch], endIODown[iBatch])); + } + + float totalUp = std::accumulate(ioUp.begin(), ioUp.end(), 0.f); + float totalDown = std::accumulate(ioDown.begin(), ioDown.end(), 0.f); + float totalKernels = std::accumulate(kerElapsed.begin(), kerElapsed.end(), 0.f); + LOGP(info, "Config: {} batches, {} blocks, {} threads", nBatches, nBlocks, nThreads); + LOGP(info, "Total I/O time: Up {} ms Avg {} ms, Down {} ms Avg {} ms", totalUp, totalUp / float(nBatches), totalDown, totalDown / (float)nBatches); + LOGP(info, "Total Kernel time: {} ms Avg {} ms", totalKernels, totalKernels / (float)nBatches); + + for (int iBatch{0}; iBatch < nBatches; ++iBatch) { + gpuCheckError(cudaEventDestroy(startIOUp[iBatch])); + gpuCheckError(cudaEventDestroy(endIOUp[iBatch])); + gpuCheckError(cudaEventDestroy(startIODown[iBatch])); + gpuCheckError(cudaEventDestroy(endIODown[iBatch])); + gpuCheckError(cudaEventDestroy(startKer[iBatch])); + gpuCheckError(cudaEventDestroy(endKer[iBatch])); + } +} + +template void processBulk(const int, + const int, + const int, + std::vector>&, + std::vector&, + std::vector&, + std::vector&); +template void processBulk(const int, + const int, + const int, + std::vector>&, + std::vector&, + std::vector&, + std::vector&, + std::vector&); +template int process(const int, const int, o2::vertexing::DCAFitterN<2>&, o2::track::TrackParCov&, o2::track::TrackParCov&); +template int process(const int, const int, o2::vertexing::DCAFitterN<3>&, o2::track::TrackParCov&, o2::track::TrackParCov&, o2::track::TrackParCov&); +template void print(const int, const int, o2::vertexing::DCAFitterN<2>&); +template void print(const int, const int, o2::vertexing::DCAFitterN<3>&); +} // namespace o2::vertexing::device \ No newline at end of file diff --git a/Common/DCAFitter/GPU/cuda/GPUInterface.cu b/Common/DCAFitter/GPU/cuda/GPUInterface.cu new file mode 100644 index 0000000000000..09f9cdc595dcd --- /dev/null +++ b/Common/DCAFitter/GPU/cuda/GPUInterface.cu @@ -0,0 +1,108 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \brief Helper interface to the GPU device, meant to be compatible with manual allocation/streams and GPUReconstruction ones. +/// \author matteo.concas@cern.ch + +#ifdef __HIPCC__ +#include "hip/hip_runtime.h" +#else +#include +#endif + +#include +#include + +#include "DeviceInterface/GPUInterface.h" + +#define gpuCheckError(x) \ + { \ + gpuAssert((x), __FILE__, __LINE__); \ + } +#define gpuCheckErrorSoft(x) \ + { \ + gpuAssert((x), __FILE__, __LINE__, false); \ + } +inline void gpuAssert(cudaError_t code, const char* file, int line, bool abort = true) +{ + if (code != cudaSuccess) { + std::cerr << "GPUassert: " << cudaGetErrorString(code) << " " << file << " " << line << std::endl; + if (abort) { + throw std::runtime_error("GPU assert failed."); + } + } +} + +namespace o2::vertexing::device +{ + +GPUInterface::GPUInterface(size_t N) +{ + resize(N); + for (auto& st : mStreams) { + gpuCheckError(cudaStreamCreate(&st)); + } +} + +GPUInterface::~GPUInterface() +{ + for (auto& st : mStreams) { + gpuCheckError(cudaStreamDestroy(st)); + } +} + +void GPUInterface::resize(size_t N) +{ + mPool.resize(N); + mStreams.resize(N); +} + +void GPUInterface::registerBuffer(void* addr, size_t bufferSize) +{ + gpuCheckError(cudaHostRegister(addr, bufferSize, cudaHostRegisterDefault)); +} + +void GPUInterface::unregisterBuffer(void* addr) +{ + gpuCheckError(cudaHostUnregister(addr)); +} + +GPUInterface* GPUInterface::sGPUInterface = nullptr; +GPUInterface* GPUInterface::Instance() +{ + if (sGPUInterface == nullptr) { + const auto* envValue = std::getenv("GPUINTERFACE_NSTREAMS"); + sGPUInterface = new GPUInterface(envValue == nullptr ? 8 : std::stoi(envValue)); + } + return sGPUInterface; +} + +void GPUInterface::allocDevice(void** addrPtr, size_t bufferSize) +{ + gpuCheckError(cudaMalloc(addrPtr, bufferSize)); +} + +void GPUInterface::freeDevice(void* addr) +{ + gpuCheckError(cudaFree(addr)); +} + +Stream& GPUInterface::getStream(unsigned short N) +{ + return mStreams[N % mStreams.size()]; +} + +Stream& GPUInterface::getNextStream() +{ + unsigned short next = mLastUsedStream.fetch_add(1) % mStreams.size(); // wrap-around + automatic wrap-around beyond 65535 + return mStreams[next]; +} +} // namespace o2::vertexing::device \ No newline at end of file diff --git a/Common/DCAFitter/GPU/cuda/test/testDCAFitterNGPU.cxx b/Common/DCAFitter/GPU/cuda/test/testDCAFitterNGPU.cxx new file mode 100644 index 0000000000000..e3f882dc755cb --- /dev/null +++ b/Common/DCAFitter/GPU/cuda/test/testDCAFitterNGPU.cxx @@ -0,0 +1,1086 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#define BOOST_TEST_MODULE Test DCAFitterN class +#define BOOST_TEST_MAIN +#define BOOST_TEST_DYN_LINK +#include + +#include "DCAFitter/DCAFitterN.h" +#include "CommonUtils/TreeStreamRedirector.h" +#include +#include +#include +#include +#include +#include + +namespace o2 +{ +namespace vertexing +{ + +using Vec3D = ROOT::Math::SVector; + +template +float checkResults(o2::utils::TreeStreamRedirector& outs, std::string& treeName, FITTER& fitter, + Vec3D& vgen, TLorentzVector& genPar, const std::vector& dtMass) +{ + int nCand = fitter.getNCandidates(); + std::array p; + float distMin = 1e9; + bool absDCA = fitter.getUseAbsDCA(); + bool useWghDCA = fitter.getWeightedFinalPCA(); + for (int ic = 0; ic < nCand; ic++) { + const auto& vtx = fitter.getPCACandidate(ic); + auto df = vgen; + df -= vtx; + + TLorentzVector moth, prong; + for (int i = 0; i < fitter.getNProngs(); i++) { + const auto& trc = fitter.getTrack(i, ic); + trc.getPxPyPzGlo(p); + prong.SetVectM({p[0], p[1], p[2]}, dtMass[i]); + moth += prong; + } + auto nIter = fitter.getNIterations(ic); + auto chi2 = fitter.getChi2AtPCACandidate(ic); + double dst = TMath::Sqrt(df[0] * df[0] + df[1] * df[1] + df[2] * df[2]); + distMin = dst < distMin ? dst : distMin; + auto parentTrack = fitter.createParentTrackParCov(ic); + outs << treeName.c_str() << "cand=" << ic << "ncand=" << nCand << "nIter=" << nIter << "chi2=" << chi2 + << "genPart=" << genPar << "recPart=" << moth + << "genX=" << vgen[0] << "genY=" << vgen[1] << "genZ=" << vgen[2] + << "dx=" << df[0] << "dy=" << df[1] << "dz=" << df[2] << "dst=" << dst + << "useAbsDCA=" << absDCA << "useWghDCA=" << useWghDCA << "parent=" << parentTrack; + for (int i = 0; i < fitter.getNProngs(); i++) { + outs << treeName.c_str() << fmt::format("prong{}=", i).c_str() << fitter.getTrack(i, ic); + } + outs << treeName.c_str() << "\n"; + } + return distMin; +} + +TLorentzVector generate(Vec3D& vtx, std::vector& vctr, float bz, + TGenPhaseSpace& genPHS, double parMass, const std::vector& dtMass, std::vector forceQ) +{ + const float errYZ = 1e-2, errSlp = 1e-3, errQPT = 2e-2; + std::array covm = { + errYZ * errYZ, + 0., errYZ * errYZ, + 0, 0., errSlp * errSlp, + 0., 0., 0., errSlp * errSlp, + 0., 0., 0., 0., errQPT * errQPT}; + bool accept = true; + TLorentzVector parent, d0, d1, d2; + do { + accept = true; + double y = gRandom->Rndm() - 0.5; + double pt = 0.1 + gRandom->Rndm() * 3; + double mt = TMath::Sqrt(parMass * parMass + pt * pt); + double pz = mt * TMath::SinH(y); + double phi = gRandom->Rndm() * TMath::Pi() * 2; + double en = mt * TMath::CosH(y); + double rdec = 10.; // radius of the decay + vtx[0] = rdec * TMath::Cos(phi); + vtx[1] = rdec * TMath::Sin(phi); + vtx[2] = rdec * pz / pt; + parent.SetPxPyPzE(pt * TMath::Cos(phi), pt * TMath::Sin(phi), pz, en); + int nd = dtMass.size(); + genPHS.SetDecay(parent, nd, dtMass.data()); + genPHS.Generate(); + vctr.clear(); + float p[4]; + for (int i = 0; i < nd; i++) { + auto* dt = genPHS.GetDecay(i); + if (dt->Pt() < 0.05) { + accept = false; + break; + } + dt->GetXYZT(p); + float s, c, x; + std::array params; + o2::math_utils::sincos(dt->Phi(), s, c); + o2::math_utils::rotateZInv(vtx[0], vtx[1], x, params[0], s, c); + + params[1] = vtx[2]; + params[2] = 0.; // since alpha = phi + params[3] = 1. / TMath::Tan(dt->Theta()); + params[4] = (i % 2 ? -1. : 1.) / dt->Pt(); + covm[14] = errQPT * errQPT * params[4] * params[4]; + // + // randomize + float r1, r2; + gRandom->Rannor(r1, r2); + params[0] += r1 * errYZ; + params[1] += r2 * errYZ; + gRandom->Rannor(r1, r2); + params[2] += r1 * errSlp; + params[3] += r2 * errSlp; + params[4] *= gRandom->Gaus(1., errQPT); + if (forceQ[i] == 0) { + params[4] = 0.; // impose straight track + } + auto& trc = vctr.emplace_back(x, dt->Phi(), params, covm); + float rad = forceQ[i] == 0 ? 600. : TMath::Abs(1. / trc.getCurvature(bz)); + if (!trc.propagateTo(trc.getX() + (gRandom->Rndm() - 0.5) * rad * 0.05, bz) || + !trc.rotate(trc.getAlpha() + (gRandom->Rndm() - 0.5) * 0.2)) { + printf("Failed to randomize "); + trc.print(); + } + } + } while (!accept); + + return parent; +} + +#ifdef DO_SINGLE_THREAD_TEST +BOOST_AUTO_TEST_CASE(DCAFitterNProngs) +{ + // gRandom->Delete(); + // gRandom = new TRandom(42); + o2::utils::TreeStreamRedirector outStream("dcafitterNTest.root"); + + TGenPhaseSpace genPHS; + constexpr double ele = 0.00051; + constexpr double gamma = 2 * ele + 1e-6; + constexpr double pion = 0.13957; + constexpr double k0 = 0.49761; + constexpr double kch = 0.49368; + constexpr double dch = 1.86965; + std::vector gammadec = {ele, ele}; + std::vector k0dec = {pion, pion}; + std::vector dchdec = {pion, kch, pion}; + std::vector vctracks; + Vec3D vtxGen; + + double bz = 5.0; + // 2 prongs vertices + { + LOG(info) << "Processing 2-prong Helix - Helix case"; + std::vector forceQ{1, 1}; + + o2::vertexing::DCAFitterN<2> ft; // 2 prong fitter + ft.setBz(bz); + ft.setPropagateToPCA(true); // After finding the vertex, propagate tracks to the DCA. This is default anyway + ft.setMaxR(200); // do not consider V0 seeds with 2D circles crossing above this R. This is default anyway + ft.setMaxDZIni(4); // do not consider V0 seeds with tracks Z-distance exceeding this. This is default anyway + ft.setMaxDXYIni(4); // do not consider V0 seeds with tracks XY-distance exceeding this. This is default anyway + ft.setMinParamChange(1e-3); // stop iterations if max correction is below this value. This is default anyway + ft.setMinRelChi2Change(0.9); // stop iterations if chi2 improves by less that this factor + + std::string treeName2A = "pr2a", treeName2AW = "pr2aw", treeName2W = "pr2w"; + TStopwatch swA, swAW, swW; + int nfoundA = 0, nfoundAW = 0, nfoundW = 0; + double meanDA = 0, meanDAW = 0, meanDW = 0; + swA.Stop(); + swAW.Stop(); + swW.Stop(); + for (int iev = 0; iev < NTest; iev++) { + auto genParent = generate(vtxGen, vctracks, bz, genPHS, k0, k0dec, forceQ); + + ft.setUseAbsDCA(true); + swA.Start(false); + int ncA = device::process(1, 1, ft, vctracks[0], vctracks[1]); // HERE WE FIT THE VERTICES + swA.Stop(); + LOG(debug) << "fit abs.dist " << iev << " NC: " << ncA << " Chi2: " << (ncA ? ft.getChi2AtPCACandidate(0) : -1); + if (ncA) { + auto minD = checkResults(outStream, treeName2A, ft, vtxGen, genParent, k0dec); + meanDA += minD; + nfoundA++; + } + + ft.setUseAbsDCA(true); + ft.setWeightedFinalPCA(true); + swAW.Start(false); + int ncAW = device::process(1, 1, ft, vctracks[0], vctracks[1]); // HERE WE FIT THE VERTICES + swAW.Stop(); + LOG(debug) << "fit abs.dist with final weighted DCA " << iev << " NC: " << ncAW << " Chi2: " << (ncAW ? ft.getChi2AtPCACandidate(0) : -1); + if (ncAW) { + auto minD = checkResults(outStream, treeName2AW, ft, vtxGen, genParent, k0dec); + meanDAW += minD; + nfoundAW++; + } + + ft.setUseAbsDCA(false); + ft.setWeightedFinalPCA(false); + swW.Start(false); + int ncW = device::process(1, 1, ft, vctracks[0], vctracks[1]); // HERE WE FIT THE VERTICES + swW.Stop(); + LOG(debug) << "fit wgh.dist " << iev << " NC: " << ncW << " Chi2: " << (ncW ? ft.getChi2AtPCACandidate(0) : -1); + if (ncW) { + auto minD = checkResults(outStream, treeName2W, ft, vtxGen, genParent, k0dec); + meanDW += minD; + nfoundW++; + } + } + + meanDA /= nfoundA ? nfoundA : 1; + meanDAW /= nfoundAW ? nfoundA : 1; + meanDW /= nfoundW ? nfoundW : 1; + LOG(info) << "Processed " << NTest << " 2-prong vertices Helix : Helix"; + LOG(info) << "2-prongs with abs.dist minimization: eff = " << float(nfoundA) / NTest + << " mean.dist to truth: " << meanDA << " Total time: " << swA.CpuTime() * 1000 << " ms"; + LOG(info) << "2-prongs with abs.dist but wghPCA: eff = " << float(nfoundAW) / NTest + << " mean.dist to truth: " << meanDAW << " Total time: " << swAW.CpuTime() * 1000 << " ms"; + LOG(info) << "2-prongs with wgh.dist minimization: eff = " << float(nfoundW) / NTest + << " mean.dist to truth: " << meanDW << " Total time: " << swW.CpuTime() * 1000 << " ms"; + BOOST_CHECK(nfoundA > 0.99 * NTest); + BOOST_CHECK(nfoundAW > 0.99 * NTest); + BOOST_CHECK(nfoundW > 0.99 * NTest); + BOOST_CHECK(meanDA < 0.1); + BOOST_CHECK(meanDAW < 0.1); + BOOST_CHECK(meanDW < 0.1); + } + + // 2 prongs vertices with collinear tracks (gamma conversion) + { + LOG(info) << "Processing 2-prong Helix - Helix case gamma conversion"; + std::vector forceQ{1, 1}; + + o2::vertexing::DCAFitterN<2> ft; // 2 prong fitter + ft.setBz(bz); + ft.setPropagateToPCA(true); // After finding the vertex, propagate tracks to the DCA. This is default anyway + ft.setMaxR(200); // do not consider V0 seeds with 2D circles crossing above this R. This is default anyway + ft.setMaxDZIni(4); // do not consider V0 seeds with tracks Z-distance exceeding this. This is default anyway + ft.setMaxDXYIni(4); // do not consider V0 seeds with tracks XY-distance exceeding this. This is default anyway + ft.setMinParamChange(1e-3); // stop iterations if max correction is below this value. This is default anyway + ft.setMinRelChi2Change(0.9); // stop iterations if chi2 improves by less that this factor + + std::string treeName2A = "gpr2a", treeName2AW = "gpr2aw", treeName2W = "gpr2w"; + TStopwatch swA, swAW, swW; + int nfoundA = 0, nfoundAW = 0, nfoundW = 0; + double meanDA = 0, meanDAW = 0, meanDW = 0; + swA.Stop(); + swAW.Stop(); + swW.Stop(); + for (int iev = 0; iev < NTest; iev++) { + auto genParent = generate(vtxGen, vctracks, bz, genPHS, gamma, gammadec, forceQ); + + ft.setUseAbsDCA(true); + swA.Start(false); + int ncA = device::process(1, 1, ft, vctracks[0], vctracks[1]); + swA.Stop(); + LOG(debug) << "fit abs.dist " << iev << " NC: " << ncA << " Chi2: " << (ncA ? ft.getChi2AtPCACandidate(0) : -1); + if (ncA) { + auto minD = checkResults(outStream, treeName2A, ft, vtxGen, genParent, gammadec); + meanDA += minD; + nfoundA++; + } + + ft.setUseAbsDCA(true); + ft.setWeightedFinalPCA(true); + swAW.Start(false); + int ncAW = device::process(1, 1, ft, vctracks[0], vctracks[1]); // HERE WE FIT THE VERTICES + swAW.Stop(); + LOG(debug) << "fit abs.dist with final weighted DCA " << iev << " NC: " << ncAW << " Chi2: " << (ncAW ? ft.getChi2AtPCACandidate(0) : -1); + if (ncAW) { + auto minD = checkResults(outStream, treeName2AW, ft, vtxGen, genParent, gammadec); + meanDAW += minD; + nfoundAW++; + } + + ft.setUseAbsDCA(false); + ft.setWeightedFinalPCA(false); + swW.Start(false); + int ncW = device::process(1, 1, ft, vctracks[0], vctracks[1]); // HERE WE FIT THE VERTICES + swW.Stop(); + LOG(debug) << "fit wgh.dist " << iev << " NC: " << ncW << " Chi2: " << (ncW ? ft.getChi2AtPCACandidate(0) : -1); + if (ncW) { + auto minD = checkResults(outStream, treeName2W, ft, vtxGen, genParent, gammadec); + meanDW += minD; + nfoundW++; + } + } + + meanDA /= nfoundA ? nfoundA : 1; + meanDAW /= nfoundA ? nfoundA : 1; + meanDW /= nfoundW ? nfoundW : 1; + LOG(info) << "Processed " << NTest << " 2-prong vertices Helix : Helix from gamma conversion"; + LOG(info) << "2-prongs with abs.dist minimization: eff = " << float(nfoundA) / NTest + << " mean.dist to truth: " << meanDA << " Total time: " << swA.CpuTime(); + LOG(info) << "2-prongs with abs.dist but wghPCA: eff = " << float(nfoundAW) / NTest + << " mean.dist to truth: " << meanDAW << " Total time: " << swAW.CpuTime(); + LOG(info) << "2-prongs with wgh.dist minimization: eff = " << float(nfoundW) / NTest + << " mean.dist to truth: " << meanDW << " Total time: " << swW.CpuTime(); + BOOST_CHECK(nfoundA > 0.99 * NTest); + BOOST_CHECK(nfoundAW > 0.99 * NTest); + BOOST_CHECK(nfoundW > 0.99 * NTest); + BOOST_CHECK(meanDA < 2.1); + BOOST_CHECK(meanDAW < 2.1); + BOOST_CHECK(meanDW < 2.1); + } + + // 2 prongs vertices with one of charges set to 0: Helix : Line + { + std::vector forceQ{1, 1}; + LOG(info) << "Processing 2-prong Helix - Line case"; + o2::vertexing::DCAFitterN<2> ft; // 2 prong fitter + ft.setBz(bz); + ft.setPropagateToPCA(true); // After finding the vertex, propagate tracks to the DCA. This is default anyway + ft.setMaxR(200); // do not consider V0 seeds with 2D circles crossing above this R. This is default anyway + ft.setMaxDZIni(4); // do not consider V0 seeds with tracks Z-distance exceeding this. This is default anyway + ft.setMinParamChange(1e-3); // stop iterations if max correction is below this value. This is default anyway + ft.setMinRelChi2Change(0.9); // stop iterations if chi2 improves by less that this factor + + std::string treeName2A = "pr2aHL", treeName2AW = "pr2awHL", treeName2W = "pr2wHL"; + TStopwatch swA, swAW, swW; + int nfoundA = 0, nfoundAW = 0, nfoundW = 0; + double meanDA = 0, meanDAW = 0, meanDW = 0; + swA.Stop(); + swAW.Stop(); + swW.Stop(); + for (int iev = 0; iev < NTest; iev++) { + forceQ[iev % 2] = 1; + forceQ[1 - iev % 2] = 0; + auto genParent = generate(vtxGen, vctracks, bz, genPHS, k0, k0dec, forceQ); + + ft.setUseAbsDCA(true); + swA.Start(false); + int ncA = device::process(1, 1, ft, vctracks[0], vctracks[1]); + swA.Stop(); + LOG(debug) << "fit abs.dist with final weighted DCA " << iev << " NC: " << ncA << " Chi2: " << (ncA ? ft.getChi2AtPCACandidate(0) : -1); + if (ncA) { + auto minD = checkResults(outStream, treeName2A, ft, vtxGen, genParent, k0dec); + meanDA += minD; + nfoundA++; + } + + ft.setUseAbsDCA(true); + ft.setWeightedFinalPCA(true); + swAW.Start(false); + int ncAW = device::process(1, 1, ft, vctracks[0], vctracks[1]); // HERE WE FIT THE VERTICES + swAW.Stop(); + LOG(debug) << "fit abs.dist " << iev << " NC: " << ncAW << " Chi2: " << (ncAW ? ft.getChi2AtPCACandidate(0) : -1); + if (ncAW) { + auto minD = checkResults(outStream, treeName2AW, ft, vtxGen, genParent, k0dec); + meanDAW += minD; + nfoundAW++; + } + + ft.setUseAbsDCA(false); + ft.setWeightedFinalPCA(false); + swW.Start(false); + int ncW = device::process(1, 1, ft, vctracks[0], vctracks[1]); // HERE WE FIT THE VERTICES + swW.Stop(); + LOG(debug) << "fit wgh.dist " << iev << " NC: " << ncW << " Chi2: " << (ncW ? ft.getChi2AtPCACandidate(0) : -1); + if (ncW) { + auto minD = checkResults(outStream, treeName2W, ft, vtxGen, genParent, k0dec); + meanDW += minD; + nfoundW++; + } + } + + meanDA /= nfoundA ? nfoundA : 1; + meanDAW /= nfoundAW ? nfoundAW : 1; + meanDW /= nfoundW ? nfoundW : 1; + LOG(info) << "Processed " << NTest << " 2-prong vertices: Helix : Line"; + LOG(info) << "2-prongs with abs.dist minimization: eff = " << float(nfoundA) / NTest + << " mean.dist to truth: " << meanDA << " Total time: " << swA.CpuTime(); + LOG(info) << "2-prongs with abs.dist but wghPCA: eff = " << float(nfoundAW) / NTest + << " mean.dist to truth: " << meanDAW << " Total time: " << swAW.CpuTime(); + LOG(info) << "2-prongs with wgh.dist minimization: eff = " << float(nfoundW) / NTest + << " mean.dist to truth: " << meanDW << " Total time: " << swW.CpuTime(); + BOOST_CHECK(nfoundA > 0.99 * NTest); + BOOST_CHECK(nfoundAW > 0.99 * NTest); + BOOST_CHECK(nfoundW > 0.99 * NTest); + BOOST_CHECK(meanDA < 0.1); + BOOST_CHECK(meanDAW < 0.1); + BOOST_CHECK(meanDW < 0.1); + } + + // 2 prongs vertices with both of charges set to 0: Line : Line + { + std::vector forceQ{0, 0}; + LOG(info) << "Processing 2-prong Line - Line case"; + o2::vertexing::DCAFitterN<2> ft; // 2 prong fitter + ft.setBz(bz); + ft.setPropagateToPCA(true); // After finding the vertex, propagate tracks to the DCA. This is default anyway + ft.setMaxR(200); // do not consider V0 seeds with 2D circles crossing above this R. This is default anyway + ft.setMaxDZIni(4); // do not consider V0 seeds with tracks Z-distance exceeding this. This is default anyway + ft.setMinParamChange(1e-3); // stop iterations if max correction is below this value. This is default anyway + ft.setMinRelChi2Change(0.9); // stop iterations if chi2 improves by less that this factor + + std::string treeName2A = "pr2aLL", treeName2AW = "pr2awLL", treeName2W = "pr2wLL"; + TStopwatch swA, swAW, swW; + int nfoundA = 0, nfoundAW = 0, nfoundW = 0; + double meanDA = 0, meanDAW = 0, meanDW = 0; + swA.Stop(); + swAW.Stop(); + swW.Stop(); + for (int iev = 0; iev < NTest; iev++) { + forceQ[0] = forceQ[1] = 0; + auto genParent = generate(vtxGen, vctracks, bz, genPHS, k0, k0dec, forceQ); + + ft.setUseAbsDCA(true); + swA.Start(false); + int ncA = device::process(1, 1, ft, vctracks[0], vctracks[1]); // HERE WE FIT THE VERTICES + swA.Stop(); + LOG(debug) << "fit abs.dist " << iev << " NC: " << ncA << " Chi2: " << (ncA ? ft.getChi2AtPCACandidate(0) : -1); + if (ncA) { + auto minD = checkResults(outStream, treeName2A, ft, vtxGen, genParent, k0dec); + meanDA += minD; + nfoundA++; + } + + ft.setUseAbsDCA(true); + ft.setWeightedFinalPCA(true); + swAW.Start(false); + int ncAW = device::process(1, 1, ft, vctracks[0], vctracks[1]); // HERE WE FIT THE VERTICES + swAW.Stop(); + LOG(debug) << "fit abs.dist " << iev << " NC: " << ncAW << " Chi2: " << (ncAW ? ft.getChi2AtPCACandidate(0) : -1); + if (ncAW) { + auto minD = checkResults(outStream, treeName2AW, ft, vtxGen, genParent, k0dec); + meanDAW += minD; + nfoundAW++; + } + + ft.setUseAbsDCA(false); + ft.setWeightedFinalPCA(false); + swW.Start(false); + int ncW = device::process(1, 1, ft, vctracks[0], vctracks[1]); // HERE WE FIT THE VERTICES + swW.Stop(); + LOG(debug) << "fit wgh.dist " << iev << " NC: " << ncW << " Chi2: " << (ncW ? ft.getChi2AtPCACandidate(0) : -1); + if (ncW) { + auto minD = checkResults(outStream, treeName2W, ft, vtxGen, genParent, k0dec); + meanDW += minD; + nfoundW++; + } + } + + meanDA /= nfoundA ? nfoundA : 1; + meanDAW /= nfoundAW ? nfoundAW : 1; + meanDW /= nfoundW ? nfoundW : 1; + LOG(info) << "Processed " << NTest << " 2-prong vertices: Line : Line"; + LOG(info) << "2-prongs with abs.dist minimization: eff = " << float(nfoundA) / NTest + << " mean.dist to truth: " << meanDA << " Total time: " << swA.CpuTime(); + LOG(info) << "2-prongs with abs.dist but wghPCA: eff = " << float(nfoundAW) / NTest + << " mean.dist to truth: " << meanDAW << " Total time: " << swAW.CpuTime(); + LOG(info) << "2-prongs with wgh.dist minimization: eff = " << float(nfoundW) / NTest + << " mean.dist to truth: " << meanDW << " Total time: " << swW.CpuTime(); + BOOST_CHECK(nfoundA > 0.99 * NTest); + BOOST_CHECK(nfoundAW > 0.99 * NTest); + BOOST_CHECK(nfoundW > 0.99 * NTest); + BOOST_CHECK(meanDA < 0.1); + BOOST_CHECK(meanDAW < 0.1); + BOOST_CHECK(meanDW < 0.1); + } + + // 3 prongs vertices + { + std::vector forceQ{1, 1, 1}; + + o2::vertexing::DCAFitterN<3> ft; // 3 prong fitter + ft.setBz(bz); + ft.setPropagateToPCA(true); // After finding the vertex, propagate tracks to the DCA. This is default anyway + ft.setMaxR(200); // do not consider V0 seeds with 2D circles crossing above this R. This is default anyway + ft.setMaxDZIni(4); // do not consider V0 seeds with tracks Z-distance exceeding this. This is default anyway + ft.setMinParamChange(1e-3); // stop iterations if max correction is below this value. This is default anyway + ft.setMinRelChi2Change(0.9); // stop iterations if chi2 improves by less that this factor + + std::string treeName3A = "pr3a", treeName3AW = "pr3aw", treeName3W = "pr3w"; + TStopwatch swA, swAW, swW; + int nfoundA = 0, nfoundAW = 0, nfoundW = 0; + double meanDA = 0, meanDAW = 0, meanDW = 0; + swA.Stop(); + swAW.Stop(); + swW.Stop(); + for (int iev = 0; iev < NTest; iev++) { + auto genParent = generate(vtxGen, vctracks, bz, genPHS, dch, dchdec, forceQ); + + ft.setUseAbsDCA(true); + swA.Start(false); + int ncA = device::process(1, 1, ft, vctracks[0], vctracks[1], vctracks[2]); + swA.Stop(); + LOG(debug) << "fit abs.dist " << iev << " NC: " << ncA << " Chi2: " << (ncA ? ft.getChi2AtPCACandidate(0) : -1); + if (ncA) { + auto minD = checkResults(outStream, treeName3A, ft, vtxGen, genParent, dchdec); + meanDA += minD; + nfoundA++; + } + + ft.setUseAbsDCA(true); + ft.setWeightedFinalPCA(true); + swAW.Start(false); + int ncAW = device::process(1, 1, ft, vctracks[0], vctracks[1], vctracks[2]); // HERE WE FIT THE VERTICES + swAW.Stop(); + LOG(debug) << "fit abs.dist " << iev << " NC: " << ncAW << " Chi2: " << (ncAW ? ft.getChi2AtPCACandidate(0) : -1); + if (ncAW) { + auto minD = checkResults(outStream, treeName3AW, ft, vtxGen, genParent, dchdec); + meanDAW += minD; + nfoundAW++; + } + + ft.setUseAbsDCA(false); + ft.setWeightedFinalPCA(false); + swW.Start(false); + int ncW = device::process(1, 1, ft, vctracks[0], vctracks[1], vctracks[2]); // HERE WE FIT THE VERTICES + swW.Stop(); + LOG(debug) << "fit wgh.dist " << iev << " NC: " << ncW << " Chi2: " << (ncW ? ft.getChi2AtPCACandidate(0) : -1); + if (ncW) { + auto minD = checkResults(outStream, treeName3W, ft, vtxGen, genParent, dchdec); + meanDW += minD; + nfoundW++; + } + } + + meanDA /= nfoundA ? nfoundA : 1; + meanDAW /= nfoundAW ? nfoundAW : 1; + meanDW /= nfoundW ? nfoundW : 1; + LOG(info) << "Processed " << NTest << " 3-prong vertices"; + LOG(info) << "3-prongs with abs.dist minimization: eff = " << float(nfoundA) / NTest + << " mean.dist to truth: " << meanDA << " Total time: " << swA.CpuTime(); + LOG(info) << "3-prongs with abs.dist but wghPCA: eff = " << float(nfoundAW) / NTest + << " mean.dist to truth: " << meanDAW << " Total time: " << swAW.CpuTime(); + LOG(info) << "3-prongs with wgh.dist minimization: eff = " << float(nfoundW) / NTest + << " mean.dist to truth: " << meanDW << " Total time: " << swW.CpuTime(); + BOOST_CHECK(nfoundA > 0.99 * NTest); + BOOST_CHECK(nfoundAW > 0.99 * NTest); + BOOST_CHECK(nfoundW > 0.99 * NTest); + BOOST_CHECK(meanDA < 0.1); + BOOST_CHECK(meanDAW < 0.1); + BOOST_CHECK(meanDW < 0.1); + } + + outStream.Close(); +} +#endif + +BOOST_AUTO_TEST_CASE(DCAFitterNProngsBulk) +{ + const char* nThreadsEnvVarName = "DCAFITTERGPU_TEST_NTHREADS"; + const char* nBlocksEnvVarName = "DCAFITTERGPU_TEST_NBLOCKS"; + const char* nBatchesEnvVarName = "DCAFITTERGPU_TEST_NBATCHES"; + const char* nTestsEnvVarName = "DCAFITTERGPU_TEST_NTESTS"; + int nBlocks = std::getenv(nBlocksEnvVarName) == nullptr ? 30 : std::stoi(std::getenv(nBlocksEnvVarName)); + int nThreads = std::getenv(nThreadsEnvVarName) == nullptr ? 256 : std::stoi(std::getenv(nThreadsEnvVarName)); + int nBatches = std::getenv(nBatchesEnvVarName) == nullptr ? 8 : std::stoi(std::getenv(nBatchesEnvVarName)); + int NTest = std::getenv(nTestsEnvVarName) == nullptr ? 100001 : std::stoi(std::getenv(nTestsEnvVarName)); + + o2::utils::TreeStreamRedirector outStreamB("dcafitterNTestBulk.root"); + + TGenPhaseSpace genPHS; + constexpr double ele = 0.00051; + constexpr double gamma = 2 * ele + 1e-6; + constexpr double pion = 0.13957; + constexpr double k0 = 0.49761; + constexpr double kch = 0.49368; + constexpr double dch = 1.86965; + std::vector gammadec = {ele, ele}; + std::vector k0dec = {pion, pion}; + std::vector dchdec = {pion, kch, pion}; + std::vector> vctracks(3, std::vector(NTest)); + std::vector vtxGen(NTest); + + double bz = 5.0; + { // 2 prongs vertices bulk processing + LOG(info) << "\n\nBulk-processing 2-prong Helix - Helix case"; + std::vector forceQ{1, 1}; + + o2::vertexing::DCAFitterN<2> ft; // 2 prong fitter + ft.setBz(bz); + ft.setPropagateToPCA(true); // After finding the vertex, propagate tracks to the DCA. This is default anyway + ft.setMaxR(200); // do not consider V0 seeds with 2D circles crossing above this R. This is default anyway + ft.setMaxDZIni(4); // do not consider V0 seeds with tracks Z-distance exceeding this. This is default anyway + ft.setMaxDXYIni(4); // do not consider V0 seeds with tracks XY-distance exceeding this. This is default anyway + ft.setMinParamChange(1e-3); // stop iterations if max correction is below this value. This is default anyway + ft.setMinRelChi2Change(0.9); // stop iterations if chi2 improves by less that this factor + + std::vector> fitters_host(NTest); + std::vector genParents(NTest); + + std::string treeName2Abulk = "pr2aBulk", treeName2AWbulk = "pr2awBulk", treeName2Wbulk = "pr2wBulk"; + TStopwatch swAb, swAWb, swWb; + int nfoundAb = 0, nfoundAWb = 0, nfoundWb = 0; + double meanDAb = 0, meanDAWb = 0, meanDWb = 0; + swAb.Stop(); + swAWb.Stop(); + swWb.Stop(); + + ft.setUseAbsDCA(true); + std::fill(fitters_host.begin(), fitters_host.end(), ft); + for (int iev = 0; iev < NTest; iev++) { + std::vector vc(2); + genParents[iev] = generate(vtxGen[iev], vc, bz, genPHS, k0, k0dec, forceQ); + vctracks[0][iev] = vc[0]; + vctracks[1][iev] = vc[1]; + } + + swAb.Start(false); + std::vector ncAb(NTest, 0); + device::processBulk(nBlocks, nThreads, nBatches, fitters_host, ncAb, vctracks[0], vctracks[1]); // HERE WE FIT THE VERTICES + swAb.Stop(); + + for (int iev = 0; iev < NTest; iev++) { + LOG(debug) << "fit abs.dist " << iev << " NC: " << ncAb[iev] << " Chi2: " << (ncAb[iev] ? fitters_host[iev].getChi2AtPCACandidate(0) : -1); + if (ncAb[iev]) { + auto minDb = checkResults(outStreamB, treeName2Abulk, fitters_host[iev], vtxGen[iev], genParents[iev], k0dec); + meanDAb += minDb; + nfoundAb++; + } + } + + ft.setUseAbsDCA(true); + ft.setWeightedFinalPCA(true); + std::fill(fitters_host.begin(), fitters_host.end(), ft); + swAWb.Start(false); + std::vector ncAWb(NTest, 0); + device::processBulk(nBlocks, nThreads, nBatches, fitters_host, ncAWb, vctracks[0], vctracks[1]); // HERE WE FIT THE VERTICES + swAWb.Stop(); + + for (int iev = 0; iev < NTest; iev++) { + LOG(debug) << "fit abs.dist with final weighted DCA " << iev << " NC: " << ncAWb[iev] << " Chi2: " << (ncAWb[iev] ? fitters_host[iev].getChi2AtPCACandidate(0) : -1); + if (ncAWb[iev]) { + auto minDb = checkResults(outStreamB, treeName2AWbulk, fitters_host[iev], vtxGen[iev], genParents[iev], k0dec); + meanDAWb += minDb; + nfoundAWb++; + } + } + + ft.setUseAbsDCA(false); + ft.setWeightedFinalPCA(false); + std::fill(fitters_host.begin(), fitters_host.end(), ft); + swWb.Start(false); + std::vector ncWb(NTest, 0); + device::processBulk(nBlocks, nThreads, nBatches, fitters_host, ncWb, vctracks[0], vctracks[1]); // HERE WE FIT THE VERTICES + swWb.Stop(); + + for (int iev = 0; iev < NTest; iev++) { + LOG(debug) << "fit wgh.dist " << iev << " NC: " << ncWb[iev] << " Chi2: " << (ncWb[iev] ? fitters_host[iev].getChi2AtPCACandidate(0) : -1); + if (ncWb[iev]) { + auto minDb = checkResults(outStreamB, treeName2Wbulk, fitters_host[iev], vtxGen[iev], genParents[iev], k0dec); + meanDWb += minDb; + nfoundWb++; + } + } + // + meanDAb /= nfoundAb ? nfoundAb : 1; + meanDAWb /= nfoundAWb ? nfoundAWb : 1; + meanDWb /= nfoundWb ? nfoundWb : 1; + LOGP(info, "Bulk-processed {} 2-prong vertices Helix : Helix", NTest); + LOG(info) << "2-prongs with abs.dist minimization: eff = " << float(nfoundAb) / NTest + << " mean.dist to truth: " << meanDAb << " Total time: " << swAb.CpuTime() * 1000 << " ms"; + LOG(info) << "2-prongs with abs.dist but wghPCA: eff = " << float(nfoundAWb) / NTest + << " mean.dist to truth: " << meanDAWb << " Total time: " << swAWb.CpuTime() * 1000 << " ms"; + LOG(info) << "2-prongs with wgh.dist minimization: eff = " << float(nfoundWb) / NTest + << " mean.dist to truth: " << meanDWb << " Total time: " << swWb.CpuTime() * 1000 << " ms"; + BOOST_CHECK(nfoundAb > 0.99 * NTest); + BOOST_CHECK(nfoundAWb > 0.99 * NTest); + BOOST_CHECK(nfoundWb > 0.99 * NTest); + BOOST_CHECK(meanDAb < 0.1); + BOOST_CHECK(meanDAWb < 0.1); + BOOST_CHECK(meanDWb < 0.1); + } + + { // 2 prongs vertices bulk processing for gamma conversion + LOG(info) << "\n\nBulk-processing 2-prong Helix - Helix case gamma conversion"; + std::vector forceQ{1, 1}; + + o2::vertexing::DCAFitterN<2> ft; // 2 prong fitter + ft.setBz(bz); + ft.setPropagateToPCA(true); // After finding the vertex, propagate tracks to the DCA. This is default anyway + ft.setMaxR(200); // do not consider V0 seeds with 2D circles crossing above this R. This is default anyway + ft.setMaxDZIni(4); // do not consider V0 seeds with tracks Z-distance exceeding this. This is default anyway + ft.setMaxDXYIni(4); // do not consider V0 seeds with tracks XY-distance exceeding this. This is default anyway + ft.setMinParamChange(1e-3); // stop iterations if max correction is below this value. This is default anyway + ft.setMinRelChi2Change(0.9); // stop iterations if chi2 improves by less that this factor + + std::vector> fitters_host(NTest); + std::vector genParents(NTest); + + std::string treeName2Abulk = "gpr2aBulk", treeName2AWbulk = "gpr2awBulk", treeName2Wbulk = "gpr2wBulk"; + TStopwatch swAb, swAWb, swWb; + int nfoundAb = 0, nfoundAWb = 0, nfoundWb = 0; + double meanDAb = 0, meanDAWb = 0, meanDWb = 0; + swAb.Stop(); + swAWb.Stop(); + swWb.Stop(); + + ft.setUseAbsDCA(true); + std::fill(fitters_host.begin(), fitters_host.end(), ft); + for (int iev = 0; iev < NTest; iev++) { + std::vector vc(2); + genParents[iev] = generate(vtxGen[iev], vc, bz, genPHS, gamma, gammadec, forceQ); + vctracks[0][iev] = vc[0]; + vctracks[1][iev] = vc[1]; + } + + swAb.Start(false); + std::vector ncAb(NTest, 0); + device::processBulk(nBlocks, nThreads, nBatches, fitters_host, ncAb, vctracks[0], vctracks[1]); // HERE WE FIT THE VERTICES + swAb.Stop(); + + for (int iev = 0; iev < NTest; iev++) { + LOG(debug) << "fit abs.dist " << iev << " NC: " << ncAb[iev] << " Chi2: " << (ncAb[iev] ? fitters_host[iev].getChi2AtPCACandidate(0) : -1); + if (ncAb[iev]) { + auto minDb = checkResults(outStreamB, treeName2Abulk, fitters_host[iev], vtxGen[iev], genParents[iev], gammadec); + meanDAb += minDb; + nfoundAb++; + } + } + // + ft.setUseAbsDCA(true); + ft.setWeightedFinalPCA(true); + std::fill(fitters_host.begin(), fitters_host.end(), ft); + swAWb.Start(false); + std::vector ncAWb(NTest, 0); + device::processBulk(nBlocks, nThreads, nBatches, fitters_host, ncAWb, vctracks[0], vctracks[1]); // HERE WE FIT THE VERTICES + swAWb.Stop(); + + for (int iev = 0; iev < NTest; iev++) { + LOG(debug) << "fit abs.dist with final weighted DCA " << iev << " NC: " << ncAWb[iev] << " Chi2: " << (ncAWb[iev] ? fitters_host[iev].getChi2AtPCACandidate(0) : -1); + if (ncAWb[iev]) { + auto minDb = checkResults(outStreamB, treeName2AWbulk, fitters_host[iev], vtxGen[iev], genParents[iev], gammadec); + meanDAWb += minDb; + nfoundAWb++; + } + } + + ft.setUseAbsDCA(false); + ft.setWeightedFinalPCA(false); + std::fill(fitters_host.begin(), fitters_host.end(), ft); + swWb.Start(false); + std::vector ncWb(NTest, 0); + device::processBulk(nBlocks, nThreads, nBatches, fitters_host, ncWb, vctracks[0], vctracks[1]); // HERE WE FIT THE VERTICES + swWb.Stop(); + + for (int iev = 0; iev < NTest; iev++) { + LOG(debug) << "fit wgh.dist " << iev << " NC: " << ncWb[iev] << " Chi2: " << (ncWb[iev] ? fitters_host[iev].getChi2AtPCACandidate(0) : -1); + if (ncWb[iev]) { + auto minDb = checkResults(outStreamB, treeName2Wbulk, fitters_host[iev], vtxGen[iev], genParents[iev], gammadec); + meanDWb += minDb; + nfoundWb++; + } + } + // + + meanDAb /= nfoundAb ? nfoundAb : 1; + meanDAWb /= nfoundAWb ? nfoundAWb : 1; + meanDWb /= nfoundWb ? nfoundWb : 1; + LOGP(info, "Bulk-processed {} 2-prong vertices Helix : Helix from gamma conversion", NTest); + LOG(info) << "2-prongs with abs.dist minimization: eff = " << float(nfoundAb) / NTest + << " mean.dist to truth: " << meanDAb << " Total time: " << swAb.CpuTime() * 1000 << " ms"; + LOG(info) << "2-prongs with abs.dist but wghPCA: eff = " << float(nfoundAWb) / NTest + << " mean.dist to truth: " << meanDAWb << " Total time: " << swAWb.CpuTime() * 1000 << " ms"; + LOG(info) << "2-prongs with wgh.dist minimization: eff = " << float(nfoundWb) / NTest + << " mean.dist to truth: " << meanDWb << " Total time: " << swWb.CpuTime() * 1000 << " ms"; + BOOST_CHECK(nfoundAb > 0.99 * NTest); + BOOST_CHECK(nfoundAWb > 0.99 * NTest); + BOOST_CHECK(nfoundWb > 0.99 * NTest); + BOOST_CHECK(meanDAb < 2.1); + BOOST_CHECK(meanDAWb < 2.1); + BOOST_CHECK(meanDWb < 2.1); + } + + // 2 prongs vertices bulk processing with one of charges set to 0: Helix : Line + { + std::vector forceQ{1, 1}; + LOG(info) << "\n\nBulk-processing 2-prong Helix - Line case"; + o2::vertexing::DCAFitterN<2> ft; // 2 prong fitter + ft.setBz(bz); + ft.setPropagateToPCA(true); // After finding the vertex, propagate tracks to the DCA. This is default anyway + ft.setMaxR(200); // do not consider V0 seeds with 2D circles crossing above this R. This is default anyway + ft.setMaxDZIni(4); // do not consider V0 seeds with tracks Z-distance exceeding this. This is default anyway + ft.setMinParamChange(1e-3); // stop iterations if max correction is below this value. This is default anyway + ft.setMinRelChi2Change(0.9); // stop iterations if chi2 improves by less that this factor + + std::vector> fitters_host(NTest); + std::vector genParents(NTest); + + std::string treeName2Abulk = "pr2aHLb", treeName2AWbulk = "pr2awHLb", treeName2Wbulk = "pr2wHLb"; + TStopwatch swAb, swAWb, swWb; + int nfoundAb = 0, nfoundAWb = 0, nfoundWb = 0; + double meanDAb = 0, meanDAWb = 0, meanDWb = 0; + swAb.Stop(); + swAWb.Stop(); + swWb.Stop(); + + for (int iev = 0; iev < NTest; iev++) { + forceQ[iev % 2] = 1; + forceQ[1 - iev % 2] = 0; + std::vector vc(2); + genParents[iev] = generate(vtxGen[iev], vc, bz, genPHS, k0, k0dec, forceQ); + vctracks[0][iev] = vc[0]; + vctracks[1][iev] = vc[1]; + } + ft.setUseAbsDCA(true); + std::fill(fitters_host.begin(), fitters_host.end(), ft); + + swAb.Start(false); + std::vector ncAb(NTest, 0); + device::processBulk(nBlocks, nThreads, nBatches, fitters_host, ncAb, vctracks[0], vctracks[1]); // HERE WE FIT THE VERTICES + swAb.Stop(); + + for (int iev = 0; iev < NTest; iev++) { + LOG(debug) << "fit abs.dist with final weighted DCA " << iev << " NC: " << ncAb[iev] << " Chi2: " << (ncAb[iev] ? fitters_host[iev].getChi2AtPCACandidate(0) : -1); + if (ncAb[iev]) { + auto minDb = checkResults(outStreamB, treeName2Abulk, fitters_host[iev], vtxGen[iev], genParents[iev], k0dec); + meanDAb += minDb; + nfoundAb++; + } + } + + ft.setUseAbsDCA(true); + ft.setWeightedFinalPCA(true); + std::fill(fitters_host.begin(), fitters_host.end(), ft); + swAWb.Start(false); + std::vector ncAWb(NTest, 0); + device::processBulk(nBlocks, nThreads, nBatches, fitters_host, ncAWb, vctracks[0], vctracks[1]); // HERE WE FIT THE VERTICES + swAWb.Stop(); + + for (int iev = 0; iev < NTest; iev++) { + LOG(debug) << "fit abs.dist " << iev << " NC: " << ncAWb[iev] << " Chi2: " << (ncAWb[iev] ? fitters_host[iev].getChi2AtPCACandidate(0) : -1); + if (ncAWb[iev]) { + auto minDb = checkResults(outStreamB, treeName2AWbulk, fitters_host[iev], vtxGen[iev], genParents[iev], k0dec); + meanDAWb += minDb; + nfoundAWb++; + } + } + + ft.setUseAbsDCA(false); + ft.setWeightedFinalPCA(false); + std::fill(fitters_host.begin(), fitters_host.end(), ft); + swWb.Start(false); + std::vector ncWb(NTest, 0); + device::processBulk(nBlocks, nThreads, nBatches, fitters_host, ncWb, vctracks[0], vctracks[1]); // HERE WE FIT THE VERTICES + swWb.Stop(); + + for (int iev = 0; iev < NTest; iev++) { + LOG(debug) << "fit wgh.dist " << iev << " NC: " << ncWb[iev] << " Chi2: " << (ncWb[iev] ? fitters_host[iev].getChi2AtPCACandidate(0) : -1); + if (ncWb[iev]) { + auto minDb = checkResults(outStreamB, treeName2Wbulk, fitters_host[iev], vtxGen[iev], genParents[iev], k0dec); + meanDWb += minDb; + nfoundWb++; + } + } + + // + meanDAb /= nfoundAb ? nfoundAb : 1; + meanDAWb /= nfoundAWb ? nfoundAWb : 1; + meanDWb /= nfoundWb ? nfoundWb : 1; + LOG(info) << "Bulk-processed " << NTest << " 2-prong vertices: Helix : Line"; + LOG(info) << "2-prongs with abs.dist minimization: eff = " << float(nfoundAb) / NTest + << " mean.dist to truth: " << meanDAb << " Total time: " << swAb.CpuTime() * 1000 << " ms"; + LOG(info) << "2-prongs with abs.dist but wghPCA: eff = " << float(nfoundAWb) / NTest + << " mean.dist to truth: " << meanDAWb << " Total time: " << swAWb.CpuTime() * 1000 << " ms"; + LOG(info) << "2-prongs with wgh.dist minimization: eff = " << float(nfoundWb) / NTest + << " mean.dist to truth: " << meanDWb << " Total time: " << swWb.CpuTime() * 1000 << " ms"; + BOOST_CHECK(nfoundAb > 0.99 * NTest); + BOOST_CHECK(nfoundAWb > 0.99 * NTest); + BOOST_CHECK(nfoundWb > 0.99 * NTest); + BOOST_CHECK(meanDAb < 0.1); + BOOST_CHECK(meanDAWb < 0.1); + BOOST_CHECK(meanDWb < 0.1); + } + + // 2 prongs vertices with both of charges set to 0: Line : Line + { + std::vector forceQ{0, 0}; + LOG(info) << "\n\nBulk-processing 2-prong Line - Line case"; + o2::vertexing::DCAFitterN<2> ft; // 2 prong fitter + ft.setBz(bz); + ft.setPropagateToPCA(true); // After finding the vertex, propagate tracks to the DCA. This is default anyway + ft.setMaxR(200); // do not consider V0 seeds with 2D circles crossing above this R. This is default anyway + ft.setMaxDZIni(4); // do not consider V0 seeds with tracks Z-distance exceeding this. This is default anyway + ft.setMinParamChange(1e-3); // stop iterations if max correction is below this value. This is default anyway + ft.setMinRelChi2Change(0.9); // stop iterations if chi2 improves by less that this factor + + std::vector> fitters_host(NTest); + std::vector genParents(NTest); + + std::string treeName2Abulk = "pr2aLL", treeName2AWbulk = "pr2awLL", treeName2Wbulk = "pr2wLL"; + TStopwatch swAb, swAWb, swWb; + int nfoundAb = 0, nfoundAWb = 0, nfoundWb = 0; + double meanDAb = 0, meanDAWb = 0, meanDWb = 0; + swAb.Stop(); + swAWb.Stop(); + swWb.Stop(); + for (int iev = 0; iev < NTest; iev++) { + forceQ[0] = forceQ[1] = 0; + std::vector vc(2); + genParents[iev] = generate(vtxGen[iev], vc, bz, genPHS, k0, k0dec, forceQ); + vctracks[0][iev] = vc[0]; + vctracks[1][iev] = vc[1]; + } + + ft.setUseAbsDCA(true); + std::fill(fitters_host.begin(), fitters_host.end(), ft); + + swAb.Start(false); + std::vector ncAb(NTest, 0); + device::processBulk(nBlocks, nThreads, nBatches, fitters_host, ncAb, vctracks[0], vctracks[1]); // HERE WE FIT THE VERTICES + swAb.Stop(); + + for (int iev = 0; iev < NTest; iev++) { + LOG(debug) << "fit abs.dist " << iev << " NC: " << ncAb[iev] << " Chi2: " << (ncAb[iev] ? fitters_host[iev].getChi2AtPCACandidate(0) : -1); + if (ncAb[iev]) { + auto minDb = checkResults(outStreamB, treeName2Abulk, fitters_host[iev], vtxGen[iev], genParents[iev], k0dec); + meanDAb += minDb; + nfoundAb++; + } + } + + ft.setUseAbsDCA(true); + ft.setWeightedFinalPCA(true); + std::fill(fitters_host.begin(), fitters_host.end(), ft); + swAWb.Start(false); + std::vector ncAWb(NTest, 0); + device::processBulk(nBlocks, nThreads, nBatches, fitters_host, ncAWb, vctracks[0], vctracks[1]); // HERE WE FIT THE VERTICES + swAWb.Stop(); + for (int iev = 0; iev < NTest; iev++) { + LOG(debug) << "fit abs.dist " << iev << " NC: " << ncAWb[iev] << " Chi2: " << (ncAWb[iev] ? fitters_host[iev].getChi2AtPCACandidate(0) : -1); + if (ncAWb[iev]) { + auto minDb = checkResults(outStreamB, treeName2AWbulk, fitters_host[iev], vtxGen[iev], genParents[iev], k0dec); + meanDAWb += minDb; + nfoundAWb++; + } + } + + ft.setUseAbsDCA(false); + ft.setWeightedFinalPCA(false); + std::fill(fitters_host.begin(), fitters_host.end(), ft); + + swWb.Start(false); + std::vector ncWb(NTest, 0); + device::processBulk(nBlocks, nThreads, nBatches, fitters_host, ncWb, vctracks[0], vctracks[1]); // HERE WE FIT THE VERTICES + swWb.Stop(); + + for (int iev = 0; iev < NTest; iev++) { + LOG(debug) << "fit wgh.dist " << iev << " NC: " << ncWb[iev] << " Chi2: " << (ncWb[iev] ? fitters_host[iev].getChi2AtPCACandidate(0) : -1); + if (ncWb[iev]) { + auto minDb = checkResults(outStreamB, treeName2Wbulk, fitters_host[iev], vtxGen[iev], genParents[iev], k0dec); + meanDWb += minDb; + nfoundWb++; + } + } + // ft.print(); + meanDAb /= nfoundAb ? nfoundAb : 1; + meanDAWb /= nfoundAWb ? nfoundAWb : 1; + meanDWb /= nfoundWb ? nfoundWb : 1; + LOG(info) << "Bulk-processed " << NTest << " 2-prong vertices: Line : Line"; + LOG(info) << "2-prongs with abs.dist minimization: eff = " << float(nfoundAb) / NTest + << " mean.dist to truth: " << meanDAb << " Total time: " << swAb.CpuTime() * 1000 << " ms"; + LOG(info) << "2-prongs with abs.dist but wghPCA: eff = " << float(nfoundAWb) / NTest + << " mean.dist to truth: " << meanDAWb << " Total time: " << swAWb.CpuTime() * 1000 << " ms"; + LOG(info) << "2-prongs with wgh.dist minimization: eff = " << float(nfoundWb) / NTest + << " mean.dist to truth: " << meanDWb << " Total time: " << swWb.CpuTime() * 1000 << " ms"; + BOOST_CHECK(nfoundAb > 0.99 * NTest); + BOOST_CHECK(nfoundAWb > 0.99 * NTest); + BOOST_CHECK(nfoundWb > 0.99 * NTest); + BOOST_CHECK(meanDAb < 0.1); + BOOST_CHECK(meanDAWb < 0.1); + BOOST_CHECK(meanDWb < 0.1); + } + + // Bulk-process 3 prongs vertices + { + LOG(info) << "\n\nBulk-processing 3-prongs"; + std::vector forceQ{1, 1, 1}; + + o2::vertexing::DCAFitterN<3> ft; // 3 prong fitter + ft.setBz(bz); + ft.setPropagateToPCA(true); // After finding the vertex, propagate tracks to the DCA. This is default anyway + ft.setMaxR(200); // do not consider V0 seeds with 2D circles crossing above this R. This is default anyway + ft.setMaxDZIni(4); // do not consider V0 seeds with tracks Z-distance exceeding this. This is default anyway + ft.setMinParamChange(1e-3); // stop iterations if max correction is below this value. This is default anyway + ft.setMinRelChi2Change(0.9); // stop iterations if chi2 improves by less that this factor + + std::vector> fitters_host(NTest); + std::vector genParents(NTest); + + std::string treeName3Abulk = "pr3a", treeName3AWbulk = "pr3aw", treeName3Wbulk = "pr3w"; + TStopwatch swAb, swAWb, swWb; + int nfoundAb = 0, nfoundAWb = 0, nfoundWb = 0; + double meanDAb = 0, meanDAWb = 0, meanDWb = 0; + swAb.Stop(); + swAWb.Stop(); + swWb.Stop(); + for (int iev = 0; iev < NTest; iev++) { + std::vector vc(3); + genParents[iev] = generate(vtxGen[iev], vc, bz, genPHS, dch, dchdec, forceQ); + + vctracks[0][iev] = vc[0]; + vctracks[1][iev] = vc[1]; + vctracks[2][iev] = vc[2]; + } + + ft.setUseAbsDCA(true); + std::fill(fitters_host.begin(), fitters_host.end(), ft); + swAb.Start(false); + std::vector ncAb(NTest, 0); + device::processBulk(nBlocks, nThreads, nBatches, fitters_host, ncAb, vctracks[0], vctracks[1], vctracks[2]); // HERE WE FIT THE VERTICES + swAb.Stop(); + for (int iev = 0; iev < NTest; iev++) { + LOG(debug) << "fit abs.dist " << iev << " NC: " << ncAb[iev] << " Chi2: " << (ncAb[iev] ? fitters_host[iev].getChi2AtPCACandidate(0) : -1); + if (ncAb[iev]) { + auto minDb = checkResults(outStreamB, treeName3Abulk, fitters_host[iev], vtxGen[iev], genParents[iev], dchdec); + meanDAb += minDb; + nfoundAb++; + } + } + + ft.setUseAbsDCA(true); + ft.setWeightedFinalPCA(true); + std::fill(fitters_host.begin(), fitters_host.end(), ft); + + swAWb.Start(false); + std::vector ncAWb(NTest, 0); + device::processBulk(nBlocks, nThreads, nBatches, fitters_host, ncAWb, vctracks[0], vctracks[1], vctracks[2]); // HERE WE FIT THE VERTICES + swAWb.Stop(); + for (int iev = 0; iev < NTest; iev++) { + LOG(debug) << "fit abs.dist " << iev << " NC: " << ncAWb[iev] << " Chi2: " << (ncAWb[iev] ? fitters_host[iev].getChi2AtPCACandidate(0) : -1); + if (ncAWb[iev]) { + auto minDb = checkResults(outStreamB, treeName3AWbulk, fitters_host[iev], vtxGen[iev], genParents[iev], dchdec); + meanDAWb += minDb; + nfoundAWb++; + } + } + + ft.setUseAbsDCA(false); + ft.setWeightedFinalPCA(false); + std::fill(fitters_host.begin(), fitters_host.end(), ft); + + swWb.Start(false); + std::vector ncWb(NTest, 0); + device::processBulk(nBlocks, nThreads, nBatches, fitters_host, ncWb, vctracks[0], vctracks[1], vctracks[2]); // HERE WE FIT THE VERTICES + swWb.Stop(); + for (int iev = 0; iev < NTest; iev++) { + LOG(debug) << "fit wgh.dist " << iev << " NC: " << ncWb[iev] << " Chi2: " << (ncWb[iev] ? fitters_host[iev].getChi2AtPCACandidate(0) : -1); + if (ncWb[iev]) { + auto minDb = checkResults(outStreamB, treeName3Wbulk, fitters_host[iev], vtxGen[iev], genParents[iev], dchdec); + meanDWb += minDb; + nfoundWb++; + } + } + + // ft.print(); + meanDAb /= nfoundAb ? nfoundAb : 1; + meanDAWb /= nfoundAWb ? nfoundAWb : 1; + meanDWb /= nfoundWb ? nfoundWb : 1; + LOG(info) << "Bulk-processed " << NTest << " 3-prong vertices"; + LOG(info) << "3-prongs with abs.dist minimization: eff = " << float(nfoundAb) / NTest + << " mean.dist to truth: " << meanDAb << " Total time: " << swAb.CpuTime() * 1000 << " ms"; + LOG(info) << "3-prongs with abs.dist but wghPCA: eff = " << float(nfoundAWb) / NTest + << " mean.dist to truth: " << meanDAWb << " Total time: " << swAWb.CpuTime() * 1000 << " ms"; + LOG(info) << "3-prongs with wgh.dist minimization: eff = " << float(nfoundWb) / NTest + << " mean.dist to truth: " << meanDWb << " Total time: " << swWb.CpuTime() * 1000 << " ms"; + BOOST_CHECK(nfoundAb > 0.99 * NTest); + BOOST_CHECK(nfoundAWb > 0.99 * NTest); + BOOST_CHECK(nfoundWb > 0.99 * NTest); + BOOST_CHECK(meanDAb < 0.1); + BOOST_CHECK(meanDAWb < 0.1); + BOOST_CHECK(meanDWb < 0.1); + } + outStreamB.Close(); +} + +} // namespace vertexing +} // namespace o2 \ No newline at end of file diff --git a/Common/DCAFitter/GPU/hip/CMakeLists.txt b/Common/DCAFitter/GPU/hip/CMakeLists.txt new file mode 100644 index 0000000000000..5e7821a0b8946 --- /dev/null +++ b/Common/DCAFitter/GPU/hip/CMakeLists.txt @@ -0,0 +1,35 @@ +# Copyright 2019-2020 CERN and copyright holders of ALICE O2. +# See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +# All rights not expressly granted are reserved. +# +# This software is distributed under the terms of the GNU General Public +# License v3 (GPL Version 3), copied verbatim in the file "COPYING". +# +# In applying this license CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization +# or submit itself to any jurisdiction. + +set(CMAKE_HIP_FLAGS "${CMAKE_HIP_FLAGS} -fgpu-rdc") +o2_add_hipified_library(DCAFitterHIP + SOURCES ../cuda/DCAFitterN.cu + ../cuda/GPUInterface.cu + PUBLIC_INCLUDE_DIRECTORIES ../../include ../ + PUBLIC_LINK_LIBRARIES O2::MathUtils + O2::ReconstructionDataFormats + O2::DetectorsBase + hip::host + PRIVATE_LINK_LIBRARIES O2::GPUTrackingHIPExternalProvider + TARGETVARNAME targetNAme) + +#o2_add_test(DCAFitterNHIP +# SOURCES ../cuda/test/testDCAFitterNGPU.cxx +# PUBLIC_LINK_LIBRARIES O2::ReconstructionDataFormats +# O2::DCAFitterHIP +# O2::DCAFitter +# ROOT::Core +# ROOT::Physics +# HIPIFIED test +# COMPONENT_NAME gpu +# LABELS vertexing +# ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage +# VMCWORKDIR=${CMAKE_BINARY_DIR}/stage/${CMAKE_INSTALL_DATADIR}) diff --git a/Common/DCAFitter/README.md b/Common/DCAFitter/README.md new file mode 100644 index 0000000000000..e385378d10caf --- /dev/null +++ b/Common/DCAFitter/README.md @@ -0,0 +1,133 @@ + + +# DCAFitterN + +Templated class to fit the Point of Closest Approach (PCA) of secondary vertex with N prongs. Allows minimization of either absolute or weighted Distances of Closest Approach (DCA) of N tracks to their common PCA. + +For every N (prongs) a separate specialization must be instantiated, e.g. +```cpp +using Track = o2::track::TrackParCov; +o2::vertexing::DCAFitterN<2,Track> ft2; // 2-prongs fitter +// or, to set at once some parameters +float bz = 5.; // field in kGauss +bool useAbsDCA = true; // use abs. DCA minimizaition instead of default weighted +bool propToDCA = true; // after fit, create a copy of tracks at the found PCA +o2::vertexing::DCAFitterN<3,Track> ft3(bz, useAbsDCA, propToDCA); // 3-prongs fitter +``` +One can also use predefined aliases ``o2::vertexing::DCAFitter2`` and ``o2::vertexing::DCAFitter3``; +The main processing method is +```cpp +o2::vertexing::DCAFitterN::process(const Track& trc1,..., cons Track& trcN); +``` + +The typical use case is (for e.g. 3-prong fitter): +```cpp +using Vec3D = ROOT::Math::SVector; // this is a type of the fitted vertex +o2::vertexing::DCAFitter3 ft; +ft.setBz(5.0); +ft.setPropagateToPCA(true); // After finding the vertex, propagate tracks to the DCA. This is default anyway +ft.setMaxR(200); // do not consider V0 seeds with 2D circles crossing above this R. This is default anyway +ft.setMaxDZIni(4); // do not consider V0 seeds with tracks Z-distance exceeding this. This is default anyway +ft.setMaxDXYIni(4); // do not consider V0 seeds with tracks XY-distance exceeding this. This is default anyway +ft.setMinParamChange(1e-3); // stop iterations if max correction is below this value. This is default anyway +ft.setMinRelChi2Change(0.9);// stop iterations if chi2 improves by less that this factor +ft.setMaxChi2(10); // discard vertices with chi2/Nprongs (or sum{DCAi^2}/Nprongs for abs. distance minimization) + +Track tr0,tr1,tr2; // decide candidate tracks +int nc = ft.process(tr0,tr1,tr2); // one can have up to 2 candidates, though the 2nd (if any) will have worse quality +if (nc) { + Vec3D vtx = ft.getPCACandidate(); // same as ft.getPCACandidate(0); + LOG(info) << "found vertex " << vtx[0] << ' ' << vtx[1] << ' ' << vtx[2]; + // access the track's X parameters at PCA + for (int i=0;i<3;i++) { + LOG(info) << "Track " << i << " at PCA for X = " << ft.getTrackX(i); + } + // access directly the tracks propagated to the DCA + for (int i=0;i<3;i++) { + const auto& track = ft.getTrack(i); + track.print(); + } +} +``` + +By default the propagation is done with bZ provided by the user and w/o material corrections applied. +One can request the propagation with full local field and/or material corrections by setting +``` +ft.setUsePropagator(true); // use must take care of initialization of the propagator (loading geometry and magnetic field) +ft.setMatCorrType(o2::base::Propagator::MatCorrType::USEMatCorrLUT); // of USEMatCorrTGeo +``` + +Note that if material correction is not default USEMatCorrNone, then the propagator will be used even if not requested (hence must be initialized by the user). + +To get the most precise results one can request `ft.setRefitWithMatCorr(true)`: in this case when `propagateTracksToVertex()` is called, the tracks will be propagated +to the V0 with requested material corrections, one new V0 minimization will be done and only after that the final propagation to final V0 position will be done. +Since this is CPU consiming, it is reccomended to disable propagation to V0 by default (`ft.setPropagateToPCA(false)`) and call separately `ft.propagateTracksToVertex()` +after preliminary checks on the V0 candidate. + +By default the final V0 position is defined as +1) With `useAbsDCA = true`: simple average of tracks position propagated to respective `X_dca` parameters and rotated to the lab. frame. +2) With `useAbsDCA = false`: weighted (by tracks covariances) average of tracks position propagated to respective `X_dca` parameters and rotated to the lab. frame. + +Extra method `setWeightedFinalPCA(bool)` is provided for the "mixed" mode: if `setWeightedFinalPCA(true)` is set with `useAbsDCA = true` before the `process` call, the minimization will be done neglecting the track covariances, +but the final V0 position will be calculated using weighted average. One can also recalculate the V0 position by the weighted average method by calling explicitly +`ft.recalculatePCAWithErrors(int icand=0)`, w/o prior call of `setWeightedFinalPCA(true)`: this will update the position returned by the `getPCACandidate(int cand = 0)`. + +The covariance matrix of the V0 position is calculated as an inverted sum of tracks inversed covariances at respective `X_dca` points. + +See ``O2/Common/DCAFitter/test/testDCAFitterN.cxx`` for more extended example. +Currently only 2 and 3 prongs permitted, thought this can be changed by modifying ``DCAFitterN::NMax`` constant. + +## Error handling + +It may happen that the track propagation to the the proximity of the PCA fails at the various stage of the fit. In this case the fit is abandoned and the failure flag is set, it can be checked using +isPropagationFailure(int cand = 0)` method. + +Also, due to the linearization errors the covariance matrix of the track propagated to some point may become non-positive defined. +In this case the relevant correlation coefficient of the cov.matrix is redefined to cure the position part of the cov.matrix and further program flow depends on the user settings for `DCAFitterN::setBadCovPolicy(v)`: + +`DCAFitterN::setBadCovPolicy(DCAFitterN::Discard);` : abandon fit (default) + +`DCAFitterN::setBadCovPolicy(DCAFitterN::Override);` : continue fit with overridden cov.matrix + +`DCAFitterN::setBadCovPolicy(DCAFitterN::OverrideAnFlag);` continue fit with overridden cov.matrix but set the propagation failure flag (can be checked using the same `isPropagationFailure(int cand = 0)` method). + +## Fit status +The fitter provides a fit status for each candidate, which can be retrieved using: +``` +FitStatus status = ft.getFitStatus(int cand = 0); +``` +The possible values are: +``` +enum FitStatus : uint8_t { // part of the DCAFitterN class + None, // no status set (should not be possible!) + + /* Good Conditions */ + Converged, // fit converged + MaxIter, // max iterations reached before fit convergence (can still be a good vertex) + + /* Error Conditions */ + NoCrossing, // no reasonable crossing was found + RejRadius, // radius of crossing was not acceptable + RejTrackX, // one candidate track x was below the minimum required radius + RejTrackRoughZ, // rejected by rough cut on tracks Z difference + RejChi2Max, // rejected by maximum chi2 cut + FailProp, // propagation of at least prong to PCA failed + FailInvCov, // inversion of cov.-matrix failed + FailInvWeight, // inversion of Ti weight matrix failed + FailInv2ndDeriv, // inversion of 2nd derivatives failed + FailCorrTracks, // correction of tracks to updated x failed + FailCloserAlt, // alternative PCA is closer +}; +``` +This is allows to track where candiate fit was abondended. +``` +int nc = ft.process(tr0,tr1,tr2); +auto status = ft.getFitStatus(); +if (nc) { + // status can either be FitStatus::Converged or FitStatus::MaxIter +} +// status can be on of the error conditions +``` +A more thorough example is given in `testDCAFitterN.cxx`. diff --git a/Common/DCAFitter/include/DCAFitter/DCAFitterN.h b/Common/DCAFitter/include/DCAFitter/DCAFitterN.h new file mode 100644 index 0000000000000..2641dec84aed9 --- /dev/null +++ b/Common/DCAFitter/include/DCAFitter/DCAFitterN.h @@ -0,0 +1,1300 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file DCAFitterN.h +/// \brief Defintions for N-prongs secondary vertex fit +/// \author ruben.shahoyan@cern.ch +/// For the formulae derivation see /afs/cern.ch/user/s/shahoian/public/O2/DCAFitter/DCAFitterN.pdf + +#ifndef _ALICEO2_DCA_FITTERN_ +#define _ALICEO2_DCA_FITTERN_ + +#include "ReconstructionDataFormats/HelixHelper.h" +#include "DetectorsBase/Propagator.h" +#include "MathUtils/Cartesian.h" +#include "ReconstructionDataFormats/Track.h" + +namespace o2 +{ +namespace vertexing +{ + +///__________________________________________________________________________________ +///< Inverse cov matrix (augmented by a dummy X error) of the point defined by the track +struct TrackCovI { + float sxx, syy, syz, szz; + + GPUdDefault() TrackCovI() = default; + + GPUd() bool set(const o2::track::TrackParCov& trc, float xerrFactor = 1.f) + { + // we assign Y error to X for DCA calculation + // (otherwise for quazi-collinear tracks the X will not be constrained) + float cyy = trc.getSigmaY2(), czz = trc.getSigmaZ2(), cyz = trc.getSigmaZY(), cxx = cyy * xerrFactor; + float detYZ = cyy * czz - cyz * cyz; + bool res = true; + if (detYZ <= 0.) { + cyz = o2::gpu::GPUCommonMath::Sqrt(cyy * czz) * (cyz > 0 ? 0.98f : -0.98f); + detYZ = cyy * czz - cyz * cyz; + res = false; + } + auto detYZI = 1. / detYZ; + sxx = 1. / cxx; + syy = czz * detYZI; + syz = -cyz * detYZI; + szz = cyy * detYZI; + return res; + } +}; + +///__________________________________________________________________________ +///< Derivative (up to 2) of the TrackParam position over its running param X +struct TrackDeriv { + float dydx, dzdx, d2ydx2, d2zdx2; + GPUdDefault() TrackDeriv() = default; + GPUd() TrackDeriv(const o2::track::TrackPar& trc, float bz) { set(trc, bz); } + GPUd() void set(const o2::track::TrackPar& trc, float bz) + { + float snp = trc.getSnp(), csp = o2::gpu::GPUCommonMath::Sqrt((1. - snp) * (1. + snp)), cspI = 1. / csp, crv2c = trc.getCurvature(bz) * cspI; + dydx = snp * cspI; // = snp/csp + dzdx = trc.getTgl() * cspI; // = tgl/csp + d2ydx2 = crv2c * cspI * cspI; // = crv/csp^3 + d2zdx2 = crv2c * dzdx * dydx; // = crv*tgl*snp/csp^3 + } +}; + +///__________________________________________________________________________ +///< Log log-throttling helper +struct LogLogThrottler { + size_t evCount{0}; + size_t nextLog{1}; + GPUdi() bool needToLog() + { + if (++evCount > nextLog) { + nextLog *= 2; + return true; + } + return false; + } + GPUdi() void clear() + { + evCount = 0; + nextLog = 1; + } +}; + +template +class DCAFitterN +{ + static constexpr double NMin = 2; + static constexpr double NMax = 4; + static constexpr double NInv = 1. / N; + static constexpr int MAXHYP = 2; + static constexpr float XerrFactor = 5.; // factor for conversion of track covYY to dummy covXX + using Track = o2::track::TrackParCov; + using TrackAuxPar = o2::track::TrackAuxPar; + using CrossInfo = o2::track::CrossInfo; + + using Vec3D = o2::math_utils::SVector; + using VecND = o2::math_utils::SVector; + using MatSym3D = o2::math_utils::SMatrix>; + using MatStd3D = o2::math_utils::SMatrix>; + using MatSymND = o2::math_utils::SMatrix>; + using MatStdND = o2::math_utils::SMatrix>; + using TrackCoefVtx = MatStd3D; + using ArrTrack = std::array; // container for prongs (tracks) at single vertex cand. + using ArrTrackCovI = std::array; // container for inv.cov.matrices at single vertex cand. + using ArrTrCoef = std::array; // container of TrackCoefVtx coefficients at single vertex cand. + using ArrTrDer = std::array; // container of Track 1st and 2nd derivative over their X param + using ArrTrPos = std::array; // container of Track positions + + public: + enum BadCovPolicy : uint8_t { // if encountering non-positive defined cov. matrix, the choice is: + Discard = 0, // stop evaluation + Override = 1, // override correlation coef. to have cov.matrix pos.def and continue + OverrideAndFlag = 2 // override correlation coef. to have cov.matrix pos.def, set mPropFailed flag of corresponding candidate to true and continue (up to the user to check the flag) + }; + + enum FitStatus : uint8_t { // fit status of crossing hypothesis + None, // no status set (should not be possible!) + + /* Good Conditions */ + Converged, // fit converged + MaxIter, // max iterations reached before fit convergence + + /* Error Conditions */ + NoCrossing, // no reasaonable crossing was found + RejRadius, // radius of crossing was not acceptable + RejTrackX, // one candidate track x was below the mimimum required radius + RejTrackRoughZ, // rejected by rough cut on tracks Z difference + RejChi2Max, // rejected by maximum chi2 cut + FailProp, // propagation of at least prong to PCA failed + FailInvCov, // inversion of cov.-matrix failed + FailInvWeight, // inversion of Ti weight matrix failed + FailInv2ndDeriv, // inversion of 2nd derivatives failed + FailCorrTracks, // correction of tracks to updated x failed + FailCloserAlt, // alternative PCA is closer + // + NStatusesDefined + }; + + static constexpr int getNProngs() { return N; } + + DCAFitterN() = default; + DCAFitterN(float bz, bool useAbsDCA, bool prop2DCA) : mBz(bz), mUseAbsDCA(useAbsDCA), mPropagateToPCA(prop2DCA) + { + static_assert(N >= NMin && N <= NMax, "N prongs outside of allowed range"); + } + + //========================================================================= + ///< return PCA candidate, by default best on is provided (no check for the index validity) + GPUd() const Vec3D& getPCACandidate(int cand = 0) const { return mPCA[mOrder[cand]]; } + GPUd() const auto getPCACandidatePos(int cand = 0) const + { + const auto& vd = mPCA[mOrder[cand]]; + return std::array{static_cast(vd[0]), static_cast(vd[1]), static_cast(vd[2])}; + } + + ///< return position of quality-ordered candidate in the internal structures + int getCandidatePosition(int cand = 0) const { return mOrder[cand]; } + + ///< return Chi2 at PCA candidate (no check for its validity) + float getChi2AtPCACandidate(int cand = 0) const { return mChi2[mOrder[cand]]; } + + ///< prepare copies of tracks at the V0 candidate (no check for the candidate validity) + /// must be called before getTrack(i,cand) query + GPUd() bool propagateTracksToVertex(int cand = 0); + + ///< check if propagation of tracks to candidate vertex was done + GPUd() bool isPropagateTracksToVertexDone(int cand = 0) const { return mTrPropDone[mOrder[cand]]; } + + ///< check if propagation of tracks to candidate vertex failed + bool isPropagationFailure(int cand = 0) const { return mPropFailed[mOrder[cand]]; } + + ///< track param propagated to V0 candidate (no check for the candidate validity) + /// propagateTracksToVertex must be called in advance + Track& getTrack(int i, int cand = 0) + { + if (!mTrPropDone[mOrder[cand]]) { +#ifndef GPUCA_GPUCODE_DEVICE + throw std::runtime_error("propagateTracksToVertex was not called yet"); +#endif + } + return mCandTr[mOrder[cand]][i]; + } + + const Track& getTrack(int i, int cand = 0) const + { + if (!mTrPropDone[mOrder[cand]]) { +#ifndef GPUCA_GPUCODE_DEVICE + throw std::runtime_error("propagateTracksToVertex was not called yet"); +#endif + } + return mCandTr[mOrder[cand]][i]; + } + + ///< create parent track param with errors for decay vertex + GPUd() o2::track::TrackParCov createParentTrackParCov(int cand = 0, bool sectorAlpha = true) const; + + ///< create parent track param w/o errors for decay vertex + GPUd() o2::track::TrackPar createParentTrackPar(int cand = 0, bool sectorAlpha = true) const; + + ///< calculate on the fly track param (no cov mat) at candidate, check isValid to make sure propagation was successful + GPUd() o2::track::TrackPar getTrackParamAtPCA(int i, int cand = 0); + + ///< recalculate PCA as a cov-matrix weighted mean, even if absDCA method was used + GPUd() bool recalculatePCAWithErrors(int cand = 0); + + GPUd() MatSym3D calcPCACovMatrix(int cand = 0) const; + + std::array calcPCACovMatrixFlat(int cand = 0) const + { + auto m = calcPCACovMatrix(cand); + return {static_cast(m(0, 0)), static_cast(m(1, 0)), static_cast(m(1, 1)), static_cast(m(2, 0)), static_cast(m(2, 1)), static_cast(m(2, 2))}; + } + + const Track* getOrigTrackPtr(int i) const { return mOrigTrPtr[i]; } + + GPUdi() FitStatus getFitStatus(int cand = 0) const noexcept { return mFitStatus[mOrder[cand]]; } + + ///< return number of iterations during minimization (no check for its validity) + GPUdi() int getNIterations(int cand = 0) const { return mNIters[mOrder[cand]]; } + GPUdi() void setPropagateToPCA(bool v = true) { mPropagateToPCA = v; } + GPUdi() void setMaxIter(int n = 20) { mMaxIter = n > 2 ? n : 2; } + GPUdi() void setMaxR(float r = 200.) { mMaxR2 = r * r; } + GPUdi() void setMaxDZIni(float d = 4.) { mMaxDZIni = d; } + GPUdi() void setMaxDXYIni(float d = 4.) { mMaxDXYIni = d > 0 ? d : 1e9; } + GPUdi() void setMaxChi2(float chi2 = 999.) { mMaxChi2 = chi2; } + GPUdi() void setBz(float bz) { mBz = o2::gpu::GPUCommonMath::Abs(bz) > o2::constants::math::Almost0 ? bz : 0.f; } + GPUdi() void setMinParamChange(float x = 1e-3) { mMinParamChange = x > 1e-4 ? x : 1.e-4; } + GPUdi() void setMinRelChi2Change(float r = 0.9) { mMinRelChi2Change = r > 0.1 ? r : 999.; } + GPUdi() void setUseAbsDCA(bool v) { mUseAbsDCA = v; } + GPUdi() void setWeightedFinalPCA(bool v) { mWeightedFinalPCA = v; } + GPUdi() void setMaxDistance2ToMerge(float v) { mMaxDist2ToMergeSeeds = v; } + GPUdi() void setMatCorrType(o2::base::Propagator::MatCorrType m = o2::base::Propagator::MatCorrType::USEMatCorrLUT) { mMatCorr = m; } + GPUdi() void setUsePropagator(bool v) { mUsePropagator = v; } + GPUdi() void setRefitWithMatCorr(bool v) { mRefitWithMatCorr = v; } + GPUdi() void setMaxSnp(float s) { mMaxSnp = s; } + GPUdi() void setMaxStep(float s) { mMaxStep = s; } + GPUdi() void setMinXSeed(float x) { mMinXSeed = x; } + GPUdi() void setCollinear(bool isCollinear) { mIsCollinear = isCollinear; } + + GPUdi() int getNCandidates() const { return mCurHyp; } + GPUdi() int getMaxIter() const { return mMaxIter; } + GPUdi() float getMaxR() const { return o2::gpu::GPUCommonMath::Sqrt(mMaxR2); } + GPUdi() float getMaxDZIni() const { return mMaxDZIni; } + GPUdi() float getMaxDXYIni() const { return mMaxDXYIni; } + GPUdi() float getMaxChi2() const { return mMaxChi2; } + GPUdi() float getMinParamChange() const { return mMinParamChange; } + GPUdi() float getBz() const { return mBz; } + GPUdi() float getMaxDistance2ToMerge() const { return mMaxDist2ToMergeSeeds; } + GPUdi() bool getUseAbsDCA() const { return mUseAbsDCA; } + GPUdi() bool getWeightedFinalPCA() const { return mWeightedFinalPCA; } + GPUdi() bool getPropagateToPCA() const { return mPropagateToPCA; } + GPUdi() o2::base::Propagator::MatCorrType getMatCorrType() const { return mMatCorr; } + GPUdi() bool getUsePropagator() const { return mUsePropagator; } + GPUdi() bool getRefitWithMatCorr() const { return mRefitWithMatCorr; } + GPUdi() float getMaxSnp() const { return mMaxSnp; } + GPUdi() float getMasStep() const { return mMaxStep; } + GPUdi() float getMinXSeed() const { return mMinXSeed; } + + template + GPUd() int process(const Tr&... args); + GPUd() void print() const; + + GPUdi() int getFitterID() const { return mFitterID; } + GPUdi() void setFitterID(int i) { mFitterID = i; } + GPUdi() size_t getCallID() const { return mCallID; } + + protected: + GPUd() bool calcPCACoefs(); + GPUd() bool calcInverseWeight(); + GPUd() void calcResidDerivatives(); + GPUd() void calcResidDerivativesNoErr(); + GPUd() void calcRMatrices(); + GPUd() void calcChi2Derivatives(); + GPUd() void calcChi2DerivativesNoErr(); + GPUd() void calcPCA(); + GPUd() void calcPCANoErr(); + GPUd() void calcTrackResiduals(); + GPUd() void calcTrackDerivatives(); + GPUd() double calcChi2() const; + GPUd() double calcChi2NoErr() const; + GPUd() bool correctTracks(const VecND& corrX); + GPUd() bool minimizeChi2(); + GPUd() bool minimizeChi2NoErr(); + GPUd() bool roughDZCut() const; + GPUd() bool closerToAlternative() const; + GPUd() bool propagateToX(o2::track::TrackParCov& t, float x); + GPUd() bool propagateParamToX(o2::track::TrackPar& t, float x); + + GPUd() static double getAbsMax(const VecND& v); + ///< track param positions at V0 candidate (no check for the candidate validity) + GPUdi() const Vec3D& getTrackPos(int i, int cand = 0) const { return mTrPos[mOrder[cand]][i]; } + + ///< track X-param at V0 candidate (no check for the candidate validity) + GPUd() float getTrackX(int i, int cand = 0) const { return getTrackPos(i, cand)[0]; } + + GPUd() MatStd3D getTrackRotMatrix(int i) const // generate 3D matrix for track rotation to global frame + { + MatStd3D mat; + mat(2, 2) = 1; + mat(0, 0) = mat(1, 1) = mTrAux[i].c; + mat(0, 1) = -mTrAux[i].s; + mat(1, 0) = mTrAux[i].s; + return mat; + } + + GPUd() MatSym3D getTrackCovMatrix(int i, int cand = 0) const // generate covariance matrix of track position, adding fake X error + { + const auto& trc = mCandTr[mOrder[cand]][i]; + MatSym3D mat; + mat(0, 0) = trc.getSigmaY2() * XerrFactor; + mat(1, 1) = trc.getSigmaY2(); + mat(2, 2) = trc.getSigmaZ2(); + mat(2, 1) = trc.getSigmaZY(); + return mat; + } + + GPUd() void assign(int) {} + template + GPUd() void assign(int i, const T& t, const Tr&... args) + { +#ifndef GPUCA_GPUCODE_DEVICE + static_assert(std::is_convertible(), "Wrong track type"); +#endif + mOrigTrPtr[i] = &t; + assign(i + 1, args...); + } + + GPUdi() void clear() + { + mCurHyp = 0; + mAllowAltPreference = true; + mOrder.fill(0); + mPropFailed.fill(false); + mTrPropDone.fill(false); + mNIters.fill(0); + mChi2.fill(-1); + mFitStatus.fill(FitStatus::None); + } + + GPUdi() static void setTrackPos(Vec3D& pnt, const Track& tr) + { + pnt[0] = tr.getX(); + pnt[1] = tr.getY(); + pnt[2] = tr.getZ(); + } + + GPUdi() void clearLogThrottlers() + { + mLoggerBadCov.clear(); + mLoggerBadInv.clear(); + mLoggerBadProp.clear(); + } + + void setBadCovPolicy(BadCovPolicy v) { mBadCovPolicy = v; } + BadCovPolicy getBadCovPolicy() const { return mBadCovPolicy; } + + private: + // vectors of 1st derivatives of track local residuals over X parameters + std::array, N> mDResidDx; + // vectors of 1nd derivatives of track local residuals over X parameters + // (cross-derivatives DR/(dx_j*dx_k) = 0 for j!=k, therefore the hessian is diagonal) + std::array, N> mD2ResidDx2; + VecND mDChi2Dx; // 1st derivatives of chi2 over tracks X params + MatSymND mD2Chi2Dx2; // 2nd derivatives of chi2 over tracks X params (symmetric matrix) + MatSymND mCosDif; // matrix with cos(alp_j-alp_i) for j mOrigTrPtr; + std::array mTrAux; // Aux track info for each track at each cand. vertex + CrossInfo mCrossings; // info on track crossing + + std::array mTrcEInv; // errors for each track at each cand. vertex + std::array mCandTr; // tracks at each cond. vertex (Note: Errors are at seed XY point) + std::array mTrCFVT; // TrackCoefVtx for each track at each cand. vertex + std::array mTrDer; // Track derivativse + std::array mTrPos; // Track positions + std::array mTrRes; // Track residuals + std::array mPCA; // PCA for each vertex candidate + std::array mChi2 = {0}; // Chi2 at PCA candidate + std::array mNIters; // number of iterations for each seed + std::array mTrPropDone{}; // Flag that the tracks are fully propagated to PCA + std::array mPropFailed{}; // Flag that some propagation failed for this PCA candidate + LogLogThrottler mLoggerBadCov{}; + LogLogThrottler mLoggerBadInv{}; + LogLogThrottler mLoggerBadProp{}; + MatSym3D mWeightInv; // inverse weight of single track, [sum{M^T E M}]^-1 in EQ.T + std::array mOrder{0}; + int mCurHyp = 0; + int mCrossIDCur = 0; + int mCrossIDAlt = -1; + BadCovPolicy mBadCovPolicy{BadCovPolicy::Discard}; // what to do in case of non-pos-def. cov. matrix, see BadCovPolicy enum + std::array mFitStatus{}; // fit status of each hypothesis fit + bool mAllowAltPreference = true; // if the fit converges to alternative PCA seed, abandon the current one + bool mUseAbsDCA = false; // use abs. distance minimization rather than chi2 + bool mWeightedFinalPCA = false; // recalculate PCA as a cov-matrix weighted mean, even if absDCA method was used + bool mPropagateToPCA = true; // create tracks version propagated to PCA + bool mUsePropagator = false; // use propagator with 3D B-field, set automatically if material correction is requested + bool mRefitWithMatCorr = false; // when doing propagateTracksToVertex, propagate tracks to V0 with material corrections and rerun minimization again + bool mIsCollinear = false; // use collinear fits when there 2 crossing points + o2::base::Propagator::MatCorrType mMatCorr = o2::base::Propagator::MatCorrType::USEMatCorrNONE; // material corrections type + int mMaxIter = 20; // max number of iterations + float mBz = 0; // bz field, to be set by user + float mMaxR2 = 200. * 200.; // reject PCA's above this radius + float mMinXSeed = -50.; // reject seed if it corresponds to X-param < mMinXSeed for one of candidates (e.g. X becomes strongly negative) + float mMaxDZIni = 4.; // reject (if>0) PCA candidate if tracks DZ exceeds threshold + float mMaxDXYIni = 4.; // reject (if>0) PCA candidate if tracks dXY exceeds threshold + float mMinParamChange = 1e-3; // stop iterations if largest change of any X is smaller than this + float mMinRelChi2Change = 0.9; // stop iterations is chi2/chi2old > this + float mMaxChi2 = 100; // abs cut on chi2 or abs distance + float mMaxDist2ToMergeSeeds = 1.; // merge 2 seeds to their average if their distance^2 is below the threshold + float mMaxSnp = 0.95; // Max snp for propagation with Propagator + float mMaxStep = 2.0; // Max step for propagation with Propagator + int mFitterID = 0; // locat fitter ID (mostly for debugging) + size_t mCallID = 0; + ClassDefNV(DCAFitterN, 3); +}; + +///_________________________________________________________________________ +template +template +GPUd() int DCAFitterN::process(const Tr&... args) +{ + // This is a main entry point: fit PCA of N tracks + mCallID++; + static_assert(sizeof...(args) == N, "incorrect number of input tracks"); + assign(0, args...); + clear(); + for (int i = 0; i < N; i++) { + mTrAux[i].set(*mOrigTrPtr[i], mBz); + } + if (!mCrossings.set(mTrAux[0], *mOrigTrPtr[0], mTrAux[1], *mOrigTrPtr[1], mMaxDXYIni, mIsCollinear)) { // even for N>2 it should be enough to test just 1 loop + mFitStatus[mCurHyp] = FitStatus::NoCrossing; + return 0; + } + if (mUseAbsDCA) { + calcRMatrices(); // needed for fast residuals derivatives calculation in case of abs. distance minimization + } + if (mCrossings.nDCA == MAXHYP) { // if there are 2 candidates and they are too close, chose their mean as a starting point + auto dst2 = (mCrossings.xDCA[0] - mCrossings.xDCA[1]) * (mCrossings.xDCA[0] - mCrossings.xDCA[1]) + + (mCrossings.yDCA[0] - mCrossings.yDCA[1]) * (mCrossings.yDCA[0] - mCrossings.yDCA[1]); + if (dst2 < mMaxDist2ToMergeSeeds) { + mCrossings.nDCA = 1; + mCrossings.xDCA[0] = 0.5 * (mCrossings.xDCA[0] + mCrossings.xDCA[1]); + mCrossings.yDCA[0] = 0.5 * (mCrossings.yDCA[0] + mCrossings.yDCA[1]); + } + } + // check all crossings + for (int ic = 0; ic < mCrossings.nDCA; ic++) { + // check if radius is acceptable + if (mCrossings.xDCA[ic] * mCrossings.xDCA[ic] + mCrossings.yDCA[ic] * mCrossings.yDCA[ic] > mMaxR2) { + mFitStatus[mCurHyp] = FitStatus::RejRadius; + continue; + } + mCrossIDCur = ic; + mCrossIDAlt = (mCrossings.nDCA == 2 && mAllowAltPreference) ? 1 - ic : -1; // works for max 2 crossings + mPCA[mCurHyp][0] = mCrossings.xDCA[ic]; + mPCA[mCurHyp][1] = mCrossings.yDCA[ic]; + + if (mUseAbsDCA ? minimizeChi2NoErr() : minimizeChi2()) { + mOrder[mCurHyp] = mCurHyp; + if (mPropagateToPCA && !propagateTracksToVertex(mCurHyp)) { + continue; // discard candidate if failed to propagate to it + } + mCurHyp++; + } + } + + for (int i = mCurHyp; i--;) { // order in quality + for (int j = i; j--;) { + if (mChi2[mOrder[i]] < mChi2[mOrder[j]]) { + o2::gpu::GPUCommonMath::Swap(mOrder[i], mOrder[j]); + } + } + } + if (mUseAbsDCA && mWeightedFinalPCA) { + for (int i = mCurHyp; i--;) { + recalculatePCAWithErrors(i); + } + } + return mCurHyp; +} + +//__________________________________________________________________________ +template +GPUd() bool DCAFitterN::calcPCACoefs() +{ + //< calculate Ti matrices for global vertex decomposition to V = sum_{0 +GPUd() bool DCAFitterN::calcInverseWeight() +{ + //< calculate [sum_{0 +GPUd() void DCAFitterN::calcResidDerivatives() +{ + //< calculate matrix of derivatives for weighted chi2: residual i vs parameter X of track j + MatStd3D matMT; + for (int i = N; i--;) { // residual being differentiated + const auto& taux = mTrAux[i]; + for (int j = N; j--;) { // track over which we differentiate + const auto& matT = mTrCFVT[mCurHyp][j]; // coefficient matrix for track J + const auto& trDx = mTrDer[mCurHyp][j]; // track point derivs over track X param + auto& dr1 = mDResidDx[i][j]; + auto& dr2 = mD2ResidDx2[i][j]; + // calculate M_i^tr * T_j + matMT[0][0] = taux.c * matT[0][0] + taux.s * matT[1][0]; + matMT[0][1] = taux.c * matT[0][1] + taux.s * matT[1][1]; + matMT[0][2] = taux.c * matT[0][2] + taux.s * matT[1][2]; + matMT[1][0] = -taux.s * matT[0][0] + taux.c * matT[1][0]; + matMT[1][1] = -taux.s * matT[0][1] + taux.c * matT[1][1]; + matMT[1][2] = -taux.s * matT[0][2] + taux.c * matT[1][2]; + matMT[2][0] = matT[2][0]; + matMT[2][1] = matT[2][1]; + matMT[2][2] = matT[2][2]; + + // calculate DResid_i/Dx_j = (delta_ij - M_i^tr * T_j) * DTrack_k/Dx_k + dr1[0] = -(matMT[0][0] + matMT[0][1] * trDx.dydx + matMT[0][2] * trDx.dzdx); + dr1[1] = -(matMT[1][0] + matMT[1][1] * trDx.dydx + matMT[1][2] * trDx.dzdx); + dr1[2] = -(matMT[2][0] + matMT[2][1] * trDx.dydx + matMT[2][2] * trDx.dzdx); + + // calculate D2Resid_I/(Dx_J Dx_K) = (delta_ijk - M_i^tr * T_j * delta_jk) * D2Track_k/dx_k^2 + dr2[0] = -(matMT[0][1] * trDx.d2ydx2 + matMT[0][2] * trDx.d2zdx2); + dr2[1] = -(matMT[1][1] * trDx.d2ydx2 + matMT[1][2] * trDx.d2zdx2); + dr2[2] = -(matMT[2][1] * trDx.d2ydx2 + matMT[2][2] * trDx.d2zdx2); + + if (i == j) { + dr1[0] += 1.; + dr1[1] += trDx.dydx; + dr1[2] += trDx.dzdx; + + dr2[1] += trDx.d2ydx2; + dr2[2] += trDx.d2zdx2; + } + } // track over which we differentiate + } // residual being differentiated +} + +//__________________________________________________________________________ +template +GPUd() void DCAFitterN::calcResidDerivativesNoErr() +{ + //< calculate matrix of derivatives for absolute distance chi2: residual i vs parameter X of track j + constexpr double NInv1 = 1. - NInv; // profit from Rii = I/Ninv + for (int i = N; i--;) { // residual being differentiated + const auto& trDxi = mTrDer[mCurHyp][i]; // track point derivs over track X param + auto& dr1ii = mDResidDx[i][i]; + auto& dr2ii = mD2ResidDx2[i][i]; + dr1ii[0] = NInv1; + dr1ii[1] = NInv1 * trDxi.dydx; + dr1ii[2] = NInv1 * trDxi.dzdx; + + dr2ii[0] = 0; + dr2ii[1] = NInv1 * trDxi.d2ydx2; + dr2ii[2] = NInv1 * trDxi.d2zdx2; + + for (int j = i; j--;) { // track over which we differentiate + auto& dr1ij = mDResidDx[i][j]; + auto& dr1ji = mDResidDx[j][i]; + const auto& trDxj = mTrDer[mCurHyp][j]; // track point derivs over track X param + auto cij = mCosDif[i][j], sij = mSinDif[i][j]; // M_i^T*M_j / N matrices non-trivial elements = {ci*cj+si*sj , si*cj-ci*sj }, see 5 in ref. + + // calculate DResid_i/Dx_j = (delta_ij - R_ij) * DTrack_j/Dx_j for j +GPUd() void DCAFitterN::calcRMatrices() +{ + //< calculate Rij = 1/N M_i^T * M_j matrices (rotation from j-th track to i-th track frame) + for (int i = N; i--;) { + const auto& mi = mTrAux[i]; + for (int j = i; j--;) { + const auto& mj = mTrAux[j]; + mCosDif[i][j] = (mi.c * mj.c + mi.s * mj.s) * NInv; // cos(alp_i-alp_j) / N + mSinDif[i][j] = (mi.s * mj.c - mi.c * mj.s) * NInv; // sin(alp_i-alp_j) / N + } + } +} + +//__________________________________________________________________________ +template +GPUd() void DCAFitterN::calcChi2Derivatives() +{ + //< calculate 1st and 2nd derivatives of wighted DCA (chi2) over track parameters X, see EQ.Chi2 in the ref + std::array, N> covIDrDx; // tempory vectors of covI_j * dres_j/dx_i + + // chi2 1st derivative + for (int i = N; i--;) { + auto& dchi1 = mDChi2Dx[i]; // DChi2/Dx_i = sum_j { res_j * covI_j * Dres_j/Dx_i } + dchi1 = 0; + for (int j = N; j--;) { + const auto& res = mTrRes[mCurHyp][j]; // vector of residuals of track j + const auto& covI = mTrcEInv[mCurHyp][j]; // inverse cov matrix of track j + const auto& dr1 = mDResidDx[j][i]; // vector of j-th residuals 1st derivative over X param of track i + auto& cidr = covIDrDx[i][j]; // vector covI_j * dres_j/dx_i, save for 2nd derivative calculation + cidr[0] = covI.sxx * dr1[0]; + cidr[1] = covI.syy * dr1[1] + covI.syz * dr1[2]; + cidr[2] = covI.syz * dr1[1] + covI.szz * dr1[2]; + // calculate res_i * covI_j * dres_j/dx_i + dchi1 += o2::math_utils::Dot(res, cidr); + } + } + // chi2 2nd derivative + for (int i = N; i--;) { + for (int j = i + 1; j--;) { // symmetric matrix + auto& dchi2 = mD2Chi2Dx2[i][j]; // D2Chi2/Dx_i/Dx_j = sum_k { Dres_k/Dx_j * covI_k * Dres_k/Dx_i + res_k * covI_k * D2res_k/Dx_i/Dx_j } + dchi2 = 0; + for (int k = N; k--;) { + const auto& dr1j = mDResidDx[k][j]; // vector of k-th residuals 1st derivative over X param of track j + const auto& cidrkj = covIDrDx[i][k]; // vector covI_k * dres_k/dx_i + dchi2 += o2::math_utils::Dot(dr1j, cidrkj); + if (k == j) { + const auto& res = mTrRes[mCurHyp][k]; // vector of residuals of track k + const auto& covI = mTrcEInv[mCurHyp][k]; // inverse cov matrix of track k + const auto& dr2ij = mD2ResidDx2[k][j]; // vector of k-th residuals 2nd derivative over X params of track j + dchi2 += res[0] * covI.sxx * dr2ij[0] + res[1] * (covI.syy * dr2ij[1] + covI.syz * dr2ij[2]) + res[2] * (covI.syz * dr2ij[1] + covI.szz * dr2ij[2]); + } + } + } + } +} + +//__________________________________________________________________________ +template +GPUd() void DCAFitterN::calcChi2DerivativesNoErr() +{ + //< calculate 1st and 2nd derivatives of abs DCA (chi2) over track parameters X, see (6) in the ref + for (int i = N; i--;) { + auto& dchi1 = mDChi2Dx[i]; // DChi2/Dx_i = sum_j { res_j * Dres_j/Dx_i } + dchi1 = 0; // chi2 1st derivative + for (int j = N; j--;) { + const auto& res = mTrRes[mCurHyp][j]; // vector of residuals of track j + const auto& dr1 = mDResidDx[j][i]; // vector of j-th residuals 1st derivative over X param of track i + dchi1 += o2::math_utils::Dot(res, dr1); + if (i >= j) { // symmetrix matrix + // chi2 2nd derivative + auto& dchi2 = mD2Chi2Dx2[i][j]; // D2Chi2/Dx_i/Dx_j = sum_k { Dres_k/Dx_j * covI_k * Dres_k/Dx_i + res_k * covI_k * D2res_k/Dx_i/Dx_j } + dchi2 = o2::math_utils::Dot(mTrRes[mCurHyp][i], mD2ResidDx2[i][j]); + for (int k = N; k--;) { + dchi2 += o2::math_utils::Dot(mDResidDx[k][i], mDResidDx[k][j]); + } + } + } + } +} + +//___________________________________________________________________ +template +GPUd() void DCAFitterN::calcPCA() +{ + // calculate point of closest approach for N prongs + mPCA[mCurHyp] = mTrCFVT[mCurHyp][N - 1] * mTrPos[mCurHyp][N - 1]; + for (int i = N - 1; i--;) { + mPCA[mCurHyp] += mTrCFVT[mCurHyp][i] * mTrPos[mCurHyp][i]; + } +} + +//___________________________________________________________________ +template +GPUd() bool DCAFitterN::recalculatePCAWithErrors(int cand) +{ + // recalculate PCA as a cov-matrix weighted mean, even if absDCA method was used + if (isPropagateTracksToVertexDone(cand) && !propagateTracksToVertex(cand)) { + return false; + } + int saveCurHyp = mCurHyp; + mCurHyp = mOrder[cand]; + if (mUseAbsDCA) { + for (int i = N; i--;) { + if (!mTrcEInv[mCurHyp][i].set(mCandTr[mCurHyp][i], XerrFactor)) { // prepare inverse cov.matrices at starting point + if (mLoggerBadCov.needToLog()) { +#ifndef GPUCA_GPUCODE + printf("fitter %d: error (%ld muted): overrode invalid track covariance from %s\n", + mFitterID, mLoggerBadCov.evCount, mCandTr[mCurHyp][i].asString().c_str()); +#else + printf("fitter %d: error (%ld muted): overrode invalid track covariance cyy:%e czz:%e cyz:%e\n", + mFitterID, mLoggerBadCov.evCount, mCandTr[mCurHyp][i].getSigmaY2(), mCandTr[mCurHyp][i].getSigmaZ2(), mCandTr[mCurHyp][i].getSigmaZY()); +#endif + } + mFitStatus[mCurHyp] = FitStatus::FailInvCov; + if (mBadCovPolicy == Discard) { + return false; + } else if (mBadCovPolicy == OverrideAndFlag) { + mPropFailed[mCurHyp] = true; + } // otherwise, just use overridden errors w/o flagging + } + } + if (!calcPCACoefs()) { + mCurHyp = saveCurHyp; + return false; + } + } + auto oldPCA = mPCA[mOrder[cand]]; + calcPCA(); + mCurHyp = saveCurHyp; + return true; +} + +//___________________________________________________________________ +template +GPUd() void DCAFitterN::calcPCANoErr() +{ + // calculate point of closest approach for N prongs w/o errors + auto& pca = mPCA[mCurHyp]; + o2::math_utils::rotateZd(mTrPos[mCurHyp][N - 1][0], mTrPos[mCurHyp][N - 1][1], pca[0], pca[1], mTrAux[N - 1].s, mTrAux[N - 1].c); + // RRRR mTrAux[N-1].loc2glo(mTrPos[mCurHyp][N-1][0], mTrPos[mCurHyp][N-1][1], pca[0], pca[1] ); + pca[2] = mTrPos[mCurHyp][N - 1][2]; + for (int i = N - 1; i--;) { + double x, y; + o2::math_utils::rotateZd(mTrPos[mCurHyp][i][0], mTrPos[mCurHyp][i][1], x, y, mTrAux[i].s, mTrAux[i].c); + // RRRR mTrAux[i].loc2glo(mTrPos[mCurHyp][i][0], mTrPos[mCurHyp][i][1], x, y ); + pca[0] += x; + pca[1] += y; + pca[2] += mTrPos[mCurHyp][i][2]; + } + pca[0] *= NInv; + pca[1] *= NInv; + pca[2] *= NInv; +} + +//___________________________________________________________________ +template +GPUd() o2::math_utils::SMatrix> DCAFitterN::calcPCACovMatrix(int cand) const +{ + // calculate covariance matrix for the point of closest approach + MatSym3D covm; + int nAdded = 0; + for (int i = N; i--;) { // calculate sum of inverses + // MatSym3D covTr = o2::math_utils::Similarity(mUseAbsDCA ? getTrackRotMatrix(i) : mTrCFVT[mOrder[cand]][i], getTrackCovMatrix(i, cand)); + // RS by using Similarity(mTrCFVT[mOrder[cand]][i], getTrackCovMatrix(i, cand)) we underestimate the error, use simple rotation + MatSym3D covTr = o2::math_utils::Similarity(getTrackRotMatrix(i), getTrackCovMatrix(i, cand)); + if (covTr.Invert()) { + covm += covTr; + nAdded++; + } + } + if (nAdded && covm.Invert()) { + return covm; + } + // correct way has failed, use simple sum + MatSym3D covmSum; + for (int i = N; i--;) { + MatSym3D covTr = o2::math_utils::Similarity(getTrackRotMatrix(i), getTrackCovMatrix(i, cand)); + covmSum += covTr; + } + return covmSum; +} + +//___________________________________________________________________ +template +GPUd() void DCAFitterN::calcTrackResiduals() +{ + // calculate residuals + Vec3D vtxLoc; + for (int i = N; i--;) { + mTrRes[mCurHyp][i] = mTrPos[mCurHyp][i]; + vtxLoc = mPCA[mCurHyp]; + o2::math_utils::rotateZInvd(vtxLoc[0], vtxLoc[1], vtxLoc[0], vtxLoc[1], mTrAux[i].s, mTrAux[i].c); // glo->loc + mTrRes[mCurHyp][i] -= vtxLoc; + } +} + +//___________________________________________________________________ +template +GPUdi() void DCAFitterN::calcTrackDerivatives() +{ + // calculate track derivatives over X param + for (int i = N; i--;) { + mTrDer[mCurHyp][i].set(mCandTr[mCurHyp][i], mBz); + } +} + +//___________________________________________________________________ +template +GPUdi() double DCAFitterN::calcChi2() const +{ + // calculate current chi2 + double chi2 = 0; + for (int i = N; i--;) { + const auto& res = mTrRes[mCurHyp][i]; + const auto& covI = mTrcEInv[mCurHyp][i]; + chi2 += res[0] * res[0] * covI.sxx + res[1] * res[1] * covI.syy + res[2] * res[2] * covI.szz + 2. * res[1] * res[2] * covI.syz; + } + return chi2; +} + +//___________________________________________________________________ +template +GPUdi() double DCAFitterN::calcChi2NoErr() const +{ + // calculate current chi2 of abs. distance minimization + double chi2 = 0; + for (int i = N; i--;) { + const auto& res = mTrRes[mCurHyp][i]; + chi2 += res[0] * res[0] + res[1] * res[1] + res[2] * res[2]; + } + return chi2; +} + +//___________________________________________________________________ +template +GPUd() bool DCAFitterN::correctTracks(const VecND& corrX) +{ + // propagate tracks to updated X + for (int i = N; i--;) { + const auto& trDer = mTrDer[mCurHyp][i]; + auto dx2h = 0.5 * corrX[i] * corrX[i]; + mTrPos[mCurHyp][i][0] -= corrX[i]; + mTrPos[mCurHyp][i][1] -= trDer.dydx * corrX[i] - dx2h * trDer.d2ydx2; + mTrPos[mCurHyp][i][2] -= trDer.dzdx * corrX[i] - dx2h * trDer.d2zdx2; + } + return true; +} + +//___________________________________________________________________ +template +GPUd() bool DCAFitterN::propagateTracksToVertex(int icand) +{ + // propagate tracks to current vertex + int ord = mOrder[icand]; + if (mTrPropDone[ord]) { + return true; + } + + // need to refit taking as a seed already found vertex + if (mRefitWithMatCorr) { + int curHypSav = mCurHyp, curCrosIDAlt = mCrossIDAlt; // save + mCurHyp = ord; + mCrossIDAlt = -1; // disable alternative check + auto restore = [this, curHypSav, curCrosIDAlt]() { this->mCurHyp = curHypSav; this->mCrossIDAlt = curCrosIDAlt; }; + if (!(mUseAbsDCA ? minimizeChi2NoErr() : minimizeChi2())) { // do final propagation + restore(); + return false; + } + restore(); + } + + for (int i = N; i--;) { + if (mUseAbsDCA || mUsePropagator || mMatCorr != o2::base::Propagator::MatCorrType::USEMatCorrNONE) { + mCandTr[ord][i] = *mOrigTrPtr[i]; // fetch the track again, as mCandTr might have been propagated w/o errors or material corrections might be wrong + } + auto x = mTrAux[i].c * mPCA[ord][0] + mTrAux[i].s * mPCA[ord][1]; // X of PCA in the track frame + if (!propagateToX(mCandTr[ord][i], x)) { + return false; + } + } + + mTrPropDone[ord] = true; + return true; +} + +//___________________________________________________________________ +template +GPUdi() o2::track::TrackPar DCAFitterN::getTrackParamAtPCA(int i, int icand) +{ + // propagate tracks param only to current vertex (if not already done) + int ord = mOrder[icand]; + o2::track::TrackPar trc(mCandTr[ord][i]); + if (!mTrPropDone[ord]) { + auto x = mTrAux[i].c * mPCA[ord][0] + mTrAux[i].s * mPCA[ord][1]; // X of PCA in the track frame + if (!propagateParamToX(trc, x)) { + trc.invalidate(); + } + } + return trc; +} + +//___________________________________________________________________ +template +GPUdi() double DCAFitterN::getAbsMax(const VecND& v) +{ + double mx = -1; + for (int i = N; i--;) { + auto vai = o2::gpu::GPUCommonMath::Abs(v[i]); + if (mx < vai) { + mx = vai; + } + } + return mx; +} + +//___________________________________________________________________ +template +GPUd() bool DCAFitterN::minimizeChi2() +{ + // find best chi2 (weighted DCA) of N tracks in the vicinity of the seed PCA + for (int i = N; i--;) { + mCandTr[mCurHyp][i] = *mOrigTrPtr[i]; + auto x = mTrAux[i].c * mPCA[mCurHyp][0] + mTrAux[i].s * mPCA[mCurHyp][1]; // X of PCA in the track frame + if (x < mMinXSeed) { + mFitStatus[mCurHyp] = FitStatus::RejTrackX; + return false; + } + if (!propagateToX(mCandTr[mCurHyp][i], x)) { + return false; + } + setTrackPos(mTrPos[mCurHyp][i], mCandTr[mCurHyp][i]); // prepare positions + if (!mTrcEInv[mCurHyp][i].set(mCandTr[mCurHyp][i], XerrFactor)) { // prepare inverse cov.matrices at starting point + if (mLoggerBadCov.needToLog()) { +#ifndef GPUCA_GPUCODE + printf("fitter %d: error (%ld muted): overrode invalid track covariance from %s\n", + mFitterID, mLoggerBadCov.evCount, mCandTr[mCurHyp][i].asString().c_str()); +#else + printf("fitter %d: error (%ld muted): overrode invalid track covariance cyy:%e czz:%e cyz:%e\n", + mFitterID, mLoggerBadCov.evCount, mCandTr[mCurHyp][i].getSigmaY2(), mCandTr[mCurHyp][i].getSigmaZ2(), mCandTr[mCurHyp][i].getSigmaZY()); +#endif + } + mFitStatus[mCurHyp] = FitStatus::FailInvCov; + if (mBadCovPolicy == Discard) { + return false; + } else if (mBadCovPolicy == OverrideAndFlag) { + mPropFailed[mCurHyp] = true; + } // otherwise, just use overridden errors w/o flagging + } + } + + if (mMaxDZIni > 0 && !roughDZCut()) { // apply rough cut on tracks Z difference + mFitStatus[mCurHyp] = FitStatus::RejTrackRoughZ; + return false; + } + + if (!calcPCACoefs()) { // prepare tracks contribution matrices to the global PCA + return false; + } + calcPCA(); // current PCA + calcTrackResiduals(); // current track residuals + float chi2Upd, chi2 = calcChi2(); + do { + calcTrackDerivatives(); // current track derivatives (1st and 2nd) + calcResidDerivatives(); // current residals derivatives (1st and 2nd) + calcChi2Derivatives(); // current chi2 derivatives (1st and 2nd) + + // do Newton-Rapson iteration with corrections = - dchi2/d{x0..xN} * [ d^2chi2/d{x0..xN}^2 ]^-1 + if (!mD2Chi2Dx2.Invert()) { + if (mLoggerBadInv.needToLog()) { + printf("fitter %d: error (%ld muted): Inversion failed\n", mFitterID, mLoggerBadCov.evCount); + } + mFitStatus[mCurHyp] = FitStatus::FailInv2ndDeriv; + return false; + } + VecND dx = mD2Chi2Dx2 * mDChi2Dx; + if (!correctTracks(dx)) { + mFitStatus[mCurHyp] = FitStatus::FailCorrTracks; + return false; + } + calcPCA(); // updated PCA + if (mCrossIDAlt >= 0 && closerToAlternative()) { + mFitStatus[mCurHyp] = FitStatus::FailCloserAlt; + mAllowAltPreference = false; + return false; + } + calcTrackResiduals(); // updated residuals + chi2Upd = calcChi2(); // updated chi2 + if (getAbsMax(dx) < mMinParamChange || chi2Upd > chi2 * mMinRelChi2Change) { + chi2 = chi2Upd; + mFitStatus[mCurHyp] = FitStatus::Converged; + break; // converged + } + chi2 = chi2Upd; + } while (++mNIters[mCurHyp] < mMaxIter); + if (mNIters[mCurHyp] == mMaxIter) { + mFitStatus[mCurHyp] = FitStatus::MaxIter; + } + // + mChi2[mCurHyp] = chi2 * NInv; + if (mChi2[mCurHyp] >= mMaxChi2) { + mFitStatus[mCurHyp] = FitStatus::RejChi2Max; + return false; + } + return true; +} + +//___________________________________________________________________ +template +GPUd() bool DCAFitterN::minimizeChi2NoErr() +{ + // find best chi2 (absolute DCA) of N tracks in the vicinity of the PCA seed + + for (int i = N; i--;) { + mCandTr[mCurHyp][i] = *mOrigTrPtr[i]; + auto x = mTrAux[i].c * mPCA[mCurHyp][0] + mTrAux[i].s * mPCA[mCurHyp][1]; // X of PCA in the track frame + if (x < mMinXSeed) { + mFitStatus[mCurHyp] = FitStatus::RejTrackX; + return false; + } + if (!propagateParamToX(mCandTr[mCurHyp][i], x)) { + return false; + } + setTrackPos(mTrPos[mCurHyp][i], mCandTr[mCurHyp][i]); // prepare positions + } + if (mMaxDZIni > 0 && !roughDZCut()) { // apply rough cut on tracks Z difference + mFitStatus[mCurHyp] = FitStatus::RejTrackRoughZ; + return false; + } + + calcPCANoErr(); // current PCA + calcTrackResiduals(); // current track residuals + float chi2Upd, chi2 = calcChi2NoErr(); + do { + calcTrackDerivatives(); // current track derivatives (1st and 2nd) + calcResidDerivativesNoErr(); // current residals derivatives (1st and 2nd) + calcChi2DerivativesNoErr(); // current chi2 derivatives (1st and 2nd) + + // do Newton-Rapson iteration with corrections = - dchi2/d{x0..xN} * [ d^2chi2/d{x0..xN}^2 ]^-1 + if (!mD2Chi2Dx2.Invert()) { + if (mLoggerBadInv.needToLog()) { + printf("fitter %d: error (%ld muted): Inversion failed\n", mFitterID, mLoggerBadCov.evCount); + } + mFitStatus[mCurHyp] = FitStatus::FailInv2ndDeriv; + return false; + } + VecND dx = mD2Chi2Dx2 * mDChi2Dx; + if (!correctTracks(dx)) { + mFitStatus[mCurHyp] = FitStatus::FailCorrTracks; + return false; + } + calcPCANoErr(); // updated PCA + if (mCrossIDAlt >= 0 && closerToAlternative()) { + mFitStatus[mCurHyp] = FitStatus::FailCloserAlt; + mAllowAltPreference = false; + return false; + } + calcTrackResiduals(); // updated residuals + chi2Upd = calcChi2NoErr(); // updated chi2 + if (getAbsMax(dx) < mMinParamChange || chi2Upd > chi2 * mMinRelChi2Change) { + chi2 = chi2Upd; + mFitStatus[mCurHyp] = FitStatus::Converged; + break; // converged + } + chi2 = chi2Upd; + } while (++mNIters[mCurHyp] < mMaxIter); + if (mNIters[mCurHyp] == mMaxIter) { + mFitStatus[mCurHyp] = FitStatus::MaxIter; + } + // + mChi2[mCurHyp] = chi2 * NInv; + if (mChi2[mCurHyp] >= mMaxChi2) { + mFitStatus[mCurHyp] = FitStatus::RejChi2Max; + return false; + } + return true; +} + +//___________________________________________________________________ +template +GPUd() bool DCAFitterN::roughDZCut() const +{ + // apply rough cut on DZ between the tracks in the seed point + bool accept = true; + for (int i = N; accept && i--;) { + for (int j = i; j--;) { + if (o2::gpu::GPUCommonMath::Abs(mCandTr[mCurHyp][i].getZ() - mCandTr[mCurHyp][j].getZ()) > mMaxDZIni) { + accept = false; + break; + } + } + } + return accept; +} + +//___________________________________________________________________ +template +GPUd() bool DCAFitterN::closerToAlternative() const +{ + // check if the point current PCA point is closer to the seeding XY point being tested or to alternative see (if any) + auto dxCur = mPCA[mCurHyp][0] - mCrossings.xDCA[mCrossIDCur], dyCur = mPCA[mCurHyp][1] - mCrossings.yDCA[mCrossIDCur]; + auto dxAlt = mPCA[mCurHyp][0] - mCrossings.xDCA[mCrossIDAlt], dyAlt = mPCA[mCurHyp][1] - mCrossings.yDCA[mCrossIDAlt]; + return dxCur * dxCur + dyCur * dyCur > dxAlt * dxAlt + dyAlt * dyAlt; +} + +//___________________________________________________________________ +template +GPUd() void DCAFitterN::print() const +{ +#ifndef GPUCA_GPUCODE_DEVICE + LOG(info) << N << "-prong vertex fitter in " << (mUseAbsDCA ? "abs." : "weighted") << " distance minimization mode, collinear tracks mode: " << (mIsCollinear ? "ON" : "OFF"); + LOG(info) << "Bz: " << mBz << " MaxIter: " << mMaxIter << " MaxChi2: " << mMaxChi2 << " MatCorrType: " << int(mMatCorr); + LOG(info) << "Stopping condition: Max.param change < " << mMinParamChange << " Rel.Chi2 change > " << mMinRelChi2Change; + LOG(info) << "Discard candidates for : Rvtx > " << getMaxR() << " DZ between tracks > " << mMaxDZIni; + LOG(info) << "PropagateToPCA:" << mPropagateToPCA << " WeightedFinalPCA:" << mWeightedFinalPCA << " UsePropagator:" << mUsePropagator << " RefitWithMatCorr:" << mRefitWithMatCorr; + std::string rep{}; + for (int i = 0; i < mCrossings.nDCA; i++) { + rep += fmt::format("seed{}:{}/{} ", i, mTrPropDone[i], mPropFailed[i]); + } + LOG(info) << "Last call: NCand:" << mCurHyp << " from " << mCrossings.nDCA << " seeds, prop.done/failed: " << rep; +#else + if (mUseAbsDCA) { + printf("%d-prong vertex fitter in abs. distance minimization mode\n", N); + } else { + printf("%d-prong vertex fitter in weighted distance minimization mode\n", N); + } + printf("Bz: %1.f MaxIter: %3.d MaxChi2: %2.3f\n", mBz, mMaxIter, mMaxChi2); + printf("Stopping condition: Max.param change < %2.3f Rel.Chi2 change > %2.3f\n", mMinParamChange, mMinRelChi2Change); + printf("Discard candidates for : Rvtx > %2.3f DZ between tracks > %2.3f\n", getMaxR(), mMaxDZIni); +#endif +} + +//___________________________________________________________________ +template +GPUd() o2::track::TrackParCov DCAFitterN::createParentTrackParCov(int cand, bool sectorAlpha) const +{ + const auto& trP = getTrack(0, cand); + const auto& trN = getTrack(1, cand); + std::array covV = {0.}; + std::array pvecV = {0.}; + int q = 0; + for (int it = 0; it < N; it++) { + const auto& trc = getTrack(it, cand); + std::array pvecT = {0.}; + std::array covT = {0.}; + trc.getPxPyPzGlo(pvecT); + trc.getCovXYZPxPyPzGlo(covT); + constexpr int MomInd[6] = {9, 13, 14, 18, 19, 20}; // cov matrix elements for momentum component + for (int i = 0; i < 6; i++) { + covV[MomInd[i]] += covT[MomInd[i]]; + } + for (int i = 0; i < 3; i++) { + pvecV[i] += pvecT[i]; + } + q += trc.getCharge(); + } + auto covVtxV = calcPCACovMatrix(cand); + covV[0] = covVtxV(0, 0); + covV[1] = covVtxV(1, 0); + covV[2] = covVtxV(1, 1); + covV[3] = covVtxV(2, 0); + covV[4] = covVtxV(2, 1); + covV[5] = covVtxV(2, 2); + return o2::track::TrackParCov(getPCACandidatePos(cand), pvecV, covV, q, sectorAlpha); +} + +//___________________________________________________________________ +template +GPUd() o2::track::TrackPar DCAFitterN::createParentTrackPar(int cand, bool sectorAlpha) const +{ + const auto& trP = getTrack(0, cand); + const auto& trN = getTrack(1, cand); + const auto& wvtx = getPCACandidate(cand); + std::array pvecV = {0.}; + int q = 0; + for (int it = 0; it < N; it++) { + const auto& trc = getTrack(it, cand); + std::array pvecT = {0.}; + trc.getPxPyPzGlo(pvecT); + for (int i = 0; i < 3; i++) { + pvecV[i] += pvecT[i]; + } + q += trc.getCharge(); + } + const std::array vertex = {(float)wvtx[0], (float)wvtx[1], (float)wvtx[2]}; + return o2::track::TrackPar(vertex, pvecV, q, sectorAlpha); +} + +//___________________________________________________________________ +template +GPUdi() bool DCAFitterN::propagateParamToX(o2::track::TrackPar& t, float x) +{ + bool res = true; + if (mUsePropagator || mMatCorr != o2::base::Propagator::MatCorrType::USEMatCorrNONE) { +#ifndef GPUCA_GPUCODE + res = o2::base::Propagator::Instance()->PropagateToXBxByBz(t, x, mMaxSnp, mMaxStep, mMatCorr); +#endif + } else { + res = t.propagateParamTo(x, mBz); + } + if (!res) { + mFitStatus[mCurHyp] = FitStatus::FailProp; + mPropFailed[mCurHyp] = true; + if (mLoggerBadProp.needToLog()) { +#ifndef GPUCA_GPUCODE + printf("fitter %d: error (%ld muted): propagation failed for %s\n", mFitterID, mLoggerBadProp.evCount, t.asString().c_str()); +#else + printf("fitter %d: error (%ld muted): propagation failed\n", mFitterID, mLoggerBadProp.evCount); +#endif + } + } + return res; +} + +//___________________________________________________________________ +template +GPUdi() bool DCAFitterN::propagateToX(o2::track::TrackParCov& t, float x) +{ + bool res = true; + if (mUsePropagator || mMatCorr != o2::base::Propagator::MatCorrType::USEMatCorrNONE) { +#ifndef GPUCA_GPUCODE + res = o2::base::Propagator::Instance()->PropagateToXBxByBz(t, x, mMaxSnp, mMaxStep, mMatCorr); +#endif + } else { + res = t.propagateTo(x, mBz); + } + if (!res) { + mFitStatus[mCurHyp] = FitStatus::FailProp; + mPropFailed[mCurHyp] = true; + if (mLoggerBadProp.needToLog()) { +#ifndef GPUCA_GPUCODE + printf("fitter %d: error (%ld muted): propagation failed for %s\n", mFitterID, mLoggerBadProp.evCount, t.asString().c_str()); +#else + printf("fitter %d: error (%ld muted): propagation failed\n", mFitterID, mLoggerBadProp.evCount); +#endif + } + } + return res; +} + +using DCAFitter2 = DCAFitterN<2, o2::track::TrackParCov>; +using DCAFitter3 = DCAFitterN<3, o2::track::TrackParCov>; + +namespace device +{ +template +void print(const int nBlocks, const int nThreads, Fitter& ft); + +template +int process(const int nBlocks, const int nThreads, Fitter&, Tr&... args); + +template +void processBulk(const int nBlocks, const int nThreads, const int nBatches, std::vector& fitters, std::vector& results, std::vector&... args); +} // namespace device + +} // namespace vertexing +} // namespace o2 +#endif // _ALICEO2_DCA_FITTERN_ diff --git a/Detectors/Vertexing/include/DetectorsVertexing/FwdDCAFitterN.h b/Common/DCAFitter/include/DCAFitter/FwdDCAFitterN.h similarity index 93% rename from Detectors/Vertexing/include/DetectorsVertexing/FwdDCAFitterN.h rename to Common/DCAFitter/include/DCAFitter/FwdDCAFitterN.h index 8715a2f01d5c7..d5bc6631575af 100644 --- a/Detectors/Vertexing/include/DetectorsVertexing/FwdDCAFitterN.h +++ b/Common/DCAFitter/include/DCAFitter/FwdDCAFitterN.h @@ -20,8 +20,10 @@ #include "MathUtils/Cartesian.h" #include "ReconstructionDataFormats/TrackFwd.h" #include "ReconstructionDataFormats/Track.h" -#include "DetectorsVertexing/HelixHelper.h" +#include "ReconstructionDataFormats/HelixHelper.h" #include +#include "DetectorsBase/Propagator.h" +#include "DetectorsBase/GeometryManager.h" namespace o2 { @@ -158,6 +160,12 @@ class FwdDCAFitterN void setMinParamChange(float x = 1e-3) { mMinParamChange = x > 1e-4 ? x : 1.e-4; } void setMinRelChi2Change(float r = 0.9) { mMinRelChi2Change = r > 0.1 ? r : 999.; } void setUseAbsDCA(bool v) { mUseAbsDCA = v; } + void setMatLUT(const o2::base::MatLayerCylSet* m) + { + mMatLUT = m; + mUseMatBudget = true; + } + void setTGeoMat(bool v = true) { mTGeoFallBackAllowed = v; } void setMaxDistance2ToMerge(float v) { mMaxDist2ToMergeSeeds = v; } int getNCandidates() const { return mCurHyp; } @@ -202,6 +210,7 @@ class FwdDCAFitterN bool roughDXCut() const; bool closerToAlternative() const; static double getAbsMax(const VecND& v); + bool propagateToVtx(o2::track::TrackParCovFwd& t, const std::array& p, const std::array& cov) const; ///< track param positions at V0 candidate (no check for the candidate validity) const Vec3D& getTrackPos(int i, int cand = 0) const { return mTrPos[mOrder[cand]][i]; } @@ -282,6 +291,8 @@ class FwdDCAFitterN bool mAllowAltPreference = true; // if the fit converges to alternative PCA seed, abandon the current one bool mUseAbsDCA = false; // use abs. distance minimization rather than chi2 bool mPropagateToPCA = true; // create tracks version propagated to PCA + bool mUseMatBudget = false; // include MCS effects in track propagation + bool mTGeoFallBackAllowed = true; // use TGeo for precise estimate of mat. budget int mMaxIter = 60; // max number of iterations float mBz = 0; // bz field, to be set by user float mMaxR2 = 200. * 200.; // reject PCA's above this radius @@ -290,6 +301,7 @@ class FwdDCAFitterN float mMinRelChi2Change = 0.98; // stop iterations is chi2/chi2old > this float mMaxChi2 = 100; // abs cut on chi2 or abs distance float mMaxDist2ToMergeSeeds = 1.; // merge 2 seeds to their average if their distance^2 is below the threshold + const o2::base::MatLayerCylSet* mMatLUT = nullptr; // use to compute material budget to include MCS effects ClassDefNV(FwdDCAFitterN, 1); }; @@ -694,14 +706,15 @@ bool FwdDCAFitterN::FwdpropagateTracksToVertex(int icand) return true; } const Vec3D& pca = mPCA[ord]; + std::array covMatrixPCA = calcPCACovMatrixFlat(ord); + std::array cov = {covMatrixPCA[0], covMatrixPCA[2]}; for (int i = N; i--;) { - if (mUseAbsDCA) { - mCandTr[ord][i] = *mOrigTrPtr[i]; // fetch the track again, as mCandTr might have been propagated w/o errors - } + mCandTr[ord][i] = *mOrigTrPtr[i]; // fetch the track again, as mCandTr might have been propagated w/o errors auto& trc = mCandTr[ord][i]; - auto z = pca[2]; - - trc.propagateToZquadratic(z, mBz); + const std::array p = {(float)pca[0], (float)pca[1], (float)pca[2]}; + if (!propagateToVtx(trc, p, cov)) { + return false; + } } mTrPropDone[ord] = true; @@ -719,8 +732,8 @@ float FwdDCAFitterN::findZatXY(int mCurHyp) // Between 2 tracks double z[2] = {startPoint, startPoint}; double newX[2], newY[2]; - double X = mPCA[mCurHyp][0]; //X seed - double Y = mPCA[mCurHyp][1]; //Y seed + double X = mPCA[mCurHyp][0]; // X seed + double Y = mPCA[mCurHyp][1]; // Y seed mCandTr[mCurHyp][0] = *mOrigTrPtr[0]; mCandTr[mCurHyp][1] = *mOrigTrPtr[1]; @@ -780,15 +793,15 @@ void FwdDCAFitterN::findZatXY_mid(int mCurHyp) double epsilon = 0.0001; - double X = mPCA[mCurHyp][0]; //X seed - double Y = mPCA[mCurHyp][1]; //Y seed + double X = mPCA[mCurHyp][0]; // X seed + double Y = mPCA[mCurHyp][1]; // Y seed mCandTr[mCurHyp][0] = *mOrigTrPtr[0]; mCandTr[mCurHyp][1] = *mOrigTrPtr[1]; double finalZ; - double dstXY[2]; //0 -> distance btwn both tracks at startPoint + double dstXY[2]; // 0 -> distance btwn both tracks at startPoint while (DeltaZ > epsilon) { @@ -837,13 +850,13 @@ void FwdDCAFitterN::findZatXY_lineApprox(int mCurHyp) double startPoint = 1.; double endPoint = 50.; // first disk - double X = mPCA[mCurHyp][0]; //X seed - double Y = mPCA[mCurHyp][1]; //Y seed + double X = mPCA[mCurHyp][0]; // X seed + double Y = mPCA[mCurHyp][1]; // Y seed mCandTr[mCurHyp][0] = *mOrigTrPtr[0]; mCandTr[mCurHyp][1] = *mOrigTrPtr[1]; - double y[2][2]; //Y00: y track 0 at point 0; Y01: y track 0 at point 1 + double y[2][2]; // Y00: y track 0 at point 0; Y01: y track 0 at point 1 double z[2][2]; double x[2][2]; @@ -877,7 +890,7 @@ void FwdDCAFitterN::findZatXY_lineApprox(int mCurHyp) aXZ[i] = (x[i][0] - bXZ[i]) / z[i][0]; } - //z seed: equ. for intersection of these lines + // z seed: equ. for intersection of these lines finalZ = 0.5 * ((bYZ[0] - bYZ[1]) / (aYZ[1] - aYZ[0]) + (bXZ[0] - bXZ[1]) / (aXZ[1] - aXZ[0])); mPCA[mCurHyp][2] = finalZ; @@ -890,8 +903,8 @@ void FwdDCAFitterN::findZatXY_quad(int mCurHyp) double startPoint = 0.; double endPoint = 40.; // first disk - double X = mPCA[mCurHyp][0]; //X seed - double Y = mPCA[mCurHyp][1]; //Y seed + double X = mPCA[mCurHyp][0]; // X seed + double Y = mPCA[mCurHyp][1]; // Y seed mCandTr[mCurHyp][0] = *mOrigTrPtr[0]; mCandTr[mCurHyp][1] = *mOrigTrPtr[1]; @@ -920,8 +933,8 @@ void FwdDCAFitterN::findZatXY_quad(int mCurHyp) double finalZ[2]; // find all variables for 2 tracks at z0 = startPoint - //set A, B, C variables for x/y equation for 2 tracks - //calculate Deltax/y for both and roots + // set A, B, C variables for x/y equation for 2 tracks + // calculate Deltax/y for both and roots for (int i = 0; i < 2; i++) { mCandTr[mCurHyp][i].propagateToZquadratic(startPoint, mBz); @@ -955,7 +968,7 @@ void FwdDCAFitterN::findZatXY_quad(int mCurHyp) } else { negX[i] = true; z12X[i] = 0; - } //discard + } // discard if (deltaY[i] > 0) { posY[i] = true; @@ -999,8 +1012,8 @@ void FwdDCAFitterN::findZatXY_linear(int mCurHyp) double startPoint = 0.; - double X = mPCA[mCurHyp][0]; //X seed - double Y = mPCA[mCurHyp][1]; //Y seed + double X = mPCA[mCurHyp][0]; // X seed + double Y = mPCA[mCurHyp][1]; // Y seed mCandTr[mCurHyp][0] = *mOrigTrPtr[0]; mCandTr[mCurHyp][1] = *mOrigTrPtr[1]; @@ -1019,9 +1032,9 @@ void FwdDCAFitterN::findZatXY_linear(int mCurHyp) double finalZ[2]; - //find all variables for 2 tracks at z0 = startPoint - //set A, B variables for x/y equation for 2 tracks - //calculate root + // find all variables for 2 tracks at z0 = startPoint + // set A, B variables for x/y equation for 2 tracks + // calculate root for (int i = 0; i < 2; i++) { mCandTr[mCurHyp][i].propagateToZlinear(startPoint); @@ -1125,7 +1138,7 @@ bool FwdDCAFitterN::minimizeChi2() VecND dz = mD2Chi2Dz2 * mDChi2Dz; - if (!FwdcorrectTracks(dz)) { //calculate new Pi (mTrPos) following Newton-Rapson iteration + if (!FwdcorrectTracks(dz)) { // calculate new Pi (mTrPos) following Newton-Rapson iteration return false; } @@ -1256,6 +1269,25 @@ void FwdDCAFitterN::print() const LOG(info) << "Stopping condition: Max.param change < " << mMinParamChange << " Rel.Chi2 change > " << mMinRelChi2Change; LOG(info) << "Discard candidates for : Rvtx > " << getMaxR() << " DZ between tracks > " << mMaxDXIni; } +//___________________________________________________________________ +template +inline bool FwdDCAFitterN::propagateToVtx(o2::track::TrackParCovFwd& t, const std::array& p, const std::array& cov) const +{ + // propagate track to vertex including MCS effects if material budget included, simple propagation to Z otherwise + float x2x0 = 0; + if (mUseMatBudget) { + auto mb = mMatLUT->getMatBudget(t.getX(), t.getY(), t.getZ(), p[0], p[1], p[2]); + x2x0 = (float)mb.meanX2X0; + return t.propagateToVtxhelixWithMCS(p[2], {p[0], p[1]}, cov, mBz, x2x0); + } else if (mTGeoFallBackAllowed) { + auto geoMan = o2::base::GeometryManager::meanMaterialBudget(t.getX(), t.getY(), t.getZ(), p[0], p[1], p[2]); + x2x0 = (float)geoMan.meanX2X0; + return t.propagateToVtxhelixWithMCS(p[2], {p[0], p[1]}, cov, mBz, x2x0); + } else { + t.propagateToZhelix(p[2], mBz); + return true; + } +} using FwdDCAFitter2 = FwdDCAFitterN<2, o2::track::TrackParCovFwd>; using FwdDCAFitter3 = FwdDCAFitterN<3, o2::track::TrackParCovFwd>; diff --git a/Common/DCAFitter/src/DCAFitterLinkDef.h b/Common/DCAFitter/src/DCAFitterLinkDef.h new file mode 100644 index 0000000000000..6883369c1b9b6 --- /dev/null +++ b/Common/DCAFitter/src/DCAFitterLinkDef.h @@ -0,0 +1,24 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifdef __CLING__ + +#pragma link off all globals; +#pragma link off all classes; +#pragma link off all functions; + +#pragma link C++ class o2::vertexing::DCAFitterN < 2, o2::track::TrackParCov> + ; +#pragma link C++ class o2::vertexing::DCAFitterN < 3, o2::track::TrackParCov> + ; + +#pragma link C++ function o2::vertexing::DCAFitter2::process(const o2::track::TrackParCov&, const o2::track::TrackParCov&); +#pragma link C++ function o2::vertexing::DCAFitter3::process(const o2::track::TrackParCov&, const o2::track::TrackParCov&, const o2::track::TrackParCov&); + +#endif diff --git a/Detectors/Vertexing/src/DCAFitterN.cxx b/Common/DCAFitter/src/DCAFitterN.cxx similarity index 95% rename from Detectors/Vertexing/src/DCAFitterN.cxx rename to Common/DCAFitter/src/DCAFitterN.cxx index e585648418bbc..dbc992bcf687d 100644 --- a/Detectors/Vertexing/src/DCAFitterN.cxx +++ b/Common/DCAFitter/src/DCAFitterN.cxx @@ -13,7 +13,7 @@ /// \brief Defintions for N-prongs secondary vertex fit /// \author ruben.shahoyan@cern.ch -#include "DetectorsVertexing/DCAFitterN.h" +#include "DCAFitter/DCAFitterN.h" namespace o2 { diff --git a/Detectors/Vertexing/src/FwdDCAFitterN.cxx b/Common/DCAFitter/src/FwdDCAFitterN.cxx similarity index 95% rename from Detectors/Vertexing/src/FwdDCAFitterN.cxx rename to Common/DCAFitter/src/FwdDCAFitterN.cxx index f7176aa5039fd..3198c6dfbfd52 100644 --- a/Detectors/Vertexing/src/FwdDCAFitterN.cxx +++ b/Common/DCAFitter/src/FwdDCAFitterN.cxx @@ -13,7 +13,7 @@ /// \brief Defintions for N-prongs secondary vertex fit /// \author ruben.shahoyan@cern.ch, adapted from central barrel to fwd rapidities by Rita Sadek, rita.sadek@cern.ch -#include "DetectorsVertexing/FwdDCAFitterN.h" +#include "DCAFitter/FwdDCAFitterN.h" namespace o2 { diff --git a/Common/DCAFitter/test/testDCAFitterN.cxx b/Common/DCAFitter/test/testDCAFitterN.cxx new file mode 100644 index 0000000000000..bd00b5bed841e --- /dev/null +++ b/Common/DCAFitter/test/testDCAFitterN.cxx @@ -0,0 +1,607 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#define BOOST_TEST_MODULE Test DCAFitterN class +#define BOOST_TEST_MAIN +#define BOOST_TEST_DYN_LINK +#include + +#include "DCAFitter/DCAFitterN.h" +#include "CommonUtils/TreeStreamRedirector.h" +#include +#include +#include +#include +#include +#include + +namespace o2 +{ +namespace vertexing +{ + +using Vec3D = ROOT::Math::SVector; + +template +float checkResults(o2::utils::TreeStreamRedirector& outs, std::string& treeName, FITTER& fitter, + Vec3D& vgen, TLorentzVector& genPar, const std::vector& dtMass) +{ + int nCand = fitter.getNCandidates(); + std::array p; + float distMin = 1e9; + bool absDCA = fitter.getUseAbsDCA(); + bool useWghDCA = fitter.getWeightedFinalPCA(); + for (int ic = 0; ic < nCand; ic++) { + const auto& vtx = fitter.getPCACandidate(ic); + auto df = vgen; + df -= vtx; + + TLorentzVector moth, prong; + for (int i = 0; i < fitter.getNProngs(); i++) { + const auto& trc = fitter.getTrack(i, ic); + trc.getPxPyPzGlo(p); + prong.SetVectM({p[0], p[1], p[2]}, dtMass[i]); + moth += prong; + } + auto nIter = fitter.getNIterations(ic); + auto chi2 = fitter.getChi2AtPCACandidate(ic); + double dst = TMath::Sqrt(df[0] * df[0] + df[1] * df[1] + df[2] * df[2]); + distMin = dst < distMin ? dst : distMin; + auto parentTrack = fitter.createParentTrackParCov(ic); + // float genX + outs << treeName.c_str() << "cand=" << ic << "ncand=" << nCand << "nIter=" << nIter << "chi2=" << chi2 + << "genPart=" << genPar << "recPart=" << moth + << "genX=" << vgen[0] << "genY=" << vgen[1] << "genZ=" << vgen[2] + << "dx=" << df[0] << "dy=" << df[1] << "dz=" << df[2] << "dst=" << dst + << "useAbsDCA=" << absDCA << "useWghDCA=" << useWghDCA << "parent=" << parentTrack; + for (int i = 0; i < fitter.getNProngs(); i++) { + outs << treeName.c_str() << fmt::format("prong{}=", i).c_str() << fitter.getTrack(i, ic); + } + outs << treeName.c_str() << "\n"; + } + return distMin; +} + +TLorentzVector generate(Vec3D& vtx, std::vector& vctr, float bz, + TGenPhaseSpace& genPHS, double parMass, const std::vector& dtMass, std::vector forceQ) +{ + const float errYZ = 1e-2, errSlp = 1e-3, errQPT = 2e-2; + std::array covm = { + errYZ * errYZ, + 0., errYZ * errYZ, + 0, 0., errSlp * errSlp, + 0., 0., 0., errSlp * errSlp, + 0., 0., 0., 0., errQPT * errQPT}; + bool accept = true; + TLorentzVector parent, d0, d1, d2; + do { + accept = true; + double y = gRandom->Rndm() - 0.5; + double pt = 0.1 + gRandom->Rndm() * 3; + double mt = TMath::Sqrt(parMass * parMass + pt * pt); + double pz = mt * TMath::SinH(y); + double phi = gRandom->Rndm() * TMath::Pi() * 2; + double en = mt * TMath::CosH(y); + double rdec = 10.; // radius of the decay + vtx[0] = rdec * TMath::Cos(phi); + vtx[1] = rdec * TMath::Sin(phi); + vtx[2] = rdec * pz / pt; + parent.SetPxPyPzE(pt * TMath::Cos(phi), pt * TMath::Sin(phi), pz, en); + int nd = dtMass.size(); + genPHS.SetDecay(parent, nd, dtMass.data()); + genPHS.Generate(); + vctr.clear(); + float p[4]; + for (int i = 0; i < nd; i++) { + auto* dt = genPHS.GetDecay(i); + if (dt->Pt() < 0.05) { + accept = false; + break; + } + dt->GetXYZT(p); + float s, c, x; + std::array params; + o2::math_utils::sincos(dt->Phi(), s, c); + o2::math_utils::rotateZInv(vtx[0], vtx[1], x, params[0], s, c); + + params[1] = vtx[2]; + params[2] = 0.; // since alpha = phi + params[3] = 1. / TMath::Tan(dt->Theta()); + params[4] = (i % 2 ? -1. : 1.) / dt->Pt(); + covm[14] = errQPT * errQPT * params[4] * params[4]; + // + // randomize + float r1, r2; + gRandom->Rannor(r1, r2); + params[0] += r1 * errYZ; + params[1] += r2 * errYZ; + gRandom->Rannor(r1, r2); + params[2] += r1 * errSlp; + params[3] += r2 * errSlp; + params[4] *= gRandom->Gaus(1., errQPT); + if (forceQ[i] == 0) { + params[4] = 0.; // impose straight track + } + auto& trc = vctr.emplace_back(x, dt->Phi(), params, covm); + float rad = forceQ[i] == 0 ? 600. : TMath::Abs(1. / trc.getCurvature(bz)); + if (!trc.propagateTo(trc.getX() + (gRandom->Rndm() - 0.5) * rad * 0.05, bz) || + !trc.rotate(trc.getAlpha() + (gRandom->Rndm() - 0.5) * 0.2)) { + LOGP(error, "Failed to randomize "); + trc.print(); + } + } + } while (!accept); + + return parent; +} + +static constexpr int NFitStatus{14}; +using FitStatusArray = std::array, NFitStatus>; +static constexpr const char* FitStatusNames[NFitStatus] = { + "None", "Converged", "MaxIter", "NoCrossing", "RejRadius", "RejTrackX", "RejTrackRoughZ", "RejChi2Max", + "FailProp", "FailInvConv", "FailInvWeight", "FailInv2ndDeriv", "FailCorrTracks", "FailCloserAlt"}; +inline void printStat(const FitStatusArray& a) +{ + LOGP(info, "FitStatus summary : ....A / ..AWD / ...WD (A=abs.dist;AWD=abs.wghPCA.dist;WD=wgh.dist)"); + for (int i{0}; i < NFitStatus; ++i) { + LOGP(info, "{:2d}={:20s}: {:5d} / {:5d} / {:5d}", i, FitStatusNames[i], a[i][0], a[i][1], a[i][2]); + } + BOOST_CHECK(a[0][0] == 0); // ensure coverage of all possible states + BOOST_CHECK(a[0][1] == 0); + BOOST_CHECK(a[0][2] == 0); +} + +BOOST_AUTO_TEST_CASE(DCAFitterNProngs) +{ + constexpr int NTest = 10000; + o2::utils::TreeStreamRedirector outStream("dcafitterNTest.root"); + + TGenPhaseSpace genPHS; + constexpr double ele = 0.00051; + constexpr double gamma = 2 * ele + 1e-6; + constexpr double pion = 0.13957; + constexpr double k0 = 0.49761; + constexpr double kch = 0.49368; + constexpr double dch = 1.86965; + std::vector gammadec = {ele, ele}; + std::vector k0dec = {pion, pion}; + std::vector dchdec = {pion, kch, pion}; + std::vector vctracks; + FitStatusArray fitstat; + Vec3D vtxGen; + + double bz = 5.0; + // 2 prongs vertices + { + LOG(info) << "\n\nProcessing 2-prong Helix - Helix case"; + std::vector forceQ{1, 1}; + std::memset(fitstat.data(), 0, sizeof(fitstat)); + + o2::vertexing::DCAFitterN<2> ft; // 2 prong fitter + ft.setBz(bz); + ft.setPropagateToPCA(true); // After finding the vertex, propagate tracks to the DCA. This is default anyway + ft.setMaxR(200); // do not consider V0 seeds with 2D circles crossing above this R. This is default anyway + ft.setMaxDZIni(4); // do not consider V0 seeds with tracks Z-distance exceeding this. This is default anyway + ft.setMaxDXYIni(4); // do not consider V0 seeds with tracks XY-distance exceeding this. This is default anyway + ft.setMinParamChange(1e-3); // stop iterations if max correction is below this value. This is default anyway + ft.setMinRelChi2Change(0.9); // stop iterations if chi2 improves by less that this factor + + std::string treeName2A = "pr2a", treeName2AW = "pr2aw", treeName2W = "pr2w"; + TStopwatch swA, swAW, swW; + int nfoundA = 0, nfoundAW = 0, nfoundW = 0; + double meanDA = 0, meanDAW = 0, meanDW = 0; + swA.Stop(); + swAW.Stop(); + swW.Stop(); + for (int iev = 0; iev < NTest; iev++) { + auto genParent = generate(vtxGen, vctracks, bz, genPHS, k0, k0dec, forceQ); + + ft.setUseAbsDCA(true); + swA.Start(false); + int ncA = ft.process(vctracks[0], vctracks[1]); // HERE WE FIT THE VERTICES + swA.Stop(); + LOG(debug) << "fit abs.dist " << iev << " NC: " << ncA << " Chi2: " << (ncA ? ft.getChi2AtPCACandidate(0) : -1); + if (ncA) { + auto minD = checkResults(outStream, treeName2A, ft, vtxGen, genParent, k0dec); + meanDA += minD; + nfoundA++; + } + ++fitstat[ft.getFitStatus()][0]; + + ft.setUseAbsDCA(true); + ft.setWeightedFinalPCA(true); + swAW.Start(false); + int ncAW = ft.process(vctracks[0], vctracks[1]); // HERE WE FIT THE VERTICES + swAW.Stop(); + LOG(debug) << "fit abs.dist with final weighted DCA " << iev << " NC: " << ncAW << " Chi2: " << (ncAW ? ft.getChi2AtPCACandidate(0) : -1); + if (ncAW) { + auto minD = checkResults(outStream, treeName2AW, ft, vtxGen, genParent, k0dec); + meanDAW += minD; + nfoundAW++; + } + ++fitstat[ft.getFitStatus()][1]; + + ft.setUseAbsDCA(false); + ft.setWeightedFinalPCA(false); + swW.Start(false); + int ncW = ft.process(vctracks[0], vctracks[1]); // HERE WE FIT THE VERTICES + swW.Stop(); + LOG(debug) << "fit wgh.dist " << iev << " NC: " << ncW << " Chi2: " << (ncW ? ft.getChi2AtPCACandidate(0) : -1); + if (ncW) { + auto minD = checkResults(outStream, treeName2W, ft, vtxGen, genParent, k0dec); + meanDW += minD; + nfoundW++; + } + ++fitstat[ft.getFitStatus()][2]; + } + // ft.print(); + meanDA /= nfoundA ? nfoundA : 1; + meanDAW /= nfoundAW ? nfoundAW : 1; + meanDW /= nfoundW ? nfoundW : 1; + LOG(info) << "Processed " << NTest << " 2-prong vertices Helix : Helix"; + LOG(info) << "2-prongs with abs.dist minization: eff= " << float(nfoundA) / NTest + << " mean.dist to truth: " << meanDA << " CPU time: " << swA.CpuTime() * 1000 << " ms"; + LOG(info) << "2-prongs with abs.dist but wghPCA: eff= " << float(nfoundAW) / NTest + << " mean.dist to truth: " << meanDAW << " CPU time: " << swAW.CpuTime() * 1000 << " ms"; + LOG(info) << "2-prongs with wgh.dist minization: eff= " << float(nfoundW) / NTest + << " mean.dist to truth: " << meanDW << " CPU time: " << swW.CpuTime() * 1000 << " ms"; + printStat(fitstat); + BOOST_CHECK(nfoundA > 0.99 * NTest); + BOOST_CHECK(nfoundAW > 0.99 * NTest); + BOOST_CHECK(nfoundW > 0.99 * NTest); + BOOST_CHECK(meanDA < 0.1); + BOOST_CHECK(meanDAW < 0.1); + BOOST_CHECK(meanDW < 0.1); + ft.print(); + } + + // 2 prongs vertices with collinear tracks (gamma conversion) + { + LOG(info) << "\n\nProcessing 2-prong Helix - Helix case gamma conversion"; + std::vector forceQ{1, 1}; + std::memset(fitstat.data(), 0, sizeof(fitstat)); + + o2::vertexing::DCAFitterN<2> ft; // 2 prong fitter + ft.setBz(bz); + ft.setPropagateToPCA(true); // After finding the vertex, propagate tracks to the DCA. This is default anyway + ft.setMaxR(200); // do not consider V0 seeds with 2D circles crossing above this R. This is default anyway + ft.setMaxDZIni(4); // do not consider V0 seeds with tracks Z-distance exceeding this. This is default anyway + ft.setMaxDXYIni(4); // do not consider V0 seeds with tracks XY-distance exceeding this. This is default anyway + ft.setMinParamChange(1e-3); // stop iterations if max correction is below this value. This is default anyway + ft.setMinRelChi2Change(0.9); // stop iterations if chi2 improves by less that this factor + ft.setMaxChi2(); + ft.setCollinear(true); + + std::string treeName2A = "gpr2a", treeName2AW = "gpr2aw", treeName2W = "gpr2w"; + TStopwatch swA, swAW, swW; + int nfoundA = 0, nfoundAW = 0, nfoundW = 0; + double meanDA = 0, meanDAW = 0, meanDW = 0; + swA.Stop(); + swAW.Stop(); + swW.Stop(); + for (int iev = 0; iev < NTest; iev++) { + auto genParent = generate(vtxGen, vctracks, bz, genPHS, gamma, gammadec, forceQ); + + ft.setUseAbsDCA(true); + swA.Start(false); + int ncA = ft.process(vctracks[0], vctracks[1]); // HERE WE FIT THE VERTICES + swA.Stop(); + LOG(debug) << "fit abs.dist " << iev << " NC: " << ncA << " Chi2: " << (ncA ? ft.getChi2AtPCACandidate(0) : -1); + if (ncA) { + auto minD = checkResults(outStream, treeName2A, ft, vtxGen, genParent, gammadec); + meanDA += minD; + nfoundA++; + } + ++fitstat[ft.getFitStatus()][0]; + + ft.setUseAbsDCA(true); + ft.setWeightedFinalPCA(true); + swAW.Start(false); + int ncAW = ft.process(vctracks[0], vctracks[1]); // HERE WE FIT THE VERTICES + swAW.Stop(); + LOG(debug) << "fit abs.dist with final weighted DCA " << iev << " NC: " << ncAW << " Chi2: " << (ncAW ? ft.getChi2AtPCACandidate(0) : -1); + if (ncAW) { + auto minD = checkResults(outStream, treeName2AW, ft, vtxGen, genParent, gammadec); + meanDAW += minD; + nfoundAW++; + } + ++fitstat[ft.getFitStatus()][1]; + + ft.setUseAbsDCA(false); + ft.setWeightedFinalPCA(false); + swW.Start(false); + int ncW = ft.process(vctracks[0], vctracks[1]); // HERE WE FIT THE VERTICES + swW.Stop(); + LOG(debug) << "fit wgh.dist " << iev << " NC: " << ncW << " Chi2: " << (ncW ? ft.getChi2AtPCACandidate(0) : -1); + if (ncW) { + auto minD = checkResults(outStream, treeName2W, ft, vtxGen, genParent, gammadec); + meanDW += minD; + nfoundW++; + } + ++fitstat[ft.getFitStatus()][2]; + } + // ft.print(); + meanDA /= nfoundA ? nfoundA : 1; + meanDAW /= nfoundAW ? nfoundAW : 1; + meanDW /= nfoundW ? nfoundW : 1; + LOG(info) << "Processed " << NTest << " 2-prong vertices Helix : Helix from gamma conversion"; + LOG(info) << "2-prongs with abs.dist minization: eff= " << float(nfoundA) / NTest + << " mean.dist to truth: " << meanDA << " CPU time: " << swA.CpuTime() * 1000 << " ms"; + LOG(info) << "2-prongs with abs.dist but wghPCA: eff= " << float(nfoundAW) / NTest + << " mean.dist to truth: " << meanDAW << " CPU time: " << swAW.CpuTime() * 1000 << " ms"; + LOG(info) << "2-prongs with wgh.dist minization: eff= " << float(nfoundW) / NTest + << " mean.dist to truth: " << meanDW << " CPU time: " << swW.CpuTime() * 1000 << " ms"; + printStat(fitstat); + BOOST_CHECK(nfoundA > 0.99 * NTest); + BOOST_CHECK(nfoundAW > 0.99 * NTest); + BOOST_CHECK(nfoundW > 0.99 * NTest); + BOOST_CHECK(meanDA < 2.1); + BOOST_CHECK(meanDAW < 2.1); + BOOST_CHECK(meanDW < 2.1); + ft.print(); + } + + // 2 prongs vertices with one of charges set to 0: Helix : Line + { + std::vector forceQ{1, 1}; + LOG(info) << "\n\nProcessing 2-prong Helix - Line case"; + std::memset(fitstat.data(), 0, sizeof(fitstat)); + + o2::vertexing::DCAFitterN<2> ft; // 2 prong fitter + ft.setBz(bz); + ft.setPropagateToPCA(true); // After finding the vertex, propagate tracks to the DCA. This is default anyway + ft.setMaxR(200); // do not consider V0 seeds with 2D circles crossing above this R. This is default anyway + ft.setMaxDZIni(4); // do not consider V0 seeds with tracks Z-distance exceeding this. This is default anyway + ft.setMinParamChange(1e-3); // stop iterations if max correction is below this value. This is default anyway + ft.setMinRelChi2Change(0.9); // stop iterations if chi2 improves by less that this factor + + std::string treeName2A = "pr2aHL", treeName2AW = "pr2awHL", treeName2W = "pr2wHL"; + TStopwatch swA, swAW, swW; + int nfoundA = 0, nfoundAW = 0, nfoundW = 0; + double meanDA = 0, meanDAW = 0, meanDW = 0; + swA.Stop(); + swAW.Stop(); + swW.Stop(); + for (int iev = 0; iev < NTest; iev++) { + forceQ[iev % 2] = 1; + forceQ[1 - iev % 2] = 0; + auto genParent = generate(vtxGen, vctracks, bz, genPHS, k0, k0dec, forceQ); + + ft.setUseAbsDCA(true); + swA.Start(false); + int ncA = ft.process(vctracks[0], vctracks[1]); // HERE WE FIT THE VERTICES + swA.Stop(); + LOG(debug) << "fit abs.dist with final weighted DCA " << iev << " NC: " << ncA << " Chi2: " << (ncA ? ft.getChi2AtPCACandidate(0) : -1); + if (ncA) { + auto minD = checkResults(outStream, treeName2A, ft, vtxGen, genParent, k0dec); + meanDA += minD; + nfoundA++; + } + ++fitstat[ft.getFitStatus()][0]; + + ft.setUseAbsDCA(true); + ft.setWeightedFinalPCA(true); + swAW.Start(false); + int ncAW = ft.process(vctracks[0], vctracks[1]); // HERE WE FIT THE VERTICES + swAW.Stop(); + LOG(debug) << "fit abs.dist " << iev << " NC: " << ncAW << " Chi2: " << (ncAW ? ft.getChi2AtPCACandidate(0) : -1); + if (ncAW) { + auto minD = checkResults(outStream, treeName2AW, ft, vtxGen, genParent, k0dec); + meanDAW += minD; + nfoundAW++; + } + ++fitstat[ft.getFitStatus()][1]; + + ft.setUseAbsDCA(false); + ft.setWeightedFinalPCA(false); + swW.Start(false); + int ncW = ft.process(vctracks[0], vctracks[1]); // HERE WE FIT THE VERTICES + swW.Stop(); + LOG(debug) << "fit wgh.dist " << iev << " NC: " << ncW << " Chi2: " << (ncW ? ft.getChi2AtPCACandidate(0) : -1); + if (ncW) { + auto minD = checkResults(outStream, treeName2W, ft, vtxGen, genParent, k0dec); + meanDW += minD; + nfoundW++; + } + ++fitstat[ft.getFitStatus()][2]; + } + // ft.print(); + meanDA /= nfoundA ? nfoundA : 1; + meanDAW /= nfoundAW ? nfoundAW : 1; + meanDW /= nfoundW ? nfoundW : 1; + LOG(info) << "Processed " << NTest << " 2-prong vertices: Helix : Line"; + LOG(info) << "2-prongs with abs.dist minization: eff= " << float(nfoundA) / NTest + << " mean.dist to truth: " << meanDA << " CPU time: " << swA.CpuTime() * 1000 << " ms"; + LOG(info) << "2-prongs with abs.dist but wghPCA: eff= " << float(nfoundAW) / NTest + << " mean.dist to truth: " << meanDAW << " CPU time: " << swAW.CpuTime() * 1000 << " ms"; + LOG(info) << "2-prongs with wgh.dist minization: eff= " << float(nfoundW) / NTest + << " mean.dist to truth: " << meanDW << " CPU time: " << swW.CpuTime() * 1000 << " ms"; + printStat(fitstat); + BOOST_CHECK(nfoundA > 0.99 * NTest); + BOOST_CHECK(nfoundAW > 0.99 * NTest); + BOOST_CHECK(nfoundW > 0.99 * NTest); + BOOST_CHECK(meanDA < 0.1); + BOOST_CHECK(meanDAW < 0.1); + BOOST_CHECK(meanDW < 0.1); + ft.print(); + } + + // 2 prongs vertices with both of charges set to 0: Line : Line + { + std::vector forceQ{0, 0}; + LOG(info) << "\n\nProcessing 2-prong Line - Line case"; + std::memset(fitstat.data(), 0, sizeof(fitstat)); + + o2::vertexing::DCAFitterN<2> ft; // 2 prong fitter + ft.setBz(bz); + ft.setPropagateToPCA(true); // After finding the vertex, propagate tracks to the DCA. This is default anyway + ft.setMaxR(200); // do not consider V0 seeds with 2D circles crossing above this R. This is default anyway + ft.setMaxDZIni(4); // do not consider V0 seeds with tracks Z-distance exceeding this. This is default anyway + ft.setMinParamChange(1e-3); // stop iterations if max correction is below this value. This is default anyway + ft.setMinRelChi2Change(0.9); // stop iterations if chi2 improves by less that this factor + + std::string treeName2A = "pr2aLL", treeName2AW = "pr2awLL", treeName2W = "pr2wLL"; + TStopwatch swA, swAW, swW; + int nfoundA = 0, nfoundAW = 0, nfoundW = 0; + double meanDA = 0, meanDAW = 0, meanDW = 0; + swA.Stop(); + swAW.Stop(); + swW.Stop(); + for (int iev = 0; iev < NTest; iev++) { + forceQ[0] = forceQ[1] = 0; + auto genParent = generate(vtxGen, vctracks, bz, genPHS, k0, k0dec, forceQ); + + ft.setUseAbsDCA(true); + swA.Start(false); + int ncA = ft.process(vctracks[0], vctracks[1]); // HERE WE FIT THE VERTICES + swA.Stop(); + LOG(debug) << "fit abs.dist " << iev << " NC: " << ncA << " Chi2: " << (ncA ? ft.getChi2AtPCACandidate(0) : -1); + if (ncA) { + auto minD = checkResults(outStream, treeName2A, ft, vtxGen, genParent, k0dec); + meanDA += minD; + nfoundA++; + } + ++fitstat[ft.getFitStatus()][0]; + + ft.setUseAbsDCA(true); + ft.setWeightedFinalPCA(true); + swAW.Start(false); + int ncAW = ft.process(vctracks[0], vctracks[1]); // HERE WE FIT THE VERTICES + swAW.Stop(); + LOG(debug) << "fit abs.dist " << iev << " NC: " << ncAW << " Chi2: " << (ncAW ? ft.getChi2AtPCACandidate(0) : -1); + if (ncAW) { + auto minD = checkResults(outStream, treeName2AW, ft, vtxGen, genParent, k0dec); + meanDAW += minD; + nfoundAW++; + } + ++fitstat[ft.getFitStatus()][1]; + + ft.setUseAbsDCA(false); + ft.setWeightedFinalPCA(false); + swW.Start(false); + int ncW = ft.process(vctracks[0], vctracks[1]); // HERE WE FIT THE VERTICES + swW.Stop(); + LOG(debug) << "fit wgh.dist " << iev << " NC: " << ncW << " Chi2: " << (ncW ? ft.getChi2AtPCACandidate(0) : -1); + if (ncW) { + auto minD = checkResults(outStream, treeName2W, ft, vtxGen, genParent, k0dec); + meanDW += minD; + nfoundW++; + } + ++fitstat[ft.getFitStatus()][2]; + } + // ft.print(); + meanDA /= nfoundA ? nfoundA : 1; + meanDAW /= nfoundAW ? nfoundAW : 1; + meanDW /= nfoundW ? nfoundW : 1; + LOG(info) << "Processed " << NTest << " 2-prong vertices: Line : Line"; + LOG(info) << "2-prongs with abs.dist minization: eff= " << float(nfoundA) / NTest + << " mean.dist to truth: " << meanDA << " CPU time: " << swA.CpuTime() * 1000 << " ms"; + LOG(info) << "2-prongs with abs.dist but wghPCA: eff= " << float(nfoundAW) / NTest + << " mean.dist to truth: " << meanDAW << " CPU time: " << swAW.CpuTime() * 1000 << " ms"; + LOG(info) << "2-prongs with wgh.dist minization: eff= " << float(nfoundW) / NTest + << " mean.dist to truth: " << meanDW << " CPU time: " << swW.CpuTime() * 1000 << " ms"; + printStat(fitstat); + BOOST_CHECK(nfoundA > 0.99 * NTest); + BOOST_CHECK(nfoundAW > 0.99 * NTest); + BOOST_CHECK(nfoundW > 0.99 * NTest); + BOOST_CHECK(meanDA < 0.1); + BOOST_CHECK(meanDAW < 0.1); + BOOST_CHECK(meanDW < 0.1); + ft.print(); + } + + // 3 prongs vertices + { + LOG(info) << "\n\nProcessing 3-prong vertices"; + std::vector forceQ{1, 1, 1}; + std::memset(fitstat.data(), 0, sizeof(fitstat)); + + o2::vertexing::DCAFitterN<3> ft; // 3 prong fitter + ft.setBz(bz); + ft.setPropagateToPCA(true); // After finding the vertex, propagate tracks to the DCA. This is default anyway + ft.setMaxR(200); // do not consider V0 seeds with 2D circles crossing above this R. This is default anyway + ft.setMaxDZIni(4); // do not consider V0 seeds with tracks Z-distance exceeding this. This is default anyway + ft.setMinParamChange(1e-3); // stop iterations if max correction is below this value. This is default anyway + ft.setMinRelChi2Change(0.9); // stop iterations if chi2 improves by less that this factor + + std::string treeName3A = "pr3a", treeName3AW = "pr3aw", treeName3W = "pr3w"; + TStopwatch swA, swAW, swW; + int nfoundA = 0, nfoundAW = 0, nfoundW = 0; + double meanDA = 0, meanDAW = 0, meanDW = 0; + swA.Stop(); + swAW.Stop(); + swW.Stop(); + for (int iev = 0; iev < NTest; iev++) { + auto genParent = generate(vtxGen, vctracks, bz, genPHS, dch, dchdec, forceQ); + + ft.setUseAbsDCA(true); + swA.Start(false); + int ncA = ft.process(vctracks[0], vctracks[1], vctracks[2]); // HERE WE FIT THE VERTICES + swA.Stop(); + LOG(debug) << "fit abs.dist " << iev << " NC: " << ncA << " Chi2: " << (ncA ? ft.getChi2AtPCACandidate(0) : -1); + if (ncA) { + auto minD = checkResults(outStream, treeName3A, ft, vtxGen, genParent, dchdec); + meanDA += minD; + nfoundA++; + } + ++fitstat[ft.getFitStatus()][0]; + + ft.setUseAbsDCA(true); + ft.setWeightedFinalPCA(true); + swAW.Start(false); + int ncAW = ft.process(vctracks[0], vctracks[1], vctracks[2]); // HERE WE FIT THE VERTICES + swAW.Stop(); + LOG(debug) << "fit abs.dist " << iev << " NC: " << ncAW << " Chi2: " << (ncAW ? ft.getChi2AtPCACandidate(0) : -1); + if (ncAW) { + auto minD = checkResults(outStream, treeName3AW, ft, vtxGen, genParent, dchdec); + meanDAW += minD; + nfoundAW++; + } + ++fitstat[ft.getFitStatus()][1]; + + ft.setUseAbsDCA(false); + ft.setWeightedFinalPCA(false); + swW.Start(false); + int ncW = ft.process(vctracks[0], vctracks[1], vctracks[2]); // HERE WE FIT THE VERTICES + swW.Stop(); + LOG(debug) << "fit wgh.dist " << iev << " NC: " << ncW << " Chi2: " << (ncW ? ft.getChi2AtPCACandidate(0) : -1); + if (ncW) { + auto minD = checkResults(outStream, treeName3W, ft, vtxGen, genParent, dchdec); + meanDW += minD; + nfoundW++; + } + ++fitstat[ft.getFitStatus()][2]; + } + // ft.print(); + meanDA /= nfoundA ? nfoundA : 1; + meanDAW /= nfoundAW ? nfoundAW : 1; + meanDW /= nfoundW ? nfoundW : 1; + LOG(info) << "Processed " << NTest << " 3-prong vertices"; + LOG(info) << "3-prongs with abs.dist minization: eff= " << float(nfoundA) / NTest + << " mean.dist to truth: " << meanDA << " CPU time: " << swA.CpuTime() * 1000 << " ms"; + LOG(info) << "3-prongs with abs.dist but wghPCA: eff= " << float(nfoundAW) / NTest + << " mean.dist to truth: " << meanDAW << " CPU time: " << swAW.CpuTime() * 1000 << " ms"; + LOG(info) << "3-prongs with wgh.dist minization: eff= " << float(nfoundW) / NTest + << " mean.dist to truth: " << meanDW << " CPU time: " << swW.CpuTime() * 1000 << " ms"; + printStat(fitstat); + BOOST_CHECK(nfoundA > 0.99 * NTest); + BOOST_CHECK(nfoundAW > 0.99 * NTest); + BOOST_CHECK(nfoundW > 0.99 * NTest); + BOOST_CHECK(meanDA < 0.1); + BOOST_CHECK(meanDAW < 0.1); + BOOST_CHECK(meanDW < 0.1); + ft.print(); + } + outStream.Close(); +} + +} // namespace vertexing +} // namespace o2 diff --git a/Common/Field/CMakeLists.txt b/Common/Field/CMakeLists.txt index 5531132eb9a59..fd00d77accfd3 100644 --- a/Common/Field/CMakeLists.txt +++ b/Common/Field/CMakeLists.txt @@ -16,7 +16,8 @@ o2_add_library(Field src/MagFieldParam.cxx src/MagneticField.cxx src/MagneticWrapperChebyshev.cxx - PUBLIC_LINK_LIBRARIES O2::MathUtils) + src/ALICE3MagneticField.cxx + PUBLIC_LINK_LIBRARIES O2::MathUtils FairRoot::Base O2::CommonUtils) o2_target_root_dictionary(Field HEADERS include/Field/MagneticWrapperChebyshev.h @@ -24,7 +25,8 @@ o2_target_root_dictionary(Field include/Field/MagFieldParam.h include/Field/MagFieldContFact.h include/Field/MagFieldFast.h - include/Field/MagFieldFact.h) + include/Field/MagFieldFact.h + include/Field/ALICE3MagneticField.h) o2_add_test(MagneticField SOURCES test/testMagneticField.cxx diff --git a/Common/Field/include/Field/ALICE3MagneticField.h b/Common/Field/include/Field/ALICE3MagneticField.h new file mode 100644 index 0000000000000..703d9a3768901 --- /dev/null +++ b/Common/Field/include/Field/ALICE3MagneticField.h @@ -0,0 +1,83 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file ALICE3MagneticField.h +/// \brief A simple magnetic field class for ALICE3 R&D +/// \author sandro.wenzel@cern.ch + +#ifndef ALICEO2_FIELD_ALICE3MAGNETICFIELD_H_ +#define ALICEO2_FIELD_ALICE3MAGNETICFIELD_H_ + +#include "FairField.h" // for FairField +#include "Rtypes.h" // for ClassDef + +namespace o2 +{ +namespace field +{ + +/// A simple magnetic field class for ALICE3 R&D. Can easily +/// be used in Virtual Monte Carlo simulations. +class ALICE3MagneticField : public FairField +{ + public: + ALICE3MagneticField() : FairField() + { + fType = 2; + init(); + } + + ~ALICE3MagneticField() override = default; + + /// X component, avoid using since slow + Double_t GetBx(Double_t x, Double_t y, Double_t z) override + { + double xyz[3] = {x, y, z}, b[3]; + ALICE3MagneticField::Field(xyz, b); + return b[0]; + } + + /// Y component, avoid using since slow + Double_t GetBy(Double_t x, Double_t y, Double_t z) override + { + double xyz[3] = {x, y, z}, b[3]; + ALICE3MagneticField::Field(xyz, b); + return b[1]; + } + + /// Z component + Double_t GetBz(Double_t x, Double_t y, Double_t z) override + { + double xyz[3] = {x, y, z}, b[3]; + ALICE3MagneticField::Field(xyz, b); + return b[2]; + } + + /// Method to calculate the field at point xyz + /// Main interface from TVirtualMagField used in simulation + void Field(const Double_t* __restrict__ point, Double_t* __restrict__ bField) override; + + private: + // defining a function type, that could be initialized during runtime from a ROOT macro + // to ease fast R&D prototyping + typedef std::function FieldEvalFcn; + FieldEvalFcn mJITFieldFunction; //! + + void init(); + void initJITFieldFunction(); + + // ClassDefOverride(o2::field::ALICE3MagneticField, 1) +}; + +} // end namespace field +} // end namespace o2 + +#endif diff --git a/Common/Field/include/Field/MagFieldFast.h b/Common/Field/include/Field/MagFieldFast.h index acff8f528ad06..ae6431a477923 100644 --- a/Common/Field/include/Field/MagFieldFast.h +++ b/Common/Field/include/Field/MagFieldFast.h @@ -57,7 +57,7 @@ class MagFieldFast bool Field(const math_utils::Point3D xyz, double bxyz[3]) const; bool GetBcomp(EDim comp, const double xyz[3], double& b) const; bool GetBcomp(EDim comp, const float xyz[3], float& b) const; - bool GetBcomp(EDim comp, const math_utils::Point3D xyz, double& b) const; + bool GetBcomp(EDim comp, const math_utils::Point3D xyz, double& b) const; bool GetBcomp(EDim comp, const math_utils::Point3D xyz, float& b) const; bool GetBx(const double xyz[3], double& bx) const { return GetBcomp(kX, xyz, bx); } @@ -66,6 +66,8 @@ class MagFieldFast bool GetBy(const float xyz[3], float& by) const { return GetBcomp(kY, xyz, by); } bool GetBz(const double xyz[3], double& bz) const { return GetBcomp(kZ, xyz, bz); } bool GetBz(const float xyz[3], float& bz) const { return GetBcomp(kZ, xyz, bz); } + bool GetBz(const math_utils::Point3D xyz, double& bz) const { return GetBcomp(kZ, xyz, bz); } + bool GetBz(const math_utils::Point3D xyz, float& bz) const { return GetBcomp(kZ, xyz, bz); } void setFactorSol(float v = 1.f) { mFactorSol = v; } float getFactorSol() const { return mFactorSol; } diff --git a/Common/Field/src/ALICE3MagneticField.cxx b/Common/Field/src/ALICE3MagneticField.cxx new file mode 100644 index 0000000000000..32c69d8131a8e --- /dev/null +++ b/Common/Field/src/ALICE3MagneticField.cxx @@ -0,0 +1,57 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file ALICE3MagneticField.cxx +/// \brief Implementation of the ALICE3 MagF class +/// \author sandro.wenzel@cern.ch + +#include +#include // for FairLogger +#include // for just-in-time compilation of macros +#include + +using namespace o2::field; +// ClassImp(ALICE3MagneticField) + +// initializes the just-in-time implementation of the field function +void ALICE3MagneticField::initJITFieldFunction() +{ + // for now we check if there is an env variable, pointing to a macro file + auto filename = getenv("ALICE3_MAGFIELD_MACRO"); + if (filename) { + LOG(info) << "Taking ALICE3 magnetic field implementation from macro (just in time)"; + if (std::filesystem::exists(filename)) { + // if this file exists we will compile the hook on the fly + mJITFieldFunction = o2::conf::GetFromMacro(filename, "field()", "function", "o2mc_alice3_field_hook"); + LOG(info) << "Hook initialized from file " << filename; + } else { + LOG(error) << "Did not find file " << filename; + } + } +} + +void ALICE3MagneticField::init() +{ + LOG(info) << "Initializing ALICE3 magnetic field"; + initJITFieldFunction(); +} + +void ALICE3MagneticField::Field(const Double_t* __restrict__ xyz, Double_t* __restrict__ b) +{ + if (mJITFieldFunction) { + mJITFieldFunction(xyz, b); + } else { + // TODO: These values are just toy; Real implementation should go here + b[0] = 0.; + b[1] = 0.; + b[2] = -10.; // -10 kGauss + } +} diff --git a/Common/Field/src/FieldLinkDef.h b/Common/Field/src/FieldLinkDef.h index 737a73180af76..cd1b035341284 100644 --- a/Common/Field/src/FieldLinkDef.h +++ b/Common/Field/src/FieldLinkDef.h @@ -20,5 +20,6 @@ #pragma link C++ class o2::field::MagFieldContFact + ; #pragma link C++ class o2::field::MagFieldFact + ; #pragma link C++ class o2::field::MagFieldFast + ; +#pragma link C++ class o2::field::ALICE3MagneticField + ; #endif diff --git a/Common/Field/src/MagFieldContFact.cxx b/Common/Field/src/MagFieldContFact.cxx index 4336ca9d251b4..29d953b71796b 100644 --- a/Common/Field/src/MagFieldContFact.cxx +++ b/Common/Field/src/MagFieldContFact.cxx @@ -14,7 +14,7 @@ /// \author ruben.shahoyan@cern.ch #include // for strcmp, NULL -#include "FairLogger.h" // for FairLogger +#include // for FairLogger #include "FairRuntimeDb.h" // for FairRuntimeD #include "FairParSet.h" #include "Field/MagFieldParam.h" // for FairConstPar diff --git a/Common/Field/src/MagFieldFact.cxx b/Common/Field/src/MagFieldFact.cxx index 2ea25c73acfb5..f71b9cf587d97 100644 --- a/Common/Field/src/MagFieldFact.cxx +++ b/Common/Field/src/MagFieldFact.cxx @@ -11,7 +11,7 @@ #include "Field/MagFieldFact.h" #include "FairField.h" -#include "FairLogger.h" +#include #include "FairRuntimeDb.h" #include "Field/MagFieldParam.h" #include "Field/MagneticField.h" diff --git a/Common/Field/src/MagFieldFast.cxx b/Common/Field/src/MagFieldFast.cxx index e652dffa396c5..02ef9c153d189 100644 --- a/Common/Field/src/MagFieldFast.cxx +++ b/Common/Field/src/MagFieldFast.cxx @@ -145,7 +145,7 @@ bool MagFieldFast::GetBcomp(EDim comp, const double xyz[3], double& b) const } //_______________________________________________________________________ -bool MagFieldFast::GetBcomp(EDim comp, const math_utils::Point3D xyz, double& b) const +bool MagFieldFast::GetBcomp(EDim comp, const math_utils::Point3D xyz, double& b) const { // get field int zSeg, rSeg, quadrant; @@ -238,7 +238,7 @@ bool MagFieldFast::Field(const math_utils::Point3D xyz, double bxyz[3]) bool MagFieldFast::GetSegment(float x, float y, float z, int& zSeg, int& rSeg, int& quadrant) const { // get segment of point location - const float zGridSpaceInv = 1.f / (kSolZMax * 2 / kNSolZRanges); + const float zGridSpaceInv = 1.f / (kSolZMax * 2 / (float)kNSolZRanges); zSeg = -1; if (z < kSolZMax) { if (z > -kSolZMax) { diff --git a/Common/Field/src/MagneticField.cxx b/Common/Field/src/MagneticField.cxx index 536112a128f64..5df6bbc0b0d34 100644 --- a/Common/Field/src/MagneticField.cxx +++ b/Common/Field/src/MagneticField.cxx @@ -17,7 +17,7 @@ #include // for TFile #include // for TPRegexp #include // for TSystem, gSystem -#include "FairLogger.h" // for FairLogger +#include // for FairLogger #include "FairParamList.h" #include "FairRun.h" #include "FairRuntimeDb.h" @@ -539,7 +539,7 @@ void MagneticField::rescaleField(float l3Cur, float diCur, bool uniform, int con float sclL3 = l3Cur, sclDip = diCur; MagFieldParam::BMap_t map = getFieldMapScale(sclL3, sclDip, uniform); if (map != mMapType) { - LOGP(fatal, "Provided L3current={} DipCurrent={} uniform={} leads to map type {}, incompatible with loaded {}", l3Cur, diCur, uniform, map, mMapType); + LOGP(fatal, "Provided L3current={} DipCurrent={} uniform={} leads to map type {}, incompatible with loaded {}", l3Cur, diCur, uniform, (int)map, (int)mMapType); } setFactorSolenoid(sclL3); setFactorDipole(sclDip); @@ -558,18 +558,29 @@ MagFieldParam::BMap_t MagneticField::getFieldMapScale(float& l3, float& dip, boo MagFieldParam::BMap_t map = MagFieldParam::k5kG; float sclL3, sclDip; + float l3sav = l3, dipsav = dip; float l3Pol = l3 > 0 ? 1 : -1; float diPol = dip > 0 ? 1 : -1; l3 = TMath::Abs(l3); dip = TMath::Abs(dip); + static bool overrideL3 = std::getenv("O2_OVERRIDE_L3_CURRENT") != nullptr; + static bool overrideDIP = std::getenv("O2_OVERRIDE_DIPOLE_CURRENT") != nullptr; + static bool warnL3Done = false, warnDipDone = false, warnPolarityDone = false; if (TMath::Abs((sclDip = dip / diNominalCurrent) - 1.) > tolerance && !uniform) { if (dip <= zero) { sclDip = 0.; // some small current.. -> Dipole OFF } else { - LOG(fatal) << "MagneticField::createFieldMap: Wrong dipole current (" << dip << " A)!"; + if (!overrideDIP) { + LOG(fatal) << "MagneticField::createFieldMap: Wrong dipole current (" << dipsav << " A)!"; + } else { + if (!warnDipDone) { + LOGP(error, "Dipole current was overridden to unsupported value {}", dipsav); + warnDipDone = true; + } + } } } if (uniform) { @@ -587,15 +598,33 @@ MagFieldParam::BMap_t MagneticField::getFieldMapScale(float& l3, float& dip, boo sclDip = 0; map = MagFieldParam::k5kGUniform; } else { - LOG(fatal) << "MagneticField::createFieldMap: Wrong L3 current (" << l3 << " A)!"; + if (!overrideL3) { + LOG(fatal) << "MagneticField::createFieldMap: Wrong L3 current (" << l3sav << " A)!"; + } else { + if (!warnL3Done) { + LOGP(error, "L3 current was overridden to unsupported value {}", l3sav); + warnL3Done = true; + } + map = MagFieldParam::k5kG; + sclL3 = l3 / l3NominalCurrent1; + } } } if (sclDip != 0 && map != MagFieldParam::k5kGUniform) { if ((l3 <= zero) || ((convention == kConvLHC && l3Pol != diPol) || (convention == kConvDCS2008 && l3Pol == diPol))) { - LOG(fatal) << "MagneticField::createFieldMap: Wrong combination for L3/Dipole polarities (" - << (l3Pol > 0 ? '+' : '-') << "/" << (diPol > 0 ? '+' : '-') << ") for convention " - << getPolarityConvention(); + if (overrideL3 || overrideDIP) { + if (!warnPolarityDone) { + LOG(error) << "Overriden currents have wrong combination for L3/Dipole polarities (" + << (l3Pol > 0 ? '+' : '-') << "/" << (diPol > 0 ? '+' : '-') << ") for convention " + << getPolarityConvention(); + warnPolarityDone = true; + } + } else { + LOG(fatal) << "MagneticField::createFieldMap: Wrong combination for L3/Dipole polarities (" + << (l3Pol > 0 ? '+' : '-') << "/" << (diPol > 0 ? '+' : '-') << ") for convention " + << getPolarityConvention(); + } } } l3 = (l3Pol < 0) ? -sclL3 : sclL3; diff --git a/Common/Field/src/MagneticWrapperChebyshev.cxx b/Common/Field/src/MagneticWrapperChebyshev.cxx index 566fd3df2eb7f..98f1e191129b4 100644 --- a/Common/Field/src/MagneticWrapperChebyshev.cxx +++ b/Common/Field/src/MagneticWrapperChebyshev.cxx @@ -19,7 +19,7 @@ #include // for TSystem, gSystem #include // for printf, fprintf, fclose, fopen, FILE #include // for memcpy -#include "FairLogger.h" // for FairLogger +#include // for FairLogger #include "TMath.h" // for BinarySearch, Sort #include "TMathBase.h" // for Abs #include "TNamed.h" // for TNamed @@ -1206,7 +1206,7 @@ void MagneticWrapperChebyshev::buildTable(Int_t npar, TObjArray* parArr, Int_t& nSegYDipArr.Set(nZSeg); float xyz[3]; for (int iz = 0; iz < nZSeg; iz++) { - printf("\nZSegment#%d %+e : %+e\n", iz, tmpSegZ[iz], tmpSegZ[iz + 1]); + LOGF(debug, "\nZSegment#%d %+e : %+e\n", iz, tmpSegZ[iz], tmpSegZ[iz + 1]); int ny = segmentDimension(&tmpSegY, parArr, npar, 1, 1, -1, 1, -1, tmpSegZ[iz], tmpSegZ[iz + 1]) - 1; segYArr.Set(ny + nYSeg); for (int iy = 0; iy < ny; iy++) { @@ -1214,7 +1214,7 @@ void MagneticWrapperChebyshev::buildTable(Int_t npar, TObjArray* parArr, Int_t& } begSegYDipArr[iz] = nYSeg; nSegYDipArr[iz] = ny; - printf(" Found %d YSegments, to start from %d\n", ny, begSegYDipArr[iz]); + LOGF(debug, " Found %d YSegments, to start from %d\n", ny, begSegYDipArr[iz]); // for each slice in Z and Y create segmentation in X begSegXDipArr.Set(nYSeg + ny); @@ -1223,7 +1223,7 @@ void MagneticWrapperChebyshev::buildTable(Int_t npar, TObjArray* parArr, Int_t& for (int iy = 0; iy < ny; iy++) { int isg = nYSeg + iy; - printf("\n YSegment#%d %+e : %+e\n", iy, tmpSegY[iy], tmpSegY[iy + 1]); + LOGF(debug, "\n YSegment#%d %+e : %+e\n", iy, tmpSegY[iy], tmpSegY[iy + 1]); int nx = segmentDimension(&tmpSegX, parArr, npar, 0, 1, -1, tmpSegY[iy], tmpSegY[iy + 1], tmpSegZ[iz], tmpSegZ[iz + 1]) - 1; @@ -1234,7 +1234,7 @@ void MagneticWrapperChebyshev::buildTable(Int_t npar, TObjArray* parArr, Int_t& } begSegXDipArr[isg] = nXSeg; nSegXDipArr[isg] = nx; - printf(" Found %d XSegments, to start from %d\n", nx, begSegXDipArr[isg]); + LOGF(debug, " Found %d XSegments, to start from %d\n", nx, begSegXDipArr[isg]); segIDArr.Set(nXSeg + nx); diff --git a/Common/Field/test/testMagneticField.cxx b/Common/Field/test/testMagneticField.cxx index fc4ac11f91c8c..9fa8c92260458 100644 --- a/Common/Field/test/testMagneticField.cxx +++ b/Common/Field/test/testMagneticField.cxx @@ -17,7 +17,7 @@ #include "Field/MagneticField.h" #include "Field/MagFieldFast.h" #include -#include "FairLogger.h" // for FairLogger +#include // for FairLogger #include #include diff --git a/Common/ML/CMakeLists.txt b/Common/ML/CMakeLists.txt new file mode 100644 index 0000000000000..0ed52e1a23e20 --- /dev/null +++ b/Common/ML/CMakeLists.txt @@ -0,0 +1,22 @@ +# Copyright 2019-2020 CERN and copyright holders of ALICE O2. +# See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +# All rights not expressly granted are reserved. +# +# This software is distributed under the terms of the GNU General Public +# License v3 (GPL Version 3), copied verbatim in the file "COPYING". +# +# In applying this license CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization +# or submit itself to any jurisdiction. + +o2_add_library(ML + SOURCES src/OrtInterface.cxx + TARGETVARNAME targetName + PRIVATE_LINK_LIBRARIES O2::GPUCommon onnxruntime::onnxruntime) + +# Pass ORT variables as a preprocessor definition +target_compile_definitions(${targetName} PRIVATE + $<$:ORT_ROCM_BUILD> + $<$:ORT_CUDA_BUILD> + $<$:ORT_MIGRAPHX_BUILD> + $<$:ORT_TENSORRT_BUILD>) diff --git a/Common/ML/include/ML/3rdparty/GPUORTFloat16.h b/Common/ML/include/ML/3rdparty/GPUORTFloat16.h new file mode 100644 index 0000000000000..75e146d872cd1 --- /dev/null +++ b/Common/ML/include/ML/3rdparty/GPUORTFloat16.h @@ -0,0 +1,887 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// This code was created from: +// - https://github.com/microsoft/onnxruntime/blob/main/include/onnxruntime/core/session/onnxruntime_float16.h +// - https://github.com/microsoft/onnxruntime/blob/main/include/onnxruntime/core/session/onnxruntime_cxx_api.h + +#ifndef GPUORTFLOAT16_H +#define GPUORTFLOAT16_H + +#ifndef GPUCA_GPUCODE_DEVICE +#include +#include +#include +#include +#endif + +#include "GPUCommonDef.h" +#include "GPUCommonMath.h" + +namespace o2 +{ + +namespace OrtDataType +{ + +namespace detail +{ + +enum class endian { +#if defined(_WIN32) + little = 0, + big = 1, + native = little, +#elif defined(__GNUC__) || defined(__clang__) + little = __ORDER_LITTLE_ENDIAN__, + big = __ORDER_BIG_ENDIAN__, + native = __BYTE_ORDER__, +#else +#error OrtDataType::detail::endian is not implemented in this environment. +#endif +}; + +static_assert( + endian::native == endian::little || endian::native == endian::big, + "Only little-endian or big-endian native byte orders are supported."); + +} // namespace detail + +/// +/// Shared implementation between public and internal classes. CRTP pattern. +/// +template +struct Float16Impl { + protected: + /// + /// Converts from float to uint16_t float16 representation + /// + /// + /// + GPUd() constexpr static uint16_t ToUint16Impl(float v) noexcept; + + /// + /// Converts float16 to float + /// + /// float representation of float16 value + GPUd() float ToFloatImpl() const noexcept; + + /// + /// Creates an instance that represents absolute value. + /// + /// Absolute value + GPUd() uint16_t AbsImpl() const noexcept + { + return static_cast(val & ~kSignMask); + } + + /// + /// Creates a new instance with the sign flipped. + /// + /// Flipped sign instance + GPUd() uint16_t NegateImpl() const noexcept + { + return IsNaN() ? val : static_cast(val ^ kSignMask); + } + + public: + // uint16_t special values + static constexpr uint16_t kSignMask = 0x8000U; + static constexpr uint16_t kBiasedExponentMask = 0x7C00U; + static constexpr uint16_t kPositiveInfinityBits = 0x7C00U; + static constexpr uint16_t kNegativeInfinityBits = 0xFC00U; + static constexpr uint16_t kPositiveQNaNBits = 0x7E00U; + static constexpr uint16_t kNegativeQNaNBits = 0xFE00U; + static constexpr uint16_t kEpsilonBits = 0x4170U; + static constexpr uint16_t kMinValueBits = 0xFBFFU; // Minimum normal number + static constexpr uint16_t kMaxValueBits = 0x7BFFU; // Largest normal number + static constexpr uint16_t kOneBits = 0x3C00U; + static constexpr uint16_t kMinusOneBits = 0xBC00U; + + uint16_t val{0}; + + GPUdDefault() Float16Impl() = default; + + /// + /// Checks if the value is negative + /// + /// true if negative + GPUd() bool IsNegative() const noexcept + { + return static_cast(val) < 0; + } + + /// + /// Tests if the value is NaN + /// + /// true if NaN + GPUd() bool IsNaN() const noexcept + { + return AbsImpl() > kPositiveInfinityBits; + } + + /// + /// Tests if the value is finite + /// + /// true if finite + GPUd() bool IsFinite() const noexcept + { + return AbsImpl() < kPositiveInfinityBits; + } + + /// + /// Tests if the value represents positive infinity. + /// + /// true if positive infinity + GPUd() bool IsPositiveInfinity() const noexcept + { + return val == kPositiveInfinityBits; + } + + /// + /// Tests if the value represents negative infinity + /// + /// true if negative infinity + GPUd() bool IsNegativeInfinity() const noexcept + { + return val == kNegativeInfinityBits; + } + + /// + /// Tests if the value is either positive or negative infinity. + /// + /// True if absolute value is infinity + GPUd() bool IsInfinity() const noexcept + { + return AbsImpl() == kPositiveInfinityBits; + } + + /// + /// Tests if the value is NaN or zero. Useful for comparisons. + /// + /// True if NaN or zero. + GPUd() bool IsNaNOrZero() const noexcept + { + auto abs = AbsImpl(); + return (abs == 0 || abs > kPositiveInfinityBits); + } + + /// + /// Tests if the value is normal (not zero, subnormal, infinite, or NaN). + /// + /// True if so + GPUd() bool IsNormal() const noexcept + { + auto abs = AbsImpl(); + return (abs < kPositiveInfinityBits) // is finite + && (abs != 0) // is not zero + && ((abs & kBiasedExponentMask) != 0); // is not subnormal (has a non-zero exponent) + } + + /// + /// Tests if the value is subnormal (denormal). + /// + /// True if so + GPUd() bool IsSubnormal() const noexcept + { + auto abs = AbsImpl(); + return (abs < kPositiveInfinityBits) // is finite + && (abs != 0) // is not zero + && ((abs & kBiasedExponentMask) == 0); // is subnormal (has a zero exponent) + } + + /// + /// Creates an instance that represents absolute value. + /// + /// Absolute value + GPUd() Derived Abs() const noexcept { return Derived::FromBits(AbsImpl()); } + + /// + /// Creates a new instance with the sign flipped. + /// + /// Flipped sign instance + GPUd() Derived Negate() const noexcept { return Derived::FromBits(NegateImpl()); } + + /// + /// IEEE defines that positive and negative zero are equal, this gives us a quick equality check + /// for two values by or'ing the private bits together and stripping the sign. They are both zero, + /// and therefore equivalent, if the resulting value is still zero. + /// + /// first value + /// second value + /// True if both arguments represent zero + GPUd() static bool AreZero(const Float16Impl& lhs, const Float16Impl& rhs) noexcept + { + return static_cast((lhs.val | rhs.val) & ~kSignMask) == 0; + } + + GPUd() bool operator==(const Float16Impl& rhs) const noexcept + { + if (IsNaN() || rhs.IsNaN()) { + // IEEE defines that NaN is not equal to anything, including itself. + return false; + } + return val == rhs.val; + } + + GPUd() bool operator!=(const Float16Impl& rhs) const noexcept { return !(*this == rhs); } + + GPUd() bool operator<(const Float16Impl& rhs) const noexcept + { + if (IsNaN() || rhs.IsNaN()) { + // IEEE defines that NaN is unordered with respect to everything, including itself. + return false; + } + + const bool left_is_negative = IsNegative(); + if (left_is_negative != rhs.IsNegative()) { + // When the signs of left and right differ, we know that left is less than right if it is + // the negative value. The exception to this is if both values are zero, in which case IEEE + // says they should be equal, even if the signs differ. + return left_is_negative && !AreZero(*this, rhs); + } + return (val != rhs.val) && ((val < rhs.val) ^ left_is_negative); + } +}; + +// The following Float16_t conversions are based on the code from +// Eigen library. + +// The conversion routines are Copyright (c) Fabian Giesen, 2016. +// The original license follows: +// +// Copyright (c) Fabian Giesen, 2016 +// All rights reserved. +// Redistribution and use in source and binary forms, with or without +// modification, are permitted. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +namespace detail +{ +union float32_bits { + unsigned int u; + float f; +}; +}; // namespace detail + +template +GPUdi() constexpr uint16_t Float16Impl::ToUint16Impl(float v) noexcept +{ + detail::float32_bits f{}; + f.f = v; + + constexpr detail::float32_bits f32infty = {255 << 23}; + constexpr detail::float32_bits f16max = {(127 + 16) << 23}; + constexpr detail::float32_bits denorm_magic = {((127 - 15) + (23 - 10) + 1) << 23}; + constexpr unsigned int sign_mask = 0x80000000u; + uint16_t val = static_cast(0x0u); + + unsigned int sign = f.u & sign_mask; + f.u ^= sign; + + // NOTE all the integer compares in this function can be safely + // compiled into signed compares since all operands are below + // 0x80000000. Important if you want fast straight SSE2 code + // (since there's no unsigned PCMPGTD). + + if (f.u >= f16max.u) { // result is Inf or NaN (all exponent bits set) + val = (f.u > f32infty.u) ? 0x7e00 : 0x7c00; // NaN->qNaN and Inf->Inf + } else { // (De)normalized number or zero + if (f.u < (113 << 23)) { // resulting FP16 is subnormal or zero + // use a magic value to align our 10 mantissa bits at the bottom of + // the float. as long as FP addition is round-to-nearest-even this + // just works. + f.f += denorm_magic.f; + + // and one integer subtract of the bias later, we have our final float! + val = static_cast(f.u - denorm_magic.u); + } else { + unsigned int mant_odd = (f.u >> 13) & 1; // resulting mantissa is odd + + // update exponent, rounding bias part 1 + // Equivalent to `f.u += ((unsigned int)(15 - 127) << 23) + 0xfff`, but + // without arithmetic overflow. + f.u += 0xc8000fffU; + // rounding bias part 2 + f.u += mant_odd; + // take the bits! + val = static_cast(f.u >> 13); + } + } + + val |= static_cast(sign >> 16); + return val; +} + +template +GPUdi() float Float16Impl::ToFloatImpl() const noexcept +{ + constexpr detail::float32_bits magic = {113 << 23}; + constexpr unsigned int shifted_exp = 0x7c00 << 13; // exponent mask after shift + detail::float32_bits o{}; + + o.u = (val & 0x7fff) << 13; // exponent/mantissa bits + unsigned int exp = shifted_exp & o.u; // just the exponent + o.u += (127 - 15) << 23; // exponent adjust + + // handle exponent special cases + if (exp == shifted_exp) { // Inf/NaN? + o.u += (128 - 16) << 23; // extra exp adjust + } else if (exp == 0) { // Zero/Denormal? + o.u += 1 << 23; // extra exp adjust + o.f -= magic.f; // re-normalize + } + + // Attempt to workaround the Internal Compiler Error on ARM64 + // for bitwise | operator, including std::bitset +#if (defined _MSC_VER) && (defined _M_ARM || defined _M_ARM64 || defined _M_ARM64EC) + if (IsNegative()) { + return -o.f; + } +#else + // original code: + o.u |= (val & 0x8000U) << 16U; // sign bit +#endif + return o.f; +} + +/// Shared implementation between public and internal classes. CRTP pattern. +template +struct BFloat16Impl { + protected: + /// + /// Converts from float to uint16_t float16 representation + /// + /// + /// + GPUd() static uint16_t ToUint16Impl(float v) noexcept; + + /// + /// Converts bfloat16 to float + /// + /// float representation of bfloat16 value + GPUd() float ToFloatImpl() const noexcept; + + /// + /// Creates an instance that represents absolute value. + /// + /// Absolute value + GPUd() uint16_t AbsImpl() const noexcept + { + return static_cast(val & ~kSignMask); + } + + /// + /// Creates a new instance with the sign flipped. + /// + /// Flipped sign instance + GPUd() uint16_t NegateImpl() const noexcept + { + return IsNaN() ? val : static_cast(val ^ kSignMask); + } + + public: + // uint16_t special values + static constexpr uint16_t kSignMask = 0x8000U; + static constexpr uint16_t kBiasedExponentMask = 0x7F80U; + static constexpr uint16_t kPositiveInfinityBits = 0x7F80U; + static constexpr uint16_t kNegativeInfinityBits = 0xFF80U; + static constexpr uint16_t kPositiveQNaNBits = 0x7FC1U; + static constexpr uint16_t kNegativeQNaNBits = 0xFFC1U; + static constexpr uint16_t kSignaling_NaNBits = 0x7F80U; + static constexpr uint16_t kEpsilonBits = 0x0080U; + static constexpr uint16_t kMinValueBits = 0xFF7FU; + static constexpr uint16_t kMaxValueBits = 0x7F7FU; + static constexpr uint16_t kRoundToNearest = 0x7FFFU; + static constexpr uint16_t kOneBits = 0x3F80U; + static constexpr uint16_t kMinusOneBits = 0xBF80U; + + uint16_t val{0}; + + GPUdDefault() BFloat16Impl() = default; + + /// + /// Checks if the value is negative + /// + /// true if negative + GPUd() bool IsNegative() const noexcept + { + return static_cast(val) < 0; + } + + /// + /// Tests if the value is NaN + /// + /// true if NaN + GPUd() bool IsNaN() const noexcept + { + return AbsImpl() > kPositiveInfinityBits; + } + + /// + /// Tests if the value is finite + /// + /// true if finite + GPUd() bool IsFinite() const noexcept + { + return AbsImpl() < kPositiveInfinityBits; + } + + /// + /// Tests if the value represents positive infinity. + /// + /// true if positive infinity + GPUd() bool IsPositiveInfinity() const noexcept + { + return val == kPositiveInfinityBits; + } + + /// + /// Tests if the value represents negative infinity + /// + /// true if negative infinity + GPUd() bool IsNegativeInfinity() const noexcept + { + return val == kNegativeInfinityBits; + } + + /// + /// Tests if the value is either positive or negative infinity. + /// + /// True if absolute value is infinity + GPUd() bool IsInfinity() const noexcept + { + return AbsImpl() == kPositiveInfinityBits; + } + + /// + /// Tests if the value is NaN or zero. Useful for comparisons. + /// + /// True if NaN or zero. + GPUd() bool IsNaNOrZero() const noexcept + { + auto abs = AbsImpl(); + return (abs == 0 || abs > kPositiveInfinityBits); + } + + /// + /// Tests if the value is normal (not zero, subnormal, infinite, or NaN). + /// + /// True if so + GPUd() bool IsNormal() const noexcept + { + auto abs = AbsImpl(); + return (abs < kPositiveInfinityBits) // is finite + && (abs != 0) // is not zero + && ((abs & kBiasedExponentMask) != 0); // is not subnormal (has a non-zero exponent) + } + + /// + /// Tests if the value is subnormal (denormal). + /// + /// True if so + GPUd() bool IsSubnormal() const noexcept + { + auto abs = AbsImpl(); + return (abs < kPositiveInfinityBits) // is finite + && (abs != 0) // is not zero + && ((abs & kBiasedExponentMask) == 0); // is subnormal (has a zero exponent) + } + + /// + /// Creates an instance that represents absolute value. + /// + /// Absolute value + GPUd() Derived Abs() const noexcept { return Derived::FromBits(AbsImpl()); } + + /// + /// Creates a new instance with the sign flipped. + /// + /// Flipped sign instance + GPUd() Derived Negate() const noexcept { return Derived::FromBits(NegateImpl()); } + + /// + /// IEEE defines that positive and negative zero are equal, this gives us a quick equality check + /// for two values by or'ing the private bits together and stripping the sign. They are both zero, + /// and therefore equivalent, if the resulting value is still zero. + /// + /// first value + /// second value + /// True if both arguments represent zero + GPUd() static bool AreZero(const BFloat16Impl& lhs, const BFloat16Impl& rhs) noexcept + { + // IEEE defines that positive and negative zero are equal, this gives us a quick equality check + // for two values by or'ing the private bits together and stripping the sign. They are both zero, + // and therefore equivalent, if the resulting value is still zero. + return static_cast((lhs.val | rhs.val) & ~kSignMask) == 0; + } +}; + +template +GPUdi() uint16_t BFloat16Impl::ToUint16Impl(float v) noexcept +{ + uint16_t result; + if (o2::gpu::CAMath::IsNaN(v)) { + result = kPositiveQNaNBits; + } else { + auto get_msb_half = [](float fl) { + uint16_t res; +#ifdef GPUCA_GPUCODE + o2::gpu::CAMath::memcpy(&res, reinterpret_cast(&fl) + sizeof(uint16_t), sizeof(uint16_t)); +#else +#ifdef __cpp_if_constexpr + if constexpr (detail::endian::native == detail::endian::little) +#else + if (detail::endian::native == detail::endian::little) +#endif + { + std::memcpy(&res, reinterpret_cast(&fl) + sizeof(uint16_t), sizeof(uint16_t)); + } else { + std::memcpy(&res, &fl, sizeof(uint16_t)); + } +#endif + return res; + }; + + uint16_t upper_bits = get_msb_half(v); + union { + uint32_t U32; + float F32; + }; + F32 = v; + U32 += (upper_bits & 1) + kRoundToNearest; + result = get_msb_half(F32); + } + return result; +} + +template +GPUdi() float BFloat16Impl::ToFloatImpl() const noexcept +{ +#ifndef __FAST_MATH__ + if (IsNaN()) { + return o2::gpu::CAMath::QuietNaN(); + } +#endif + float result; + char* const first = reinterpret_cast(&result); + char* const second = first + sizeof(uint16_t); +#ifdef GPUCA_GPUCODE + first[0] = first[1] = 0; + o2::gpu::CAMath::memcpy(second, &val, sizeof(uint16_t)); +#else +#ifdef __cpp_if_constexpr + if constexpr (detail::endian::native == detail::endian::little) +#else + if (detail::endian::native == detail::endian::little) +#endif + { + std::memset(first, 0, sizeof(uint16_t)); + std::memcpy(second, &val, sizeof(uint16_t)); + } else { + std::memcpy(first, &val, sizeof(uint16_t)); + std::memset(second, 0, sizeof(uint16_t)); + } +#endif + return result; +} + +/** \brief IEEE 754 half-precision floating point data type + * + * \details This struct is used for converting float to float16 and back + * so the user could feed inputs and fetch outputs using these type. + * + * The size of the structure should align with uint16_t and one can freely cast + * uint16_t buffers to/from Ort::Float16_t to feed and retrieve data. + * + * \code{.unparsed} + * // This example demonstrates converion from float to float16 + * constexpr float values[] = {1.f, 2.f, 3.f, 4.f, 5.f}; + * std::vector fp16_values; + * fp16_values.reserve(std::size(values)); + * std::transform(std::begin(values), std::end(values), std::back_inserter(fp16_values), + * [](float value) { return Ort::Float16_t(value); }); + * + * \endcode + */ +struct Float16_t : OrtDataType::Float16Impl { + private: + /// + /// Constructor from a 16-bit representation of a float16 value + /// No conversion is done here. + /// + /// 16-bit representation + constexpr explicit Float16_t(uint16_t v) noexcept { val = v; } + + public: + using Base = OrtDataType::Float16Impl; + + /// + /// Default constructor + /// + GPUdDefault() Float16_t() = default; + + /// + /// Explicit conversion to uint16_t representation of float16. + /// + /// uint16_t bit representation of float16 + /// new instance of Float16_t + GPUd() constexpr static Float16_t FromBits(uint16_t v) noexcept { return Float16_t(v); } + + /// + /// __ctor from float. Float is converted into float16 16-bit representation. + /// + /// float value + GPUd() explicit Float16_t(float v) noexcept { val = Base::ToUint16Impl(v); } + + /// + /// Converts float16 to float + /// + /// float representation of float16 value + GPUd() float ToFloat() const noexcept { return Base::ToFloatImpl(); } + + /// + /// Checks if the value is negative + /// + /// true if negative + using Base::IsNegative; + + /// + /// Tests if the value is NaN + /// + /// true if NaN + using Base::IsNaN; + + /// + /// Tests if the value is finite + /// + /// true if finite + using Base::IsFinite; + + /// + /// Tests if the value represents positive infinity. + /// + /// true if positive infinity + using Base::IsPositiveInfinity; + + /// + /// Tests if the value represents negative infinity + /// + /// true if negative infinity + using Base::IsNegativeInfinity; + + /// + /// Tests if the value is either positive or negative infinity. + /// + /// True if absolute value is infinity + using Base::IsInfinity; + + /// + /// Tests if the value is NaN or zero. Useful for comparisons. + /// + /// True if NaN or zero. + using Base::IsNaNOrZero; + + /// + /// Tests if the value is normal (not zero, subnormal, infinite, or NaN). + /// + /// True if so + using Base::IsNormal; + + /// + /// Tests if the value is subnormal (denormal). + /// + /// True if so + using Base::IsSubnormal; + + /// + /// Creates an instance that represents absolute value. + /// + /// Absolute value + using Base::Abs; + + /// + /// Creates a new instance with the sign flipped. + /// + /// Flipped sign instance + using Base::Negate; + + /// + /// IEEE defines that positive and negative zero are equal, this gives us a quick equality check + /// for two values by or'ing the private bits together and stripping the sign. They are both zero, + /// and therefore equivalent, if the resulting value is still zero. + /// + /// first value + /// second value + /// True if both arguments represent zero + using Base::AreZero; + + /// + /// User defined conversion operator. Converts Float16_t to float. + /// + GPUdi() explicit operator float() const noexcept { return ToFloat(); } + + using Base::operator==; + using Base::operator!=; + using Base::operator<; +}; + +static_assert(sizeof(Float16_t) == sizeof(uint16_t), "Sizes must match"); + +/** \brief bfloat16 (Brain Floating Point) data type + * + * \details This struct is used for converting float to bfloat16 and back + * so the user could feed inputs and fetch outputs using these type. + * + * The size of the structure should align with uint16_t and one can freely cast + * uint16_t buffers to/from Ort::BFloat16_t to feed and retrieve data. + * + * \code{.unparsed} + * // This example demonstrates converion from float to float16 + * constexpr float values[] = {1.f, 2.f, 3.f, 4.f, 5.f}; + * std::vector bfp16_values; + * bfp16_values.reserve(std::size(values)); + * std::transform(std::begin(values), std::end(values), std::back_inserter(bfp16_values), + * [](float value) { return Ort::BFloat16_t(value); }); + * + * \endcode + */ +struct BFloat16_t : OrtDataType::BFloat16Impl { + private: + /// + /// Constructor from a uint16_t representation of bfloat16 + /// used in FromBits() to escape overload resolution issue with + /// constructor from float. + /// No conversion is done. + /// + /// 16-bit bfloat16 value + constexpr explicit BFloat16_t(uint16_t v) noexcept { val = v; } + + public: + using Base = OrtDataType::BFloat16Impl; + + GPUdDefault() BFloat16_t() = default; + + /// + /// Explicit conversion to uint16_t representation of bfloat16. + /// + /// uint16_t bit representation of bfloat16 + /// new instance of BFloat16_t + GPUd() static constexpr BFloat16_t FromBits(uint16_t v) noexcept { return BFloat16_t(v); } + + /// + /// __ctor from float. Float is converted into bfloat16 16-bit representation. + /// + /// float value + GPUd() explicit BFloat16_t(float v) noexcept { val = Base::ToUint16Impl(v); } + + /// + /// Converts bfloat16 to float + /// + /// float representation of bfloat16 value + GPUd() float ToFloat() const noexcept { return Base::ToFloatImpl(); } + + /// + /// Checks if the value is negative + /// + /// true if negative + using Base::IsNegative; + + /// + /// Tests if the value is NaN + /// + /// true if NaN + using Base::IsNaN; + + /// + /// Tests if the value is finite + /// + /// true if finite + using Base::IsFinite; + + /// + /// Tests if the value represents positive infinity. + /// + /// true if positive infinity + using Base::IsPositiveInfinity; + + /// + /// Tests if the value represents negative infinity + /// + /// true if negative infinity + using Base::IsNegativeInfinity; + + /// + /// Tests if the value is either positive or negative infinity. + /// + /// True if absolute value is infinity + using Base::IsInfinity; + + /// + /// Tests if the value is NaN or zero. Useful for comparisons. + /// + /// True if NaN or zero. + using Base::IsNaNOrZero; + + /// + /// Tests if the value is normal (not zero, subnormal, infinite, or NaN). + /// + /// True if so + using Base::IsNormal; + + /// + /// Tests if the value is subnormal (denormal). + /// + /// True if so + using Base::IsSubnormal; + + /// + /// Creates an instance that represents absolute value. + /// + /// Absolute value + using Base::Abs; + + /// + /// Creates a new instance with the sign flipped. + /// + /// Flipped sign instance + using Base::Negate; + + /// + /// IEEE defines that positive and negative zero are equal, this gives us a quick equality check + /// for two values by or'ing the private bits together and stripping the sign. They are both zero, + /// and therefore equivalent, if the resulting value is still zero. + /// + /// first value + /// second value + /// True if both arguments represent zero + using Base::AreZero; + + /// + /// User defined conversion operator. Converts BFloat16_t to float. + /// + GPUdi() explicit operator float() const noexcept { return ToFloat(); } + + // We do not have an inherited impl for the below operators + // as the internal class implements them a little differently + bool operator==(const BFloat16_t& rhs) const noexcept; + bool operator!=(const BFloat16_t& rhs) const noexcept { return !(*this == rhs); } + bool operator<(const BFloat16_t& rhs) const noexcept; +}; + +static_assert(sizeof(BFloat16_t) == sizeof(uint16_t), "Sizes must match"); + +} // namespace OrtDataType + +} // namespace o2 +#endif diff --git a/Common/ML/include/ML/OrtInterface.h b/Common/ML/include/ML/OrtInterface.h new file mode 100644 index 0000000000000..987ce8fb4d6dd --- /dev/null +++ b/Common/ML/include/ML/OrtInterface.h @@ -0,0 +1,130 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file OrtInterface.h +/// \author Christian Sonnabend +/// \brief A header library for loading ONNX models and inferencing them on CPU and GPU + +#ifndef O2_ML_ORTINTERFACE_H +#define O2_ML_ORTINTERFACE_H + +// C++ and system includes +#include +#include +#include +#include +#include +#include + +// O2 includes +#include "GPUCommonLogger.h" + +namespace Ort +{ +struct SessionOptions; +struct MemoryInfo; +struct Env; +} // namespace Ort + +namespace o2::ml +{ + +class OrtModel +{ + + public: + // Constructors & destructors + OrtModel(); + OrtModel(std::unordered_map optionsMap); + void init(std::unordered_map optionsMap); + virtual ~OrtModel(); + + // General purpose + void initOptions(std::unordered_map optionsMap); + void initEnvironment(); + void initSession(); + void initSessionFromBuffer(const char* buffer, size_t bufferSize); + void memoryOnDevice(int32_t = 0); + bool isInitialized() { return mInitialized; } + void resetSession(); + + // Getters + std::vector> getNumInputNodes() const { return mInputShapes; } + std::vector> getNumOutputNodes() const { return mOutputShapes; } + std::vector getInputNames() const { return mInputNames; } + std::vector getOutputNames() const { return mOutputNames; } + Ort::SessionOptions* getSessionOptions(); + Ort::MemoryInfo* getMemoryInfo(); + Ort::Env* getEnv(); + int32_t getIntraOpNumThreads() const { return mIntraOpNumThreads; } + int32_t getInterOpNumThreads() const { return mInterOpNumThreads; } + + // Setters + void setDeviceId(int32_t id) { mDeviceId = id; } + void setIO(); + void setActiveThreads(int threads) { mIntraOpNumThreads = threads; } + void setIntraOpNumThreads(int threads) + { + if (mDeviceType == "CPU") { + mIntraOpNumThreads = threads; + } + } + void setInterOpNumThreads(int threads) + { + if (mDeviceType == "CPU") { + mInterOpNumThreads = threads; + } + } + void setEnv(Ort::Env*); + + // Conversion + template + std::vector v2v(std::vector&, bool = true); + + // Inferencing + template // class I is the input data type, e.g. float, class O is the output data type, e.g. OrtDataType::Float16_t from O2/Common/ML/include/ML/GPUORTFloat16.h + std::vector inference(std::vector&); + + template + std::vector inference(std::vector>&); + + template + void inference(I*, int64_t, O*); + + template + void inference(I**, int64_t, O*); + + void release(bool = false); + + private: + // ORT variables -> need to be hidden as pImpl + struct OrtVariables; + std::unique_ptr mPImplOrt; + + // Input & Output specifications of the loaded network + std::vector mInputNamesChar, mOutputNamesChar; + std::vector mInputNames, mOutputNames; + std::vector> mInputShapes, mOutputShapes, mInputShapesCopy, mOutputShapesCopy; // Input shapes + std::vector mInputSizePerNode, mOutputSizePerNode; // Output shapes + int32_t mInputsTotal = 0, mOutputsTotal = 0; // Total number of inputs and outputs + + // Environment settings + bool mInitialized = false, mDeterministicMode = false; + std::string mModelPath, mEnvName = "", mDeviceType = "CPU", mThreadAffinity = ""; // device options should be cpu, rocm, migraphx, cuda + int32_t mIntraOpNumThreads = 1, mInterOpNumThreads = 1, mDeviceId = -1, mEnableProfiling = 0, mLoggingLevel = 0, mAllocateDeviceMemory = 0, mEnableOptimizations = 0; + + std::string printShape(const std::vector&); + std::string printShape(const std::vector>&, std::vector&); +}; + +} // namespace o2::ml + +#endif // O2_ML_ORTINTERFACE_H diff --git a/Common/ML/src/OrtInterface.cxx b/Common/ML/src/OrtInterface.cxx new file mode 100644 index 0000000000000..8f88ab18dacbd --- /dev/null +++ b/Common/ML/src/OrtInterface.cxx @@ -0,0 +1,499 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file OrtInterface.cxx +/// \author Christian Sonnabend +/// \brief A header library for loading ONNX models and inferencing them on CPU and GPU + +#include "ML/OrtInterface.h" +#include "ML/3rdparty/GPUORTFloat16.h" + +// ONNX includes +#include + +#include + +namespace o2 +{ + +namespace ml +{ + +OrtModel::OrtModel() = default; +OrtModel::OrtModel(std::unordered_map optionsMap) { init(optionsMap); } +OrtModel::~OrtModel() = default; +void OrtModel::init(std::unordered_map optionsMap) +{ + initOptions(optionsMap); + initEnvironment(); +} + +struct OrtModel::OrtVariables { // The actual implementation is hidden in the .cxx file + // ORT runtime objects + Ort::RunOptions runOptions; + std::unique_ptr env = nullptr; + std::unique_ptr session = nullptr; ///< ONNX session + Ort::SessionOptions sessionOptions; + Ort::AllocatorWithDefaultOptions allocator; + Ort::MemoryInfo memoryInfo = Ort::MemoryInfo("Cpu", OrtAllocatorType::OrtDeviceAllocator, 0, OrtMemType::OrtMemTypeDefault); + std::unique_ptr ioBinding = nullptr; +}; + +// General purpose +void OrtModel::initOptions(std::unordered_map optionsMap) +{ + mPImplOrt = std::make_unique(); + + // Load from options map + if (!optionsMap.contains("model-path")) { + LOG(fatal) << "(ORT) Model path must be contained in options map!"; + } + + if (!optionsMap["model-path"].empty()) { + mModelPath = optionsMap["model-path"]; + mDeviceType = (optionsMap.contains("device-type") ? optionsMap["device-type"] : "CPU"); + mDeviceId = (optionsMap.contains("device-id") ? std::stoi(optionsMap["device-id"]) : -1); + mAllocateDeviceMemory = (optionsMap.contains("allocate-device-memory") ? std::stoi(optionsMap["allocate-device-memory"]) : 0); + mIntraOpNumThreads = (optionsMap.contains("intra-op-num-threads") ? std::stoi(optionsMap["intra-op-num-threads"]) : 0); + mInterOpNumThreads = (optionsMap.contains("inter-op-num-threads") ? std::stoi(optionsMap["inter-op-num-threads"]) : 0); + mLoggingLevel = (optionsMap.contains("logging-level") ? std::stoi(optionsMap["logging-level"]) : 0); + mEnableProfiling = (optionsMap.contains("enable-profiling") ? std::stoi(optionsMap["enable-profiling"]) : 0); + mEnableOptimizations = (optionsMap.contains("enable-optimizations") ? std::stoi(optionsMap["enable-optimizations"]) : 0); + mEnvName = (optionsMap.contains("onnx-environment-name") ? optionsMap["onnx-environment-name"] : "onnx_model_inference"); + mDeterministicMode = (optionsMap.contains("deterministic-compute") ? std::stoi(optionsMap["deterministic-compute"]) : 0); + + if (mDeviceType == "CPU") { + (mPImplOrt->sessionOptions).SetIntraOpNumThreads(mIntraOpNumThreads); + (mPImplOrt->sessionOptions).SetInterOpNumThreads(mInterOpNumThreads); + if (mIntraOpNumThreads > 1 || mInterOpNumThreads > 1) { + (mPImplOrt->sessionOptions).SetExecutionMode(ExecutionMode::ORT_PARALLEL); + } else if (mIntraOpNumThreads == 1) { + (mPImplOrt->sessionOptions).SetExecutionMode(ExecutionMode::ORT_SEQUENTIAL); + } + if (mLoggingLevel < 2) { + LOG(info) << "(ORT) CPU execution provider set with " << mIntraOpNumThreads << " (mIntraOpNumThreads) and " << mInterOpNumThreads << " (mInterOpNumThreads) threads"; + } + } + + // OrtROCMProviderOptions rocm_options{}; + // (mPImplOrt->sessionOptions).AppendExecutionProvider_ROCM(rocm_options); + + (mPImplOrt->sessionOptions).DisableMemPattern(); + (mPImplOrt->sessionOptions).DisableCpuMemArena(); + + if (mEnableProfiling) { + if (optionsMap.contains("profiling-output-path")) { + (mPImplOrt->sessionOptions).EnableProfiling((optionsMap["profiling-output-path"] + "/ORT_LOG_").c_str()); + } else { + LOG(warning) << "(ORT) If profiling is enabled, optionsMap[\"profiling-output-path\"] should be set. Disabling profiling for now."; + (mPImplOrt->sessionOptions).DisableProfiling(); + } + } else { + (mPImplOrt->sessionOptions).DisableProfiling(); + } + + if (mDeterministicMode > 0) { + (mPImplOrt->sessionOptions).AddConfigEntry("session_options.use_deterministic_compute", "1"); + } + + (mPImplOrt->sessionOptions).SetGraphOptimizationLevel(GraphOptimizationLevel(mEnableOptimizations)); + (mPImplOrt->sessionOptions).SetLogSeverityLevel(OrtLoggingLevel(mLoggingLevel)); + + mInitialized = true; + } else { + LOG(fatal) << "(ORT) Model path cannot be empty!"; + } +} + +void OrtModel::initEnvironment() +{ + mPImplOrt->env = std::make_unique( + OrtLoggingLevel(mLoggingLevel), + (mEnvName.empty() ? "ORT" : mEnvName.c_str()), + // Integrate ORT logging into Fairlogger + [](void* param, OrtLoggingLevel severity, const char* category, const char* logid, const char* code_location, const char* message) { + if (severity == ORT_LOGGING_LEVEL_VERBOSE) { + LOG(debug) << "(ORT) [" << logid << "|" << category << "|" << code_location << "]: " << message; + } else if (severity == ORT_LOGGING_LEVEL_INFO) { + LOG(info) << "(ORT) [" << logid << "|" << category << "|" << code_location << "]: " << message; + } else if (severity == ORT_LOGGING_LEVEL_WARNING) { + LOG(warning) << "(ORT) [" << logid << "|" << category << "|" << code_location << "]: " << message; + } else if (severity == ORT_LOGGING_LEVEL_ERROR) { + LOG(error) << "(ORT) [" << logid << "|" << category << "|" << code_location << "]: " << message; + } else if (severity == ORT_LOGGING_LEVEL_FATAL) { + LOG(fatal) << "(ORT) [" << logid << "|" << category << "|" << code_location << "]: " << message; + } else { + LOG(info) << "(ORT) [" << logid << "|" << category << "|" << code_location << "]: " << message; + } + }, + (void*)3); + (mPImplOrt->env)->DisableTelemetryEvents(); // Disable telemetry events +} + +void OrtModel::initSessionFromBuffer(const char* buffer, size_t bufferSize) +{ + mPImplOrt->sessionOptions.AddConfigEntry("session.load_model_format", "ONNX"); + mPImplOrt->sessionOptions.AddConfigEntry("session.use_ort_model_bytes_directly", "1"); + + mPImplOrt->session = std::make_unique(*mPImplOrt->env, + buffer, + bufferSize, + mPImplOrt->sessionOptions); + mPImplOrt->ioBinding = std::make_unique(*mPImplOrt->session); + + setIO(); + + if (mLoggingLevel < 2) { + LOG(info) << "(ORT) Model loaded successfully from buffer! (inputs: " << printShape(mInputShapes, mInputNames) << ", outputs: " << printShape(mOutputShapes, mInputNames) << ")"; + } +} + +void OrtModel::initSession() +{ + if (mAllocateDeviceMemory) { + memoryOnDevice(mDeviceId); + } + mPImplOrt->session = std::make_unique(*mPImplOrt->env, mModelPath.c_str(), mPImplOrt->sessionOptions); + mPImplOrt->ioBinding = std::make_unique(*mPImplOrt->session); + + setIO(); + + if (mLoggingLevel < 2) { + LOG(info) << "(ORT) Model loaded successfully! (inputs: " << printShape(mInputShapes, mInputNames) << ", outputs: " << printShape(mOutputShapes, mInputNames) << ")"; + } +} + +void OrtModel::memoryOnDevice(int32_t deviceIndex) +{ + if (deviceIndex >= 0) { + (mPImplOrt->runOptions).AddConfigEntry("disable_synchronize_execution_providers", "1"); + (mPImplOrt->sessionOptions).AddConfigEntry("session.use_device_allocator_for_initializers", "1"); // See kOrtSessionOptionsUseDeviceAllocatorForInitializers, https://github.com/microsoft/onnxruntime/blob/main/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h + (mPImplOrt->sessionOptions).AddConfigEntry("session.use_env_allocators", "1"); // This should enable to use the volatile memory allocation defined in O2/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizerHost.cxx; not working yet: ONNX still assigns new memory at init time + (mPImplOrt->sessionOptions).AddConfigEntry("session_options.enable_cpu_mem_arena", "0"); // This should enable to use the volatile memory allocation defined in O2/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizerHost.cxx; not working yet: ONNX still assigns new memory at init time + // Arena memory shrinkage comes at performance cost + // For now prefer to use single allocation, enabled by O2/GPU/GPUTracking/Base/cuda/GPUReconstructionCUDA.cu -> SetONNXGPUStream -> rocm_options.arena_extend_strategy = 0; + (mPImplOrt->runOptions).AddConfigEntry("memory.enable_memory_arena_shrinkage", ("gpu:" + std::to_string(deviceIndex)).c_str()); // See kOrtRunOptionsConfigEnableMemoryArenaShrinkage, https://github.com/microsoft/onnxruntime/blob/90c263f471bbce724e77d8e62831d3a9fa838b2f/include/onnxruntime/core/session/onnxruntime_run_options_config_keys.h#L27 + + std::string dev_mem_str = ""; + if (mDeviceType == "ROCM") { + dev_mem_str = "HipPinned"; + } + if (mDeviceType == "CUDA") { + dev_mem_str = "Cuda"; + } + mPImplOrt->memoryInfo = Ort::MemoryInfo(dev_mem_str.c_str(), OrtAllocatorType::OrtDeviceAllocator, deviceIndex, OrtMemType::OrtMemTypeDefault); + if (mLoggingLevel < 2) { + LOG(info) << "(ORT) Memory info set to on-device memory for device type " << mDeviceType << " with ID " << deviceIndex << " and mPImplOrt pointer " << mPImplOrt; + } + } +} + +void OrtModel::resetSession() +{ + mPImplOrt->session = std::make_unique(*(mPImplOrt->env), mModelPath.c_str(), mPImplOrt->sessionOptions); +} + +// Getters +Ort::SessionOptions* OrtModel::getSessionOptions() +{ + return &mPImplOrt->sessionOptions; +} + +Ort::MemoryInfo* OrtModel::getMemoryInfo() +{ + return &mPImplOrt->memoryInfo; +} + +Ort::Env* OrtModel::getEnv() +{ + return (mPImplOrt->env).get(); +} + +template +std::vector OrtModel::v2v(std::vector& input, bool clearInput) +{ + if constexpr (std::is_same_v) { + return input; + } else { + std::vector output(input.size()); + std::transform(std::begin(input), std::end(input), std::begin(output), [](I f) { return O(f); }); + if (clearInput) { + input.clear(); + } + return output; + } +} + +void OrtModel::setIO() +{ + for (size_t i = 0; i < (mPImplOrt->session)->GetInputCount(); ++i) { + mInputNames.push_back((mPImplOrt->session)->GetInputNameAllocated(i, mPImplOrt->allocator).get()); + } + for (size_t i = 0; i < (mPImplOrt->session)->GetInputCount(); ++i) { + mInputShapes.emplace_back((mPImplOrt->session)->GetInputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape()); + } + for (size_t i = 0; i < (mPImplOrt->session)->GetOutputCount(); ++i) { + mOutputNames.push_back((mPImplOrt->session)->GetOutputNameAllocated(i, mPImplOrt->allocator).get()); + } + for (size_t i = 0; i < (mPImplOrt->session)->GetOutputCount(); ++i) { + mOutputShapes.emplace_back((mPImplOrt->session)->GetOutputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape()); + } + + mInputNamesChar.resize(mInputNames.size(), nullptr); + std::transform(std::begin(mInputNames), std::end(mInputNames), std::begin(mInputNamesChar), + [&](const std::string& str) { return str.c_str(); }); + mOutputNamesChar.resize(mOutputNames.size(), nullptr); + std::transform(std::begin(mOutputNames), std::end(mOutputNames), std::begin(mOutputNamesChar), + [&](const std::string& str) { return str.c_str(); }); + + mInputShapesCopy = mInputShapes; + mOutputShapesCopy = mOutputShapes; + mInputSizePerNode.resize(mInputShapes.size(), 1); + mOutputSizePerNode.resize(mOutputShapes.size(), 1); + mInputsTotal = 1; + for (size_t i = 0; i < mInputShapes.size(); ++i) { + if (mInputShapes[i].size() > 0) { + for (size_t j = 1; j < mInputShapes[i].size(); ++j) { + if (mInputShapes[i][j] > 0) { + mInputsTotal *= mInputShapes[i][j]; + mInputSizePerNode[i] *= mInputShapes[i][j]; + } + } + } + } + mOutputsTotal = 1; + for (size_t i = 0; i < mOutputShapes.size(); ++i) { + if (mOutputShapes[i].size() > 0) { + for (size_t j = 1; j < mOutputShapes[i].size(); ++j) { + if (mOutputShapes[i][j] > 0) { + mOutputsTotal *= mOutputShapes[i][j]; + mOutputSizePerNode[i] *= mOutputShapes[i][j]; + } + } + } + } +} + +void OrtModel::setEnv(Ort::Env* env) +{ + mPImplOrt->env.reset(env); +} + +// Inference +template +std::vector OrtModel::inference(std::vector& input) +{ + std::vector inputShape = mInputShapes[0]; + inputShape[0] = input.size(); + for (size_t i = 1; i < mInputShapes[0].size(); ++i) { + inputShape[0] /= mInputShapes[0][i]; + } + std::vector inputTensor; + if constexpr (std::is_same_v) { + inputTensor.emplace_back(Ort::Value::CreateTensor(mPImplOrt->memoryInfo, reinterpret_cast(input.data()), input.size(), inputShape.data(), inputShape.size())); + } else { + inputTensor.emplace_back(Ort::Value::CreateTensor(mPImplOrt->memoryInfo, input.data(), input.size(), inputShape.data(), inputShape.size())); + } + // input.clear(); + auto outputTensors = (mPImplOrt->session)->Run(mPImplOrt->runOptions, mInputNamesChar.data(), inputTensor.data(), inputTensor.size(), mOutputNamesChar.data(), mOutputNamesChar.size()); + O* outputValues = outputTensors[0].template GetTensorMutableData(); + std::vector outputValuesVec{outputValues, outputValues + inputShape[0] * mOutputShapes[0][1]}; + outputTensors.clear(); + return outputValuesVec; +} + +template std::vector o2::ml::OrtModel::inference(std::vector&); +template std::vector o2::ml::OrtModel::inference(std::vector&); +template std::vector o2::ml::OrtModel::inference(std::vector&); + +template +void OrtModel::inference(I* input, int64_t input_size, O* output) +{ + // std::vector providers = Ort::GetAvailableProviders(); + // for (const auto& provider : providers) { + // LOG(info) << "Available Execution Provider: " << provider; + // } + std::vector inputShape{input_size, (int64_t)mInputShapes[0][1]}; + Ort::Value inputTensor = Ort::Value(nullptr); + if constexpr (std::is_same_v) { + inputTensor = Ort::Value::CreateTensor(mPImplOrt->memoryInfo, reinterpret_cast(input), input_size * mInputShapes[0][1], inputShape.data(), inputShape.size()); + } else { + inputTensor = Ort::Value::CreateTensor(mPImplOrt->memoryInfo, input, input_size * mInputShapes[0][1], inputShape.data(), inputShape.size()); + } + (mPImplOrt->ioBinding)->BindInput(mInputNames[0].c_str(), inputTensor); + + std::vector outputShape{input_size, mOutputShapes[0][1]}; + Ort::Value outputTensor = Ort::Value(nullptr); + if constexpr (std::is_same_v) { + outputTensor = Ort::Value::CreateTensor(mPImplOrt->memoryInfo, reinterpret_cast(output), input_size * mOutputShapes[0][1], outputShape.data(), outputShape.size()); + } else { + outputTensor = Ort::Value::CreateTensor(mPImplOrt->memoryInfo, output, input_size * mOutputShapes[0][1], outputShape.data(), outputShape.size()); + } + (mPImplOrt->ioBinding)->BindOutput(mOutputNames[0].c_str(), outputTensor); + + (mPImplOrt->session)->Run(mPImplOrt->runOptions, *mPImplOrt->ioBinding); + // mPImplOrt->session->Run( + // mPImplOrt->runOptions, + // mInputNamesChar.data(), + // &inputTensor, + // mInputNamesChar.size(), + // mOutputNamesChar.data(), + // &outputTensor, + // mOutputNamesChar.size()); +} + +template void OrtModel::inference(OrtDataType::Float16_t*, int64_t, OrtDataType::Float16_t*); +template void OrtModel::inference(OrtDataType::Float16_t*, int64_t, float*); +template void OrtModel::inference(float*, int64_t, OrtDataType::Float16_t*); +template void OrtModel::inference(float*, int64_t, float*); + +template +void OrtModel::inference(I** input, int64_t input_size, O* output) +{ + std::vector inputTensors(mInputShapesCopy.size()); + + for (size_t i = 0; i < mInputShapesCopy.size(); ++i) { + + mInputShapesCopy[i][0] = input_size; // batch-size + mOutputShapesCopy[i][0] = input_size; // batch-size + + if constexpr (std::is_same_v) { + inputTensors[i] = Ort::Value::CreateTensor( + mPImplOrt->memoryInfo, + reinterpret_cast(input[i]), + mInputSizePerNode[i] * input_size, + mInputShapesCopy[i].data(), + mInputShapesCopy[i].size()); + } else { + inputTensors[i] = Ort::Value::CreateTensor( + mPImplOrt->memoryInfo, + input[i], + mInputSizePerNode[i] * input_size, + mInputShapesCopy[i].data(), + mInputShapesCopy[i].size()); + } + } + + Ort::Value outputTensor = Ort::Value(nullptr); + if constexpr (std::is_same_v) { + outputTensor = Ort::Value::CreateTensor( + mPImplOrt->memoryInfo, + reinterpret_cast(output), + mOutputSizePerNode[0] * input_size, // assumes that there is only one output node + mOutputShapesCopy[0].data(), + mOutputShapesCopy[0].size()); + } else { + outputTensor = Ort::Value::CreateTensor( + mPImplOrt->memoryInfo, + output, + mOutputSizePerNode[0] * input_size, // assumes that there is only one output node + mOutputShapesCopy[0].data(), + mOutputShapesCopy[0].size()); + } + + // === Run inference === + mPImplOrt->session->Run( + mPImplOrt->runOptions, + mInputNamesChar.data(), + inputTensors.data(), + mInputNamesChar.size(), + mOutputNamesChar.data(), + &outputTensor, + mOutputNamesChar.size()); +} + +template void OrtModel::inference(OrtDataType::Float16_t**, int64_t, OrtDataType::Float16_t*); +template void OrtModel::inference(OrtDataType::Float16_t**, int64_t, float*); +template void OrtModel::inference(float**, int64_t, OrtDataType::Float16_t*); +template void OrtModel::inference(float**, int64_t, float*); + +template +std::vector OrtModel::inference(std::vector>& inputs) +{ + std::vector input_tensors; + + for (size_t i = 0; i < inputs.size(); ++i) { + + mInputShapesCopy[i][0] = inputs[i].size() / mInputSizePerNode[i]; // batch-size + + if constexpr (std::is_same_v) { + input_tensors.emplace_back( + Ort::Value::CreateTensor( + mPImplOrt->memoryInfo, + reinterpret_cast(inputs[i].data()), + mInputSizePerNode[i] * mInputShapesCopy[i][0], + mInputShapesCopy[i].data(), + mInputShapesCopy[i].size())); + } else { + input_tensors.emplace_back( + Ort::Value::CreateTensor( + mPImplOrt->memoryInfo, + inputs[i].data(), + mInputSizePerNode[i] * mInputShapesCopy[i][0], + mInputShapesCopy[i].data(), + mInputShapesCopy[i].size())); + } + } + + int32_t totalOutputSize = mOutputsTotal * mInputShapesCopy[0][0]; + + // === Run inference === + auto output_tensors = mPImplOrt->session->Run( + mPImplOrt->runOptions, + mInputNamesChar.data(), + input_tensors.data(), + input_tensors.size(), + mOutputNamesChar.data(), + mOutputNamesChar.size()); + + // === Extract output values === + O* output_data = output_tensors[0].template GetTensorMutableData(); + std::vector output_vec(output_data, output_data + totalOutputSize); + output_tensors.clear(); + return output_vec; +} + +template std::vector OrtModel::inference(std::vector>&); +template std::vector OrtModel::inference(std::vector>&); + +// Release session +void OrtModel::release(bool profilingEnabled) +{ + mPImplOrt.reset(); +} + +// private +std::string OrtModel::printShape(const std::vector& v) +{ + std::stringstream ss(""); + for (size_t i = 0; i < v.size() - 1; i++) { + ss << v[i] << "x"; + } + ss << v[v.size() - 1]; + return ss.str(); +} + +std::string OrtModel::printShape(const std::vector>& v, std::vector& n) +{ + std::stringstream ss(""); + for (size_t i = 0; i < v.size(); i++) { + ss << n[i] << " -> ("; + for (size_t j = 0; j < v[i].size() - 1; j++) { + ss << v[i][j] << "x"; + } + ss << v[i][v[i].size() - 1] << "); "; + } + return ss.str(); +} + +} // namespace ml + +} // namespace o2 diff --git a/Common/MathUtils/CMakeLists.txt b/Common/MathUtils/CMakeLists.txt index 5b24fcfcdf7c3..d618bb8549175 100644 --- a/Common/MathUtils/CMakeLists.txt +++ b/Common/MathUtils/CMakeLists.txt @@ -15,14 +15,17 @@ o2_add_library( src/Cartesian.cxx src/Chebyshev3D.cxx src/Chebyshev3DCalc.cxx + src/SymMatrixSolver.cxx + src/Tsallis.cxx PUBLIC_LINK_LIBRARIES ROOT::Hist - FairRoot::Base + FairLogger::FairLogger O2::CommonConstants O2::GPUCommon ROOT::GenVector ROOT::Geom - Vc::Vc) + Vc::Vc + Boost::boost) o2_target_root_dictionary( MathUtils @@ -35,7 +38,10 @@ o2_target_root_dictionary( include/MathUtils/CachingTF1.h include/MathUtils/RandomRing.h include/MathUtils/Primitive2D.h - include/MathUtils/SMatrixGPU.h) + include/MathUtils/SMatrixGPU.h + include/MathUtils/SymMatrixSolver.h + include/MathUtils/Tsallis.h + include/MathUtils/LegendrePols.h) o2_add_test( CachingTF1 diff --git a/Common/MathUtils/include/MathUtils/BetheBlochAleph.h b/Common/MathUtils/include/MathUtils/BetheBlochAleph.h new file mode 100644 index 0000000000000..bd72faffb0503 --- /dev/null +++ b/Common/MathUtils/include/MathUtils/BetheBlochAleph.h @@ -0,0 +1,35 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef AliceO2_COMMON_BETHEBLOCH_H_ +#define AliceO2_COMMON_BETHEBLOCH_H_ + +#include "GPUCommonDef.h" +#include "GPUCommonMath.h" + +namespace o2::common +{ + +template +GPUdi() T BetheBlochAleph(T bg, T kp1, T kp2, T kp3, T kp4, T kp5) +{ + T beta = bg / o2::gpu::GPUCommonMath::Sqrt(static_cast(1.) + bg * bg); + + T aa = o2::gpu::GPUCommonMath::Pow(beta, kp4); + T bb = o2::gpu::GPUCommonMath::Pow(static_cast(1.) / bg, kp5); + bb = o2::gpu::GPUCommonMath::Log(kp3 + bb); + + return (kp2 - aa - bb) * kp1 / aa; +} + +} // namespace o2::common + +#endif diff --git a/Common/MathUtils/include/MathUtils/Cartesian.h b/Common/MathUtils/include/MathUtils/Cartesian.h index 14509def6efb1..9b917707835a6 100644 --- a/Common/MathUtils/include/MathUtils/Cartesian.h +++ b/Common/MathUtils/include/MathUtils/Cartesian.h @@ -34,8 +34,8 @@ #include "GPUCommonMath.h" #include "CartesianGPU.h" #include "SMatrixGPU.h" - #endif + #include "GPUROOTCartesianFwd.h" #include "GPUROOTSMatrixFwd.h" @@ -93,6 +93,9 @@ class Rotation2D sn = mSin; } + value_t getCos() const { return mCos; } + value_t getSin() const { return mSin; } + template Point3D operator()(const Point3D& v) const { // local->master @@ -247,6 +250,34 @@ class Transform3D : public ROOT::Math::Transform3D ClassDefNV(Transform3D, 1); }; #endif // Disable for GPU + +// Aliasing of the Dot(SVector a, SVector b) operation between SVectors +#if (!defined(GPUCA_STANDALONE) || !defined(DGPUCA_NO_ROOT)) && !defined(GPUCA_GPUCODE) && !defined(GPUCOMMONRTYPES_H_ACTIVE) +template +inline T Dot(const SVector& lhs, const SVector& rhs) +{ + return ROOT::Math::Dot(lhs, rhs); +} + +template +inline SMatrix> Similarity(const SMatrix& lhs, const SMatrix>& rhs) +{ + return ROOT::Math::Similarity(lhs, rhs); +} +#else +template +GPUdi() T Dot(const SVector& lhs, const SVector& rhs) +{ + return o2::math_utils::detail::Dot(lhs, rhs); +} + +template +GPUdi() SMatrix> Similarity(const SMatrix& lhs, const SMatrix>& rhs) +{ + return o2::math_utils::detail::Similarity(lhs, rhs); +} +#endif // Disable for GPU + } // namespace math_utils } // namespace o2 diff --git a/Common/MathUtils/include/MathUtils/LegendrePols.h b/Common/MathUtils/include/MathUtils/LegendrePols.h new file mode 100644 index 0000000000000..d5e6a259573f4 --- /dev/null +++ b/Common/MathUtils/include/MathUtils/LegendrePols.h @@ -0,0 +1,213 @@ +// Copyright 2020-2022 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file LegendrePols.h +/// \brief Definition of the NDim Legendre Polynominals +/// \author felix.schlepper@cern.ch + +#ifndef LEGENDRE_NDIM_POLYNOMINAL_H_ +#define LEGENDRE_NDIM_POLYNOMINAL_H_ + +#include "TNamed.h" +#include "Math/IParamFunction.h" +#include "TMatrixD.h" + +#include + +#include + +namespace o2::math_utils +{ + +// Defines the 1D Legendre Polynominals with coefficients: +// w(u) = c_0 + c_1 * u + c_2 * 0.5 * (3 * u * u - 1) + ... +// for u in [-1.0, 1.0] +class Legendre1DPolynominal final : public TNamed, + public ROOT::Math::IParametricFunctionOneDim +{ + public: + Legendre1DPolynominal() = default; + Legendre1DPolynominal(const Legendre1DPolynominal&) = default; + Legendre1DPolynominal(Legendre1DPolynominal&&) = delete; + Legendre1DPolynominal& operator=(const Legendre1DPolynominal&) = default; + Legendre1DPolynominal& operator=(Legendre1DPolynominal&&) = delete; + Legendre1DPolynominal(unsigned int order) : fOrder(order) {} + Legendre1DPolynominal(const std::vector p) + : fOrder(p.size() - 1), fParams(p) {} + + double operator()(double x) const { return DoEvalPar(x, Parameters()); } + double operator()(int i, double x) const + { + return DoEvalParSingle(i, x, Parameters()); + } + + ~Legendre1DPolynominal() final = default; + + const double* Parameters() const final { return &fParams.front(); } + + virtual void SetParameters(const double* p) final + { + fParams = std::vector(p, p + NPar()); + } + + unsigned int NPar() const final { return fParams.size(); } + unsigned int NOrder() const { return fOrder; } + + ROOT::Math::IBaseFunctionOneDim* Clone() const final { return new Legendre1DPolynominal(fParams); } + TObject* Clone(const char* name) const final + { + auto n = new Legendre1DPolynominal(fParams); + n->SetName(name); + return n; + } + + private: + double DoEvalPar(double x, const double* p) const final + { + double sum{0.0}; + for (unsigned int iOrder{0}; iOrder <= fOrder; ++iOrder) { + sum += p[iOrder] * boost::math::legendre_p(iOrder, x); + } + return sum; + } + + double DoEvalParSingle(int i, double x, const double* p) const + { + return p[i] * boost::math::legendre_p(i, x); + } + + unsigned int fOrder{0}; + std::vector fParams; + + ClassDefOverride(o2::math_utils::Legendre1DPolynominal, 1); +}; + +// Defines the 2D Legendre Polynominals with coefficients: +// w(u, v) = c_00 + +// c_10 * u + c_11 * v + +// c_20 * 0.5 * (3 * u * u - 1) + c_21 * u * v + c_22 * (3 * v * v - 1) + +/// .... +// for u&v in [-1.0, 1.0] +class Legendre2DPolynominal final : public TNamed, + public ROOT::Math::IParametricFunctionMultiDim +{ + public: + Legendre2DPolynominal() = default; + Legendre2DPolynominal(unsigned int order) : fOrder(order) {} + Legendre2DPolynominal(const std::vector& p) + : fOrder(p.size() - 1), fParams(p) {} + Legendre2DPolynominal(const TMatrixD& p) : fOrder(p.GetNrows() - 1) + { + fParams = std::vector(NPar()); + for (unsigned int iOrder{0}; iOrder <= fOrder; ++iOrder) { + for (unsigned int jOrder{0}; jOrder <= iOrder; ++jOrder) { + fParams[getFlatIdx(iOrder, jOrder)] = p(iOrder, jOrder); + } + } + } + ~Legendre2DPolynominal() final = default; + + double operator()(const double* x) const + { + return DoEvalPar(x, Parameters()); + } + double operator()(double x, double y) const { return DoEvalPar(x, y); } + double operator()(int i, int j, const double* x) const + { + return DoEvalParSingle(i, j, x, Parameters()); + } + double operator()(int i, int j, double x, double y) const + { + return DoEvalParSingle(i, j, x, y, Parameters()); + } + + const double* Parameters() const final { return &fParams.front(); } + + void SetParameters(const double* p) final + { + fParams = std::vector(p, p + NPar()); + } + + unsigned int NPar() const final + { + return fOrder * (fOrder + 1) / 2 + fOrder + 1; + } + unsigned int NDim() const final { return 2; } + unsigned int NOrder() const { return fOrder; } + + TMatrixD getCoefficients() const + { + TMatrixD m(fOrder + 1, fOrder + 1); + for (unsigned int iOrder{0}; iOrder <= fOrder; ++iOrder) { + for (unsigned int jOrder{0}; jOrder <= iOrder; ++jOrder) { + m(iOrder, jOrder) = fParams[getFlatIdx(iOrder, jOrder)]; + } + } + return m; + } + + void printCoefficients() const { getCoefficients().Print(); } + + // Unimplemented + ROOT::Math::IBaseFunctionMultiDim* Clone() const final { return new Legendre2DPolynominal(fParams); } + TObject* Clone(const char* name) const final + { + auto n = new Legendre2DPolynominal(fParams); + n->SetName(name); + return n; + } + + private: + double DoEvalPar(const double* x, const double* p) const final + { + double sum{0.0}; + for (unsigned int iOrder{0}; iOrder <= fOrder; ++iOrder) { + for (unsigned int jOrder{0}; jOrder <= iOrder; ++jOrder) { + sum += DoEvalParSingle(iOrder, jOrder, x, p); + } + } + return sum; + } + + double DoEvalPar(double x, double y) const + { + double sum{0.0}; + for (unsigned int iOrder{0}; iOrder <= fOrder; ++iOrder) { + for (unsigned int jOrder{0}; jOrder <= iOrder; ++jOrder) { + sum += DoEvalParSingle(iOrder, jOrder, x, y, Parameters()); + } + } + return sum; + } + + double DoEvalParSingle(int i, int j, const double* x, const double* p) const + { + return DoEvalParSingle(i, j, x[0], x[1], p); + } + + double DoEvalParSingle(int i, int j, double x, double y, + const double* p) const + { + return p[getFlatIdx(i, j)] * boost::math::legendre_p(j, x) * + boost::math::legendre_p(i - j, y); + } + + inline int getFlatIdx(int i, int j) const { return i * (i + 1) / 2 + j; } + + unsigned int fOrder{0}; + std::vector fParams; + + ClassDefOverride(o2::math_utils::Legendre2DPolynominal, 1); +}; + +} // namespace o2::math_utils + +#endif diff --git a/Common/MathUtils/include/MathUtils/SMatrixGPU.h b/Common/MathUtils/include/MathUtils/SMatrixGPU.h index 2faaf118d1daa..8158a93666a92 100644 --- a/Common/MathUtils/include/MathUtils/SMatrixGPU.h +++ b/Common/MathUtils/include/MathUtils/SMatrixGPU.h @@ -15,10 +15,6 @@ /// Only parts strictly requiring STD library have been changed. /// Also some utilities to have basic checks and printouts working on GPUs have been rewritten. /// -/// Notably only templated implementation of -/// row_offsets_utils::make and row_offsets_utils::do_make -/// has been reworked to support gpustd::array as backend. -/// /// Other than that, the author is not taking any credit on the methodologies implemented /// which have been taken straight from root source code /// @@ -29,11 +25,15 @@ #define ALICEO2_SMATRIX_GPU_H #include "GPUCommonDef.h" -#include "GPUCommonArray.h" #include "GPUCommonMath.h" #include "GPUCommonAlgorithm.h" +#include "GPUCommonLogger.h" +#ifndef GPUCA_GPUCODE_DEVICE +#include +#include +#endif -namespace o2::math_utils +namespace o2::math_utils::detail { template struct Check { @@ -66,9 +66,9 @@ class SVectorGPU GPUd() SVectorGPU(); GPUd() SVectorGPU(const SVectorGPU& rhs); - GPUd() const T& operator[](unsigned int i) const; + GPUhd() const T& operator[](unsigned int i) const; + GPUhd() T& operator[](unsigned int i); GPUd() const T& operator()(unsigned int i) const; - GPUd() T& operator[](unsigned int i); GPUd() T& operator()(unsigned int i); GPUd() const T* Array() const; GPUd() T* Array(); @@ -113,19 +113,19 @@ GPUdi() const T* SVectorGPU::end() const } template -GPUdi() const T& SVectorGPU::operator[](unsigned int i) const +GPUhdi() const T& SVectorGPU::operator[](unsigned int i) const { return mArray[i]; } template -GPUdi() const T& SVectorGPU::operator()(unsigned int i) const +GPUhdi() T& SVectorGPU::operator[](unsigned int i) { return mArray[i]; } template -GPUdi() T& SVectorGPU::operator[](unsigned int i) +GPUdi() const T& SVectorGPU::operator()(unsigned int i) const { return mArray[i]; } @@ -140,7 +140,7 @@ template GPUd() SVectorGPU::SVectorGPU() { for (unsigned int i = 0; i < N; ++i) { - mArray[i] = 7; + mArray[i] = 0; } } @@ -204,7 +204,7 @@ GPUdi() SVectorGPU& SVectorGPU::operator-=(const SVectorGPU& r } template -GPUd() SVectorGPU& SVectorGPU::operator+=(const SVectorGPU& rhs) +GPUdi() SVectorGPU& SVectorGPU::operator+=(const SVectorGPU& rhs) { for (unsigned int i = 0; i < D; ++i) { mArray[i] += rhs.apply(i); @@ -283,14 +283,14 @@ struct make_indices : make_indices_impl<0, indices<>, N> { }; template -constexpr auto do_make(F f, indices) -> gpu::gpustd::array +constexpr auto do_make(F f, indices) -> std::array { - gpu::gpustd::array retarr = {f(I0 + I)...}; + std::array retarr = {f(I0 + I)...}; return retarr; } template -constexpr auto make(F f) -> gpu::gpustd::array +constexpr auto make(F f) -> std::array { return do_make(f, typename make_indices::type()); } @@ -303,31 +303,11 @@ class MatRepSymGPU public: typedef T value_type; GPUdDefault() MatRepSymGPU() = default; - GPUdi() T& operator()(unsigned int i, unsigned int j) - { - return mArray[offset(i, j)]; - } - - GPUdi() T const& operator()(unsigned int i, unsigned int j) const - { - return mArray[offset(i, j)]; - } - - GPUdi() T& operator[](unsigned int i) - { - return mArray[off(i)]; - } - - GPUdi() T const& operator[](unsigned int i) const - { - return mArray[off(i)]; - } - - GPUdi() T apply(unsigned int i) const - { - return mArray[off(i)]; - } - + GPUdi() T& operator()(unsigned int i, unsigned int j) { return mArray[offset(i, j)]; } + GPUdi() T const& operator()(unsigned int i, unsigned int j) const { return mArray[offset(i, j)]; } + GPUhdi() T& operator[](unsigned int i) { return mArray[off(i)]; } + GPUdi() T const& operator[](unsigned int i) const { return mArray[off(i)]; } + GPUdi() T apply(unsigned int i) const { return mArray[off(i)]; } GPUdi() T* Array() { return mArray; } GPUdi() const T* Array() const { return mArray; } @@ -469,6 +449,8 @@ class SMatrixGPU GPUdi() SMatrixGPU(SMatrixNoInit) {} GPUd() SMatrixGPU(SMatrixIdentity); GPUd() SMatrixGPU(const SMatrixGPU& rhs); + template + GPUd() SMatrixGPU(const SMatrixGPU& rhs); template GPUd() SMatrixGPU(const Expr& rhs); template @@ -480,14 +462,18 @@ class SMatrixGPU kCols = D2, // columns kSize = D1 * D2 // rows*columns }; - GPUd() T apply(unsigned int i) const; - GPUd() const T* Array() const; - GPUd() T* Array(); + // https://root.cern/doc/master/SMatrix_8icc_source.html#l00627 + GPUd() T apply(unsigned int i) const { return mRep[i]; } + GPUd() const T* Array() const { return mRep.Array(); } + GPUd() T* Array() { return mRep.Array(); } GPUd() iterator begin(); GPUd() iterator end(); GPUd() const T& operator()(unsigned int i, unsigned int j) const; GPUd() T& operator()(unsigned int i, unsigned int j); + template + GPUd() friend X& operator<<(Y& y, const SMatrixGPU&); + class SMatrixRowGPU { public: @@ -517,6 +503,14 @@ class SMatrixGPU GPUd() SMatrixRowGPUconst operator[](unsigned int i) const { return SMatrixRowGPUconst(*this, i); } GPUd() SMatrixRowGPU operator[](unsigned int i) { return SMatrixRowGPU(*this, i); } + template + GPUd() SMatrixGPU& operator+=(const SMatrixGPU& rhs); + GPUd() SMatrixGPU& operator*=(const T& rhs); + template + GPUd() SMatrixGPU& operator*=(const SMatrixGPU& rhs); + template + GPUd() SMatrixGPU& operator*=(const Expr& rhs); + GPUd() bool Invert(); GPUd() bool IsInUse(const T* p) const; @@ -524,6 +518,15 @@ class SMatrixGPU R mRep; }; +#ifndef __OPENCL__ // TODO: current C++ for OpenCL 2021 is at C++17, so no concepts. But we don't need this trick for OpenCL anyway, so we can just hide it. +template + requires(sizeof(typename X::traits_type::pos_type) != 0) // do not provide a template to fair::Logger, etc... (pos_type is a member type of all std::ostream classes) +GPUd() X& operator<<(Y& y, const SMatrixGPU&) +{ + return y; +} +#endif + template GPUdi() SMatrixGPU::SMatrixGPU(SMatrixIdentity) { @@ -547,6 +550,13 @@ GPUdi() SMatrixGPU::SMatrixGPU(const SMatrixGPU& rhs mRep = rhs.mRep; } +template +template +GPUd() SMatrixGPU::SMatrixGPU(const SMatrixGPU& rhs) +{ + operator=(rhs); +} + template GPUdi() T* SMatrixGPU::begin() { @@ -675,7 +685,7 @@ GPUdi() SMatrixGPU& SMatrixGPU::operator=(const Expr template template -GPUdi() SMatrixGPU& SMatrixGPU::operator=(const M& rhs) +GPUdi() SMatrixGPU& SMatrixGPU::operator=(const M & rhs) { mRep = rhs.mRep; return *this; @@ -980,7 +990,7 @@ GPUdi() void Inverter::InvertBunchKaufman(MatRepSymGPU& rhs, int& if // invert D(j:j+1,j:j+1) temp2 = *mjj * *(mjj + j + 1) - *(mjj + j) * *(mjj + j); if (temp2 == 0) { - printf("SymMatrix::bunch_invert: error in pivot choice"); + LOGF(error, "SymMatrix::bunch_invert: error in pivot choice"); } temp2 = 1. / temp2; // this quotient is guaranteed to exist by the choice @@ -1067,10 +1077,10 @@ GPUdi() void Inverter::InvertBunchKaufman(MatRepSymGPU& rhs, int& if } *mjj -= static_cast(temp2); } - } else //2x2 pivot, compute columns j and j-1 of the inverse + } else // 2x2 pivot, compute columns j and j-1 of the inverse { if (piv[j - 1] != 0) { - printf("error in piv %lf \n", piv[j - 1]); + LOGF(error, "error in piv %lf \n", static_cast(piv[j - 1])); } s = 2; if (j < nrow) { @@ -1344,7 +1354,7 @@ GPUdi() int Inverter::DfinvMatrix(MatRepStdGPU& rhs, unsigned int for (unsigned int i = 1; i < n; i++) { unsigned int ni = n - i; mIter mij = mi; - //int j; + // int j; for (unsigned j = 1; j <= i; j++) { s33 = *mij; mIter mikj = mi + n + j - 1; @@ -1398,6 +1408,37 @@ GPUdi() bool SMatrixGPU::Invert() return Inverter::Dinv((*this).mRep); } +template +template +GPUdi() SMatrixGPU& SMatrixGPU::operator+=(const SMatrixGPU& rhs) +{ + mRep += rhs.mRep; + return *this; +} + +template +GPUdi() SMatrixGPU& SMatrixGPU::operator*=(const T & rhs) +{ + for (unsigned int i = 0; i < R::kSize; ++i) { + mRep.Array()[i] *= rhs; + } + return *this; +} + +template +template +GPUdi() SMatrixGPU& SMatrixGPU::operator*=(const SMatrixGPU& rhs) +{ + return operator=(*this* rhs); +} + +template +template +GPUdi() SMatrixGPU& SMatrixGPU::operator*=(const Expr& rhs) +{ + return operator=(*this* rhs); +} + template struct TranspPolicyGPU { enum { @@ -1424,6 +1465,7 @@ class TransposeOpGPU { return mRhs.apply((i % D1) * D2 + i / D1); } + GPUdi() T operator()(unsigned int i, unsigned j) const { return mRhs(j, i); @@ -1467,5 +1509,5 @@ GPUdi() SMatrixGPU> Similarity(const SMatrixGPU +/// RS Cloned from the GPU/TPCFastTransformation, consider simply moving here + +#ifndef ALICEO2_GPUCOMMON_TPCFASTTRANSFORMATION_SYMMATRIXSOLVER_H +#define ALICEO2_GPUCOMMON_TPCFASTTRANSFORMATION_SYMMATRIXSOLVER_H + +#include "GPUCommonRtypes.h" +#include +#include +#include + +namespace o2::math_utils +{ + +/// Linear Equation Solver for a symmetric positive-definite matrix A[n x n]. +/// +/// A[n x n] * X [n x m] = B[n x m] +/// +/// A elements are stored in the upper triangle of A. +/// Thus A(i,j) and A(j,i) access the same element. +/// +class SymMatrixSolver +{ + public: + SymMatrixSolver(int N, int M) : mN(N), mM(M), mShift(mN + mM) + { + assert(N > 0 && M > 0); + mA.resize(mN * mShift, 0.); + } + + /// access to A elements + double& A(int i, int j) + { + auto ij = std::minmax(i, j); + assert(ij.first >= 0 && ij.second < mN); + return mA[ij.first * mShift + ij.second]; + } + + /// access to B elements + double& B(int i, int j) + { + assert(i >= 0 && i < mN && j >= 0 && j < mM); + return mA[i * mShift + mN + j]; + } + + /// + void solve(); + + /// + void print(); + + /// Test the class functionality. Returns 1 when ok, 0 when not ok + static int test(bool prn = 0); + + private: + private: + int mN = 0; + int mM = 0; + int mShift = 0; + std::vector mA; + + ClassDefNV(SymMatrixSolver, 0); +}; + +} // namespace o2::math_utils + +#endif diff --git a/Common/MathUtils/include/MathUtils/Tsallis.h b/Common/MathUtils/include/MathUtils/Tsallis.h new file mode 100644 index 0000000000000..328acd9e5db8b --- /dev/null +++ b/Common/MathUtils/include/MathUtils/Tsallis.h @@ -0,0 +1,48 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef ALICEO2_TSALLIS_H +#define ALICEO2_TSALLIS_H + +namespace o2 +{ + +namespace math_utils +{ + +struct Tsallis { + /// Tsallis/Hagedorn function describing charged pt spectra (m s = 62.4 GeV to 13 TeV) as in https://iopscience.iop.org/article/10.1088/2399-6528/aab00f/pdf + /// https://github.com/alisw/AliPhysics/blob/523f2dc8b45d913e9b7fda9b27e746819cbe5b09/PWGPP/AliAnalysisTaskFilteredTree.h#L145 + /// \param pt - transverse momentum + /// \param mass - mass of particle + /// \param sqrts - centre of mass energy + /// \return - invariant yields of the charged particle *pt + /// n(sqrts)= a + b/sqrt(s) - formula 6 + /// T(sqrts)= c + d/sqrt(s) - formula 7 + /// a = 6.81 ± 0.06 and b = 59.24 ± 3.53 GeV - for charged particles page 3 + /// c = 0.082 ± 0.002 GeV and d = 0.151 ± 0.048 (GeV) - for charged particles page 4 + static float tsallisCharged(float pt, float mass, float sqrts); + + /// Random downsampling trigger function using Tsallis/Hagedorn spectra fit (sqrt(s) = 62.4 GeV to 13 TeV) as in https://iopscience.iop.org/article/10.1088/2399-6528/aab00f/pdf + /// \return flat q/pt trigger + /// \param pt pat of particle + /// \param factorPt defines the sampling + /// \param sqrts centre of mass energy + /// \param weight weight which is internally calculated + /// \param rnd random value between (0->1) used to check for sampling + /// \param mass particles mass (use pion if not known) + static bool downsampleTsallisCharged(float pt, float factorPt, float sqrts, float& weight, float rnd, float mass = 0.13957); +}; + +} // namespace math_utils +} // namespace o2 + +#endif diff --git a/Common/MathUtils/include/MathUtils/Utils.h b/Common/MathUtils/include/MathUtils/Utils.h index 618d3e379f6d2..79263b4142216 100644 --- a/Common/MathUtils/include/MathUtils/Utils.h +++ b/Common/MathUtils/include/MathUtils/Utils.h @@ -128,17 +128,17 @@ GPUdi() void rotateZd(double xL, double yL, double& xG, double& yG, double snAlp return detail::rotateZ(xL, yL, xG, yG, snAlp, csAlp); } -#ifndef GPUCA_GPUCODE_DEVICE -inline void rotateZInv(float xG, float yG, float& xL, float& yL, float snAlp, float csAlp) +GPUdi() void rotateZInv(float xG, float yG, float& xL, float& yL, float snAlp, float csAlp) { detail::rotateZInv(xG, yG, xL, yL, snAlp, csAlp); } -inline void rotateZInvd(double xG, double yG, double& xL, double& yL, double snAlp, double csAlp) +GPUdi() void rotateZInvd(double xG, double yG, double& xL, double& yL, double snAlp, double csAlp) { detail::rotateZInv(xG, yG, xL, yL, snAlp, csAlp); } +#ifndef GPUCA_GPUCODE_DEVICE inline std::tuple rotateZInv(float xG, float yG, float snAlp, float csAlp) { return detail::rotateZInv(xG, yG, snAlp, csAlp); diff --git a/Common/MathUtils/include/MathUtils/detail/Bracket.h b/Common/MathUtils/include/MathUtils/detail/Bracket.h index c52de5a0e15c4..25d0dbd1d0a6e 100644 --- a/Common/MathUtils/include/MathUtils/detail/Bracket.h +++ b/Common/MathUtils/include/MathUtils/detail/Bracket.h @@ -18,8 +18,8 @@ #include #ifndef GPUCA_ALIGPUCODE -#include #include +#include #endif namespace o2 @@ -251,7 +251,9 @@ inline typename Bracket::Relation Bracket::isOutside(T t) const template std::string Bracket::asString() const { - return fmt::format("[{}:{}]", getMin(), getMax()); + std::stringstream tmp; + tmp << "[" << getMin() << ":" << getMax() << "]"; + return tmp.str(); } #endif diff --git a/Common/MathUtils/include/MathUtils/detail/StatAccumulator.h b/Common/MathUtils/include/MathUtils/detail/StatAccumulator.h index 50a09804da6ed..abb8a716cc5ee 100644 --- a/Common/MathUtils/include/MathUtils/detail/StatAccumulator.h +++ b/Common/MathUtils/include/MathUtils/detail/StatAccumulator.h @@ -66,6 +66,7 @@ struct StatAccumulator { sum += other.sum; sum2 += other.sum2; wsum += other.wsum; + n += other.n; return *this; } diff --git a/Common/MathUtils/include/MathUtils/detail/basicMath.h b/Common/MathUtils/include/MathUtils/detail/basicMath.h index 5dd66182e47d8..1abe6ee878c39 100644 --- a/Common/MathUtils/include/MathUtils/detail/basicMath.h +++ b/Common/MathUtils/include/MathUtils/detail/basicMath.h @@ -16,14 +16,15 @@ #ifndef MATHUTILS_INCLUDE_MATHUTILS_DETAIL_BASICMATH_H_ #define MATHUTILS_INCLUDE_MATHUTILS_DETAIL_BASICMATH_H_ +#include "GPUCommonDef.h" +#include "GPUCommonMath.h" +#include "CommonConstants/MathConstants.h" + #ifndef GPUCA_GPUCODE_DEVICE #include #include +#include #endif -#include "GPUCommonArray.h" -#include "GPUCommonDef.h" -#include "GPUCommonMath.h" -#include "CommonConstants/MathConstants.h" namespace o2 { @@ -65,7 +66,7 @@ GPUhdi() T abs(T x) template GPUdi() int nint(T x) { - return o2::gpu::GPUCommonMath::Nint(x); + return o2::gpu::GPUCommonMath::Float2IntRn(x); }; template @@ -113,7 +114,11 @@ GPUdi() int nint(double x) template <> GPUdi() bool finite(double x) { +#ifdef __FAST_MATH__ + return false; +#else return std::isfinite(x); +#endif } template <> GPUdi() double log(double x) @@ -126,4 +131,4 @@ GPUdi() double log(double x) } // namespace math_utils } // namespace o2 -#endif /* MATHUTILS_INCLUDE_MATHUTILS_DETAIL_BASICMATH_H_ */ \ No newline at end of file +#endif /* MATHUTILS_INCLUDE_MATHUTILS_DETAIL_BASICMATH_H_ */ diff --git a/Common/MathUtils/include/MathUtils/detail/trigonometric.h b/Common/MathUtils/include/MathUtils/detail/trigonometric.h index d624ca5c6bd67..457210202ca54 100644 --- a/Common/MathUtils/include/MathUtils/detail/trigonometric.h +++ b/Common/MathUtils/include/MathUtils/detail/trigonometric.h @@ -16,16 +16,17 @@ #ifndef MATHUTILS_INCLUDE_MATHUTILS_DETAIL_TRIGONOMETRIC_H_ #define MATHUTILS_INCLUDE_MATHUTILS_DETAIL_TRIGONOMETRIC_H_ -#ifndef GPUCA_GPUCODE_DEVICE -#include -#include -#endif -#include "GPUCommonArray.h" #include "GPUCommonDef.h" #include "GPUCommonMath.h" #include "CommonConstants/MathConstants.h" #include "MathUtils/detail/basicMath.h" +#ifndef GPUCA_GPUCODE_DEVICE +#include +#include +#include +#endif + namespace o2 { namespace math_utils @@ -156,7 +157,7 @@ GPUhdi() std::tuple rotateZInv(T xG, T yG, T snAlp, T csAlp) #endif template -GPUhdi() void rotateZ(gpu::gpustd::array& xy, T alpha) +GPUhdi() void rotateZ(std::array& xy, T alpha) { // transforms vector in tracking frame alpha to global frame T sin, cos; @@ -182,6 +183,12 @@ GPUhdi() void rotateZInv(T xG, T yG, T& xL, T& yL, T snAlp, T csAlp) rotateZ(xG, yG, xL, yL, -snAlp, csAlp); } +template +GPUhdi() constexpr T sectorDAlpha() +{ + return o2::constants::math::SectorSpanRad * 0.5; +} + template GPUhdi() int angle2Sector(T phi) { @@ -190,7 +197,7 @@ GPUhdi() int angle2Sector(T phi) if (phi < 0.f) { sect += o2::constants::math::NSectors - 1; } - return sect; + return sect % o2::constants::math::NSectors; } template @@ -209,6 +216,58 @@ GPUhdi() T angle2Alpha(T phi) return sector2Angle(angle2Sector(phi)); } +template +GPUhdi() constexpr bool okForPhiMin(T phiMin, T phi) +{ + // check if phi is above the phiMin, phi's must be in 0-2pi range + const T dphi = phi - phiMin; + return ((dphi > 0 && dphi < constants::math::PI) || dphi < -constants::math::PI) ? true : false; +} + +template +GPUhdi() constexpr bool okForPhiMax(T phiMax, T phi) +{ + // check if phi is below the phiMax, phi's must be in 0-2pi range + const T dphi = phi - phiMax; + return ((dphi < 0 && dphi > -constants::math::PI) || dphi > constants::math::PI) ? true : false; +} + +template +GPUhdi() constexpr T meanPhiSmall(T phi0, T phi1) +{ + // return mean phi, assume phis in 0:2pi + T phi; + if (!okForPhiMin(phi0, phi1)) { + phi = phi0; + phi0 = phi1; + phi1 = phi; + } + if (phi0 > phi1) { + phi = 0.5 * (phi1 - (constants::math::TwoPI - phi0)); // wrap + } else { + phi = 0.5 * (phi0 + phi1); + } + bringTo02Pi(phi); + return phi; +} + +template +GPUhdi() constexpr T deltaPhiSmall(T phi0, T phi1) +{ + // return delta phi, assume phi is in 0:2pi + T del; + if (!okForPhiMin(phi0, phi1)) { + del = phi0; + phi0 = phi1; + phi1 = del; + } + del = phi1 - phi0; + if (del < 0) { + del += constants::math::TwoPI; + } + return del; +} + template GPUhdi() T fastATan2(T y, T x) { diff --git a/Common/MathUtils/include/MathUtils/fit.h b/Common/MathUtils/include/MathUtils/fit.h index 1f7223605a99b..cd5cb415070d3 100644 --- a/Common/MathUtils/include/MathUtils/fit.h +++ b/Common/MathUtils/include/MathUtils/fit.h @@ -20,6 +20,7 @@ #include #include #include +#include #include "Rtypes.h" #include "TLinearFitter.h" @@ -59,7 +60,7 @@ template TFitResultPtr fit(const size_t nBins, const T* arr, const T xMin, const T xMax, TF1& func, std::string_view option = "") { Foption_t fitOption; - ROOT::Fit::FitOptionsMake(ROOT::Fit::kHistogram, option.data(), fitOption); + ROOT::Fit::FitOptionsMake(ROOT::Fit::EFitObjectType::kHistogram, option.data(), fitOption); ROOT::Fit::DataRange range(xMin, xMax); ROOT::Fit::DataOptions opt; @@ -69,9 +70,9 @@ TFitResultPtr fit(const size_t nBins, const T* arr, const T xMin, const T xMax, // create an empty TFitResult std::shared_ptr tfr(new TFitResult()); // create the fitter from an empty fit result - //std::shared_ptr fitter(new ROOT::Fit::Fitter(std::static_pointer_cast(tfr) ) ); + // std::shared_ptr fitter(new ROOT::Fit::Fitter(std::static_pointer_cast(tfr) ) ); ROOT::Fit::Fitter fitter(tfr); - //ROOT::Fit::FitConfig & fitConfig = fitter->Config(); + // ROOT::Fit::FitConfig & fitConfig = fitter->Config(); const double binWidth = double(xMax - xMin) / double(nBins); @@ -225,8 +226,8 @@ bool medmadGaus(size_t nBins, const T* arr, const T xMin, const T xMax, std::arr /// -1: only one point has been used for the calculation - center of gravity was uesed for calculation /// -4: invalid result!! /// -//template -//Double_t fitGaus(const size_t nBins, const T *arr, const T xMin, const T xMax, std::vector& param); +// template +// Double_t fitGaus(const size_t nBins, const T *arr, const T xMin, const T xMax, std::vector& param); template Double_t fitGaus(const size_t nBins, const T* arr, const T xMin, const T xMax, std::vector& param) { @@ -301,7 +302,7 @@ Double_t fitGaus(const size_t nBins, const T* arr, const T xMin, const T xMax, s Double_t chi2 = 0; if (npoints >= 3) { if (npoints == 3) { - //analytic calculation of the parameters for three points + // analytic calculation of the parameters for three points A.Invert(); TMatrixD res(1, 3); res.Mult(A, b); @@ -334,7 +335,7 @@ Double_t fitGaus(const size_t nBins, const T* arr, const T xMin, const T xMax, s } if (npoints == 2) { - //use center of gravity for 2 points + // use center of gravity for 2 points meanCOG /= sumCOG; rms2COG /= sumCOG; param[0] = max; @@ -524,7 +525,7 @@ R median(std::vector v) auto n = v.size() / 2; nth_element(v.begin(), v.begin() + n, v.end()); auto med = R{v[n]}; - if (!(v.size() & 1)) { //If the set size is even + if (!(v.size() & 1)) { // If the set size is even auto max_it = max_element(v.begin(), v.begin() + n); med = R{(*max_it + med) / 2.0}; } @@ -712,6 +713,245 @@ bool LTMUnbinnedSig(const std::vector& data, std::vector& index, std: params[4] = params[3] / std::sqrt(2.0); // error on RMS return true; } + +//___________________________________________________________________ +template +T selKthMin(int k, int np, T* arr) +{ + // Returns the k th smallest value in the array. The input array will be rearranged + // to have this value in location arr[k] , with all smaller elements moved before it + // (in arbitrary order) and all larger elements after (also in arbitrary order). + // From Numerical Recipes in C++ + + int i, j, mid, ir = np - 1, l = 0; + T a; + for (;;) { + if (ir <= l + 1) { + if (ir == l + 1 && arr[ir] < arr[l]) { + std::swap(arr[l], arr[ir]); + } + return arr[k]; + } else { + int mid = (l + ir) >> 1, i = l + 1; + std::swap(arr[mid], arr[i]); + if (arr[i] > arr[ir]) { + std::swap(arr[i], arr[ir]); + } + if (arr[l] > arr[ir]) { + std::swap(arr[l], arr[ir]); + } + if (arr[i] > arr[l]) { + std::swap(arr[i], arr[l]); + } + j = ir; + a = arr[l]; + for (;;) { + do { + i++; + } while (arr[i] < a); + do { + j--; + } while (arr[j] > a); + if (j < i) { + break; + } + std::swap(arr[i], arr[j]); + } + arr[l] = arr[j]; + arr[j] = a; + if (j >= k) { + ir = j - 1; + } + if (j <= k) { + l = i; + } + } + } +} + +//___________________________________________________________________ +template +T MAD2Sigma(int np, T* y) +{ + // Sigma calculated from median absolute deviations, https://en.wikipedia.org/wiki/Median_absolute_deviation + // the input array is modified + if (np < 2) { + return 0; + } + int nph = np >> 1; + float median = (np & 0x1) ? selKthMin(nph, np, y) : 0.5 * (selKthMin(nph - 1, np, y) + selKthMin(nph, np, y)); + // build abs differences to median + for (int i = np; i--;) { + y[i] = std::abs(y[i] - median); + } + // now get median of abs deviations + median = (np & 0x1) ? selKthMin(nph, np, y) : 0.5 * (selKthMin(nph - 1, np, y) + selKthMin(nph, np, y)); + return median * 1.4826; // convert to Gaussian sigma +} + +/// \return returns the index of the closest timestamps to the left and right of the given timestamp +/// \param timestamps vector of timestamps +/// \param timestamp the timestamp to find the closest timestamps for +template +std::optional> findClosestIndices(const std::vector& timestamps, DataTime timestamp) +{ + if (timestamps.empty()) { + LOGP(warning, "Timestamp vector is empty!"); + return std::nullopt; + } + + if (timestamp <= timestamps.front()) { + return std::pair{0, 0}; + } else if (timestamp >= timestamps.back()) { + return std::pair{timestamps.size() - 1, timestamps.size() - 1}; + } + + const auto it = std::lower_bound(timestamps.begin(), timestamps.end(), timestamp); + const size_t idx = std::distance(timestamps.begin(), it); + const auto prevTimestamp = timestamps[idx - 1]; + const auto nextTimestamp = timestamps[idx]; + return std::pair{(idx - 1), idx}; +} + +struct RollingStats { + RollingStats() = default; + RollingStats(const int nValues) + { + median.resize(nValues); + std.resize(nValues); + nPoints.resize(nValues); + closestDistanceL.resize(nValues); + closestDistanceR.resize(nValues); + } + + std::vector median; ///< median of rolling data + std::vector std; ///< std of rolling data + std::vector nPoints; ///< number of points used for the calculation + std::vector closestDistanceL; ///< distance of closest point to the left + std::vector closestDistanceR; ///< distance of closest point to the right + + ClassDefNV(RollingStats, 1); +}; + +/// \brief calculates the rolling statistics of the input data +/// \return returns the rolling statistics +/// \param timeData times of the input data (assumed to be sorted) +/// \param data values of the input data +/// \param times times for which to calculate the rolling statistics +/// \param deltaMax time range for which the rolling statistics is calculated +/// \param mNthreads number of threads to use for the calculation +/// \param minPoints minimum number of points to use for the calculation of the statistics - otherwise use nearest nClosestPoints points weighted with distance +/// \param nClosestPoints number of closest points in case of number of points in given range is smaller than minPoints +template +RollingStats getRollingStatistics(const DataTimeType& timeData, const DataType& data, const DataTime& times, const double deltaMax, const int mNthreads, const size_t minPoints = 4, const size_t nClosestPoints = 4) +{ + // output statistics + const size_t vecSize = times.size(); + RollingStats stats(vecSize); + + if (!std::is_sorted(timeData.begin(), timeData.end())) { + LOGP(error, "Input data is NOT sorted!"); + return stats; + } + + if (timeData.empty()) { + LOGP(error, "Input data is empty!"); + return stats; + } + + const size_t dataSize = data.size(); + const size_t timeDataSize = timeData.size(); + if (timeDataSize != dataSize) { + LOGP(error, "Input data has different sizes {}!={}", timeDataSize, dataSize); + return stats; + } + + auto myThread = [&](int iThread) { + // data in given time window for median calculation + DataType window; + for (size_t i = iThread; i < vecSize; i += mNthreads) { + const double timeI = times[i]; + + // lower index + const double timeStampLower = timeI - deltaMax; + const auto lower = std::lower_bound(timeData.begin(), timeData.end(), timeStampLower); + size_t idxStart = std::distance(timeData.begin(), lower); + + // upper index + const double timeStampUpper = timeI + deltaMax; + const auto upper = std::lower_bound(timeData.begin(), timeData.end(), timeStampUpper); + size_t idxEnd = std::distance(timeData.begin(), upper); + + // closest data point + if (auto idxClosest = findClosestIndices(timeData, timeI)) { + auto [idxLeft, idxRight] = *idxClosest; + const auto closestL = std::abs(timeData[idxLeft] - timeI); + const auto closestR = std::abs(timeData[idxRight] - timeI); + stats.closestDistanceL[i] = closestL; + stats.closestDistanceR[i] = closestR; + + // if no points are in the range use the n closest points - n from the left and n from the right + const size_t reqSize = idxEnd - idxStart; + if (reqSize < minPoints) { + // calculate weighted average + idxStart = (idxRight > nClosestPoints) ? (idxRight - nClosestPoints) : 0; + idxEnd = std::min(data.size(), idxRight + nClosestPoints); + constexpr float epsilon = 1e-6f; + double weightedSum = 0.0; + double weightTotal = 0.0; + for (size_t j = idxStart; j < idxEnd; ++j) { + const double dist = std::abs(timeI - timeData[j]); + const double weight = 1.0 / (dist + epsilon); + weightedSum += weight * data[j]; + weightTotal += weight; + } + stats.median[i] = (weightTotal > 0.) ? (weightedSum / weightTotal) : 0.0f; + } else { + // calculate statistics + stats.nPoints[i] = reqSize; + + if (idxStart >= data.size()) { + stats.median[i] = data.back(); + continue; + } + + if (reqSize <= 1) { + stats.median[i] = data[idxStart]; + continue; + } + + // calculate median + window.clear(); + if (reqSize > window.capacity()) { + window.reserve(static_cast(reqSize * 1.5)); + } + window.insert(window.end(), data.begin() + idxStart, data.begin() + idxEnd); + const size_t middle = window.size() / 2; + std::nth_element(window.begin(), window.begin() + middle, window.end()); + stats.median[i] = (window.size() % 2 == 1) ? window[middle] : ((window[middle - 1] + window[middle]) / 2.0); + + // calculate the stdev + const float mean = std::accumulate(window.begin(), window.end(), 0.0f) / window.size(); + std::transform(window.begin(), window.end(), window.begin(), [mean](const float val) { return val - mean; }); + const float sqsum = std::inner_product(window.begin(), window.end(), window.begin(), 0.0f); + const float stdev = std::sqrt(sqsum / window.size()); + stats.std[i] = stdev; + } + } + } + }; + + std::vector threads(mNthreads); + for (int i = 0; i < mNthreads; i++) { + threads[i] = std::thread(myThread, i); + } + + for (auto& th : threads) { + th.join(); + } + return stats; +} + } // namespace math_utils } // namespace o2 #endif diff --git a/Common/MathUtils/src/Chebyshev3D.cxx b/Common/MathUtils/src/Chebyshev3D.cxx index 0b0f7feba817b..1668281f8eca3 100644 --- a/Common/MathUtils/src/Chebyshev3D.cxx +++ b/Common/MathUtils/src/Chebyshev3D.cxx @@ -23,10 +23,10 @@ #include // for TSystem, gSystem #include // for printf, fprintf, FILE, fclose, fflush, etc #include "MathUtils/Chebyshev3DCalc.h" // for Chebyshev3DCalc, etc -#include "FairLogger.h" // for FairLogger #include "TMathBase.h" // for Max, Abs #include "TNamed.h" // for TNamed #include "TObjArray.h" // for TObjArray +#include // for FairLogger using namespace o2::math_utils; diff --git a/Common/MathUtils/src/MathUtilsLinkDef.h b/Common/MathUtils/src/MathUtilsLinkDef.h index a0aa954172e27..0b070e537afcd 100644 --- a/Common/MathUtils/src/MathUtilsLinkDef.h +++ b/Common/MathUtils/src/MathUtilsLinkDef.h @@ -33,11 +33,19 @@ #pragma link C++ class o2::math_utils::Rotation2Dd_t + ; #pragma link C++ class o2::math_utils::CachingTF1 + ; +#pragma link C++ class o2::math_utils::SymMatrixSolver + ; + #pragma link C++ class o2::math_utils::CircleXYf_t + ; #pragma link C++ class o2::math_utils::CircleXYd_t + ; #pragma link C++ class o2::math_utils::IntervalXYf_t + ; #pragma link C++ class o2::math_utils::IntervalXYd_t + ; #pragma link C++ class o2::math_utils::Bracketf_t + ; #pragma link C++ class o2::math_utils::Bracketd_t + ; +#pragma link C++ class o2::math_utils::Tsallis + ; + +#pragma link C++ class o2::math_utils::Legendre1DPolynominal + ; +#pragma link C++ class o2::math_utils::Legendre2DPolynominal + ; + +#pragma link C++ class o2::math_utils::RollingStats + ; #endif diff --git a/Common/MathUtils/src/SymMatrixSolver.cxx b/Common/MathUtils/src/SymMatrixSolver.cxx new file mode 100644 index 0000000000000..960aa119aa293 --- /dev/null +++ b/Common/MathUtils/src/SymMatrixSolver.cxx @@ -0,0 +1,183 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file SymMatrixSolver.cxx +/// \brief Implementation of SymMatrixSolver class +/// +/// \author Sergey Gorbunov +/// RS Cloned from the GPU/TPCFastTransformation, consider simply moving here + +#include "MathUtils/SymMatrixSolver.h" +#include "Framework/Logger.h" + +#include +#include +#include +#include + +using namespace o2::math_utils; + +void SymMatrixSolver::solve() +{ + // Upper Triangulization + for (int i = 0; i < mN; i++) { + double* rowI = &mA[i * mShift]; + double* rowIb = &mA[i * mShift + mN]; + double c = (std::fabs(rowI[i]) > 1.e-10) ? 1. / rowI[i] : 0.; + double* rowJ = rowI + mShift; + for (int j = i + 1; j < mN; j++, rowJ += mShift) { // row j + if (rowI[j] != 0.) { + double aij = c * rowI[j]; // A[i][j] / A[i][i] + for (int k = j; k < mShift; k++) { + rowJ[k] -= aij * rowI[k]; // A[j][k] -= A[i][k]/A[i][i]*A[j][i] + } + rowI[j] = aij; // A[i][j] /= A[i][i] + } + } + for (int k = 0; k < mM; k++) { + rowIb[k] *= c; + } + } + // Diagonalization + for (int i = mN - 1; i >= 0; i--) { + double* rowIb = &mA[i * mShift + mN]; + double* rowJb = rowIb - mShift; + for (int j = i - 1; j >= 0; j--, rowJb -= mShift) { // row j + double aji = mA[j * mShift + i]; + if (aji != 0.) { + for (int k = 0; k < mM; k++) { + rowJb[k] -= aji * rowIb[k]; + } + } + } + } +} + +void SymMatrixSolver::print() +{ + for (int i = 0; i < mN; i++) { + LOG(info) << ""; + for (int j = 0; j < mN; j++) { + LOG(info) << std::fixed << std::setw(5) << std::setprecision(2) << A(i, j) << " "; + } + LOG(info) << " | "; + for (int j = 0; j < mM; j++) { + LOG(info) << std::fixed << std::setw(5) << std::setprecision(2) << B(i, j) << " "; + } + } + LOG(info) << std::setprecision(-1); +} + +int SymMatrixSolver::test(bool prn) +{ + constexpr int n = 30; + constexpr int d = 3; + + // std::random_device rd; // Will be used to obtain a seed for the random + std::mt19937 gen(1); // Standard mersenne_twister_engine seeded with 1 + std::uniform_real_distribution<> uniform(-.999, .999); + + double maxDiff = 0.; + int nTries = 10000; + + auto tmpTime = std::chrono::high_resolution_clock::now(); + auto duration = std::chrono::duration_cast(tmpTime - tmpTime); + auto durationMult = duration; + + for (int iter = 0; iter < nTries; iter++) { + + double x[n][d]; + double A[n][n]; + { + for (int i = 0; i < n; i++) { + for (int j = 0; j < d; j++) { + x[i][j] = 1. * uniform(gen); + } + } + for (int i = 0; i < n; i++) { + A[i][i] = fabs(2. + uniform(gen)); + } + for (int i = 0; i < n; i++) { + for (int j = i + 1; j < n; j++) { + A[i][j] = A[i][i] * A[j][j] * uniform(gen); + A[j][i] = A[i][j]; + } + } + for (int i = 0; i < n; i++) { + A[i][i] = A[i][i] * A[i][i]; + } + if (prn && iter == nTries - 1) { + for (int i = 0; i < n; i++) { + LOG(info) << ""; + for (int j = 0; j < n; j++) { + LOG(info) << std::fixed << std::setw(5) << std::setprecision(2) << A[i][j] << " "; + } + } + LOG(info) << ""; + } + } + double b[n][d]; + auto startMult = std::chrono::high_resolution_clock::now(); + for (int i = 0; i < n; i++) { + for (int k = 0; k < d; k++) { + b[i][k] = 0.; + } + for (int j = 0; j < n; j++) { + for (int k = 0; k < d; k++) { + b[i][k] += x[j][k] * A[i][j]; + } + } + } + auto stopMult = std::chrono::high_resolution_clock::now(); + durationMult += std::chrono::duration_cast(stopMult - startMult); + + SymMatrixSolver sym(n, d); + + for (int i = 0; i < n; i++) { + for (int k = 0; k < d; k++) { + sym.B(i, k) = b[i][k]; + } + for (int j = i; j < n; j++) { + sym.A(i, j) = A[i][j]; + } + } + + auto start = std::chrono::high_resolution_clock::now(); + sym.solve(); + auto stop = std::chrono::high_resolution_clock::now(); + duration += std::chrono::duration_cast(stop - start); + + double diff = 0.; + for (int i = 0; i < n; i++) { + for (int k = 0; k < d; k++) { + double t = std::fabs(x[i][k] - sym.B(i, k)); + if (diff < t) { + diff = t; + } + } + } + if (maxDiff < diff) { + maxDiff = diff; + } + // LOG(info) << std::defaultfloat ; + // LOG(info) << "\n\n max diff " < + +namespace o2::math_utils +{ + +float Tsallis::tsallisCharged(float pt, float mass, float sqrts) +{ + const float a = 6.81; + const float b = 59.24; + const float c = 0.082; + const float d = 0.151; + const float mt = std::sqrt(mass * mass + pt * pt); + const float n = a + b / sqrts; + const float T = c + d / sqrts; + const float p0 = n * T; + return std::pow((1. + mt / p0), -n) * pt; +} + +bool Tsallis::downsampleTsallisCharged(float pt, float factorPt, float sqrts, float& weight, float rnd, float mass) +{ + const float prob = tsallisCharged(pt, mass, sqrts); + const float probNorm = tsallisCharged(1., mass, sqrts); + weight = prob / probNorm; + return (rnd * (weight * pt * pt)) < factorPt; +} + +} // namespace o2::math_utils diff --git a/Common/SimConfig/CMakeLists.txt b/Common/SimConfig/CMakeLists.txt index c6a6ed7da6c4c..f8e007209eacc 100644 --- a/Common/SimConfig/CMakeLists.txt +++ b/Common/SimConfig/CMakeLists.txt @@ -12,20 +12,29 @@ o2_add_library(SimConfig SOURCES src/SimConfig.cxx src/SimParams.cxx + src/SimDLLoader.cxx src/SimUserDecay.cxx - src/DigiParams.cxx src/G4Params.cxx + src/DigiParams.cxx + src/G4Params.cxx + src/DetectorLists.cxx src/MatMapParams.cxx + src/InteractionDiamondParam.cxx + src/GlobalProcessCutSimParam.cxx PUBLIC_LINK_LIBRARIES O2::CommonUtils - O2::DetectorsCommonDataFormats + O2::DetectorsCommonDataFormats O2::SimulationDataFormat FairRoot::Base Boost::program_options) o2_target_root_dictionary(SimConfig HEADERS include/SimConfig/SimConfig.h include/SimConfig/SimParams.h + include/SimConfig/SimDLLoader.h include/SimConfig/SimUserDecay.h - include/SimConfig/DigiParams.h + include/SimConfig/InteractionDiamondParam.h + include/SimConfig/DigiParams.h include/SimConfig/G4Params.h + include/SimConfig/DetectorLists.h + include/SimConfig/GlobalProcessCutSimParam.h include/SimConfig/MatMapParams.h) o2_add_test(Config diff --git a/Common/SimConfig/include/SimConfig/DetectorLists.h b/Common/SimConfig/include/SimConfig/DetectorLists.h new file mode 100644 index 0000000000000..bdabe71db0872 --- /dev/null +++ b/Common/SimConfig/include/SimConfig/DetectorLists.h @@ -0,0 +1,37 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef O2_DETECTORLISTS_H_ +#define O2_DETECTORLISTS_H_ + +#include +#include +#include + +#include "Framework/Logger.h" + +namespace o2::conf +{ +// Container defining different general evolutions of the ALICE experiment. Each +// evolution is given a name and a list defining the names of the detectors and +// passive elements present. +using DetectorList_t = std::vector; +using DetectorMap_t = std::unordered_map; + +// Parse the detector map from a JSON file. +// Return false if parsing failed. +bool parseDetectorMapfromJSON(const std::string& path, DetectorMap_t& map); + +// Print the DetetectorMap +void printDetMap(const DetectorMap_t& map, const std::string& list = ""); +} // namespace o2::conf + +#endif // O2_DETECTORLISTS_H_ diff --git a/Common/SimConfig/include/SimConfig/DigiParams.h b/Common/SimConfig/include/SimConfig/DigiParams.h index e8c99c91d0826..7692ceff057ea 100644 --- a/Common/SimConfig/include/SimConfig/DigiParams.h +++ b/Common/SimConfig/include/SimConfig/DigiParams.h @@ -30,7 +30,10 @@ struct DigiParams : public o2::conf::ConfigurableParamHelper { std::string digitizationgeometry_prefix = ""; // with which geometry prefix we digitized -> leave empty as this needs to be filled by the digitizer workflow std::string grpfile = ""; // which GRP file to use --> leave empty as this needs to be filled by the digitizer workflow bool mctruth = true; // whether to create labels - + int maxOrbitsToDigitize = -1; // Digitizer can use this to truncate digits that fall beyond an orbit limit (relative to start of digization) given by this param; -1 means no limit imposed + // This parameter should typically be set to coincide with a single timeframe length or multiples thereof. + std::string passName = "unanchored"; // passName for anchored MC + int seed = 0; // rndSeed to be applied in digitization; convention is that 0 is time based O2ParamDef(DigiParams, "DigiParams"); }; diff --git a/Common/SimConfig/include/SimConfig/G4Params.h b/Common/SimConfig/include/SimConfig/G4Params.h index 16547a685c6d4..aa8aa05263c0a 100644 --- a/Common/SimConfig/include/SimConfig/G4Params.h +++ b/Common/SimConfig/include/SimConfig/G4Params.h @@ -22,14 +22,33 @@ namespace conf // enumerating the possible G4 physics settings enum class EG4Physics { - kFTFP_BERT_optical = 0, /* just ordinary */ - kFTFP_BERT_optical_biasing = 1, /* with biasing enabled */ - kFTFP_INCLXX_optical = 2 /* special INCL++ version */ + kFTFP_BERT_optical = 0, /* just ordinary */ + kFTFP_BERT_optical_biasing = 1, /* with biasing enabled */ + kFTFP_INCLXX_optical = 2, /* special INCL++ version */ + kFTFP_BERT_HP_optical = 3, /* enable low energy neutron transport */ + kFTFP_BERT_EMV_optical = 4, /* just ordinary with faster electromagnetic physics */ + kFTFP_BERT_EMV_optical_biasing = 5, /* with biasing enabled with faster electromagnetic physics */ + kFTFP_INCLXX_EMV_optical = 6, /* special INCL++ version */ + kFTFP_BERT_EMV_HP_optical = 7, /* enable low energy neutron transport */ + kUSER = 8 /* allows to give own string combination */ +}; + +// enumerating possible geometry navigation modes +// (understanding that geometry description is always done with TGeo) +enum class EG4Nav { + kTGeo = 0, /* navigate with TGeo */ + kG4 = 1 /* navigate with G4 native geometry */ }; // parameters to influence the G4 engine struct G4Params : public o2::conf::ConfigurableParamHelper { - EG4Physics physicsmode = EG4Physics::kFTFP_BERT_optical; // physics mode with which to configure G4 + EG4Physics physicsmode = EG4Physics::kFTFP_BERT_EMV_optical; // default physics mode with which to configure G4 + + std::string configMacroFile = ""; // a user provided g4Config.in file (otherwise standard one fill be taken) + std::string userPhysicsList = ""; // possibility to directly give physics list as string + + EG4Nav navmode = EG4Nav::kTGeo; // geometry navigation mode (default TGeo) + std::string const& getPhysicsConfigString() const; O2ParamDef(G4Params, "G4"); diff --git a/Detectors/gconfig/include/SimSetup/GlobalProcessCutSimParam.h b/Common/SimConfig/include/SimConfig/GlobalProcessCutSimParam.h similarity index 97% rename from Detectors/gconfig/include/SimSetup/GlobalProcessCutSimParam.h rename to Common/SimConfig/include/SimConfig/GlobalProcessCutSimParam.h index c8bacfa7c4e7c..c9811c727cb32 100644 --- a/Detectors/gconfig/include/SimSetup/GlobalProcessCutSimParam.h +++ b/Common/SimConfig/include/SimConfig/GlobalProcessCutSimParam.h @@ -43,7 +43,7 @@ struct GlobalProcessCutSimParam : public o2::conf::ConfigurableParamHelper 1 MeV double DCUTM = 1.0E-3; // GeV --> 1 MeV double PPCUTM = 1.0E-3; // GeV --> 1 MeV - double TOFMAX = 1.E10; // seconds + double TOFMAX = 0.1; // seconds // boilerplate stuff + make principal key "GlobalSimProcs" O2ParamDef(GlobalProcessCutSimParam, "GlobalSimProcs"); diff --git a/Generators/include/Generators/InteractionDiamondParam.h b/Common/SimConfig/include/SimConfig/InteractionDiamondParam.h similarity index 84% rename from Generators/include/Generators/InteractionDiamondParam.h rename to Common/SimConfig/include/SimConfig/InteractionDiamondParam.h index 1a8f0504dd984..59eabbfe73ffe 100644 --- a/Generators/include/Generators/InteractionDiamondParam.h +++ b/Common/SimConfig/include/SimConfig/InteractionDiamondParam.h @@ -30,12 +30,14 @@ enum class EVertexDistribution { /** ** a parameter class/struct to keep the settings of - ** the interaction diamond (position and width) and - ** allow the user to modify them + ** the interaction diamond (position and width) and + ** allow the user to modify them **/ struct InteractionDiamondParam : public o2::conf::ConfigurableParamHelper { double position[3] = {0., 0., 0.}; - double width[3] = {0., 0., 0.}; + double width[3] = {0.01, 0.01, 0.01}; + double slopeX = 0.; // z-dependent x pos (see MeanVertexObject) + double slopeY = 0.; // z-dependent y pos (see MeanVertexObject) EVertexDistribution distribution = EVertexDistribution::kGaus; O2ParamDef(InteractionDiamondParam, "Diamond"); }; diff --git a/Common/SimConfig/include/SimConfig/SimConfig.h b/Common/SimConfig/include/SimConfig/SimConfig.h index 8a76bcbfc16b1..be88d9fbd8c33 100644 --- a/Common/SimConfig/include/SimConfig/SimConfig.h +++ b/Common/SimConfig/include/SimConfig/SimConfig.h @@ -13,20 +13,35 @@ #define O2_SIM_CONFIGURATION #include +#ifndef __CLING__ #include +#else +namespace boost::program_options +{ +class variables_map; +class options_description; +} // namespace boost::program_options +#endif namespace o2 { namespace conf { -enum SimFieldMode { +enum class SimFieldMode { kDefault = 0, kUniform = 1, kCCDB = 2 }; -enum TimeStampMode { +enum class VertexMode { + kNoVertex = 0, // no vertexing should be applied in the generator + kDiamondParam = 1, // Diamond param will influence vertexing + kCCDB = 2, // vertex should be taken from CCDB (Calib/MeanVertex object) + kCollCxt = 3 // vertex should be taken from collision context +}; + +enum class TimeStampMode { kNow = 0, kManual = 1, kRun = 2 @@ -34,39 +49,45 @@ enum TimeStampMode { // configuration struct (which can be passed around) struct SimConfigData { - std::vector mActiveModules; // list of active modules - std::vector mReadoutDetectors; // list of readout detectors - std::string mMCEngine; // chosen VMC engine - std::string mGenerator; // chosen VMC generator - std::string mTrigger; // chosen VMC generator trigger - unsigned int mNEvents; // number of events to be simulated - std::string mExtKinFileName; // file name of external kinematics file (needed for ext kinematics generator) - std::string mEmbedIntoFileName; // filename containing the reference events to be used for the embedding - unsigned int mStartEvent; // index of first event to be taken - float mBMax; // maximum for impact parameter sampling - bool mIsMT; // chosen MT mode (Geant4 only) - std::string mOutputPrefix; // prefix to be used for output files - std::string mLogSeverity; // severity for FairLogger - std::string mLogVerbosity; // loglevel for FairLogger - std::string mKeyValueTokens; // a string holding arbitrary sequence of key-value tokens - // Foo.parameter1=x,Bar.parameter2=y,Baz.paramter3=hello - // (can be used to **loosely** change any configuration parameter from command-line) - std::string mConfigFile; // path to a JSON or INI config file (file extension is required to determine type). - // values within the config file will override values set in code by the param classes - // but will themselves be overridden by any values given in mKeyValueTokens. - int mPrimaryChunkSize; // defining max granularity for input primaries of a sim job - int mInternalChunkSize; // - int mStartSeed; // base for random number seeds - int mSimWorkers = 1; // number of parallel sim workers (when it applies) - bool mFilterNoHitEvents = false; // whether to filter out events not leaving any response - std::string mCCDBUrl; // the URL where to find CCDB - uint64_t mTimestamp; // timestamp in ms to anchor transport simulation to - TimeStampMode mTimestampMode = kNow; // telling of timestamp was given as option or defaulted to now - int mRunNumber = -1; // ALICE run number (if set != -1); the timestamp should be compatible - int mField; // L3 field setting in kGauss: +-2,+-5 and 0 - SimFieldMode mFieldMode = kDefault; // uniform magnetic field - bool mAsService = false; // if simulation should be run as service/deamon (does not exit after run) - bool mNoGeant = false; // if Geant transport should be turned off (when one is only interested in the generated events) + std::vector mActiveModules; // list of active modules + std::vector mReadoutDetectors; // list of readout detectors + std::string mMCEngine; // chosen VMC engine + std::string mGenerator; // chosen VMC generator + std::string mTrigger; // chosen VMC generator trigger + unsigned int mNEvents; // number of events to be simulated + std::string mExtKinFileName; // file name of external kinematics file (needed for ext kinematics generator) + std::string mEmbedIntoFileName; // filename containing the reference events to be used for the embedding + unsigned int mStartEvent; // index of first event to be taken + float mBMax; // maximum for impact parameter sampling + bool mIsMT; // chosen MT mode (Geant4 only) + std::string mOutputPrefix; // prefix to be used for output files + std::string mLogSeverity; // severity for FairLogger + std::string mLogVerbosity; // loglevel for FairLogger + std::string mKeyValueTokens; // a string holding arbitrary sequence of key-value tokens + // Foo.parameter1=x,Bar.parameter2=y,Baz.paramter3=hello + // (can be used to **loosely** change any configuration parameter from command-line) + std::string mConfigFile; // path to a JSON or INI config file (file extension is required to determine type). + // values within the config file will override values set in code by the param classes + // but will themselves be overridden by any values given in mKeyValueTokens. + unsigned int mPrimaryChunkSize; // defining max granularity for input primaries of a sim job + int mInternalChunkSize; // + ULong_t mStartSeed; // base for random number seeds + int mSimWorkers = 1; // number of parallel sim workers (when it applies) + bool mFilterNoHitEvents = false; // whether to filter out events not leaving any response + std::string mCCDBUrl; // the URL where to find CCDB + uint64_t mTimestamp; // timestamp in ms to anchor transport simulation to + TimeStampMode mTimestampMode = TimeStampMode::kNow; // telling of timestamp was given as option or defaulted to now + int mRunNumber = -1; // ALICE run number (if set != -1); the timestamp should be compatible + int mField; // L3 field setting in kGauss: +-2,+-5 and 0 + SimFieldMode mFieldMode = SimFieldMode::kDefault; // uniform magnetic field + bool mAsService = false; // if simulation should be run as service/deamon (does not exit after run) + bool mNoGeant = false; // if Geant transport should be turned off (when one is only interested in the generated events) + bool mIsUpgrade = false; // true if the simulation is for Run 5 + std::string mFromCollisionContext = ""; // string denoting a collision context file; If given, this file will be used to determine number of events + // + bool mForwardKine = false; // true if tracks and event headers are to be published on a FairMQ channel (for reading by other consumers) + bool mWriteToDisc = true; // whether we write simulation products (kine, hits) to disc + VertexMode mVertexMode = VertexMode::kDiamondParam; // by default we should use die InteractionDiamond parameter ClassDefNV(SimConfigData, 4); }; @@ -93,7 +114,13 @@ class SimConfig return conf; } - static void initOptions(boost::program_options::options_description&); + // makes a new instance that can be used as a local object + static SimConfig make() + { + return SimConfig(); + } + + static void initOptions(boost::program_options::options_description&, bool isUpgrade = false); // initializes the configuration from command line arguments // returns true of correctly initialized and not --help called @@ -114,11 +141,14 @@ class SimConfig // static helper functions to determine list of active / readout modules // can also be used from outside - static void determineActiveModules(std::vector const& input, std::vector const& skipped, std::vector& active); + static void determineActiveModules(std::vector const& input, std::vector const& skipped, std::vector& active, bool isUpgrade = false); + static bool determineActiveModulesList(const std::string& version, std::vector const& input, std::vector const& skipped, std::vector& active); static void determineReadoutDetectors(std::vector const& active, std::vector const& enabledRO, std::vector const& skippedRO, std::vector& finalRO); // helper to parse field option static bool parseFieldString(std::string const& fieldstring, int& fieldvalue, o2::conf::SimFieldMode& mode); + // helper to parse vertex option; returns true if parsing ok, false if failure + static bool parseVertexModeString(std::string const& vertexstring, o2::conf::VertexMode& mode); // get selected generator (to be used to select a genconfig) std::string getGenerator() const { return mConfigData.mGenerator; } @@ -137,17 +167,31 @@ class SimConfig std::string getConfigFile() const { return mConfigData.mConfigFile; } int getPrimChunkSize() const { return mConfigData.mPrimaryChunkSize; } int getInternalChunkSize() const { return mConfigData.mInternalChunkSize; } - int getStartSeed() const { return mConfigData.mStartSeed; } + ULong_t getStartSeed() const { return mConfigData.mStartSeed; } int getNSimWorkers() const { return mConfigData.mSimWorkers; } bool isFilterOutNoHitEvents() const { return mConfigData.mFilterNoHitEvents; } bool asService() const { return mConfigData.mAsService; } uint64_t getTimestamp() const { return mConfigData.mTimestamp; } int getRunNumber() const { return mConfigData.mRunNumber; } bool isNoGeant() const { return mConfigData.mNoGeant; } + void setRun5(bool value = true) { mConfigData.mIsUpgrade = value; } + bool forwardKine() const { return mConfigData.mForwardKine; } + bool writeToDisc() const { return mConfigData.mWriteToDisc; } + VertexMode getVertexMode() const { return mConfigData.mVertexMode; } + + // returns the pair of collision context filename as well as event prefix encoded + // in the mFromCollisionContext string. Returns empty string if information is not available or set. + std::pair getCollContextFilenameAndEventPrefix() const; private: SimConfigData mConfigData; //! + // Filter out skipped elements in the list + static bool filterSkippedElements(std::vector& elements, std::vector const& skipped); + + // adjust/overwrite some option settings when collision context is used + void adjustFromCollContext(std::string const& collcontextfile, std::string const& prefix); + ClassDefNV(SimConfig, 1); }; @@ -172,9 +216,10 @@ struct SimReconfigData { std::string configFile; // path to a JSON or INI config file (file extension is required to determine type). // values within the config file will override values set in code by the param classes // but will themselves be overridden by any values given in mKeyValueTokens. - unsigned int primaryChunkSize; // defining max granularity for input primaries of a sim job - int startSeed; // base for random number seeds - bool stop; // to shut down the service + unsigned int primaryChunkSize; // defining max granularity for input primaries of a sim job + ULong_t startSeed; // base for random number seeds + bool stop; // to shut down the service + std::string mFromCollisionContext = ""; // string denoting a collision context file; If given, this file will be used to determine number of events ClassDefNV(SimReconfigData, 1); }; diff --git a/Common/SimConfig/include/SimConfig/SimDLLoader.h b/Common/SimConfig/include/SimConfig/SimDLLoader.h new file mode 100644 index 0000000000000..e778023b28276 --- /dev/null +++ b/Common/SimConfig/include/SimConfig/SimDLLoader.h @@ -0,0 +1,27 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef SIMDLLOADER_H_ +#define SIMDLLOADER_H_ + +#include "CommonUtils/DLLoaderBase.h" + +namespace o2::conf +{ + +class SimDLLoader : public o2::utils::DLLoaderBase +{ + O2DLLoaderDef(SimDLLoader) +}; + +} // namespace o2::conf + +#endif // SIMDLLOADER_H_ diff --git a/Common/SimConfig/include/SimConfig/SimParams.h b/Common/SimConfig/include/SimConfig/SimParams.h index 69ed040430d8d..b5f975d1b0c6e 100644 --- a/Common/SimConfig/include/SimConfig/SimParams.h +++ b/Common/SimConfig/include/SimConfig/SimParams.h @@ -23,6 +23,9 @@ namespace conf // (mostly used in O2MCApplication stepping) struct SimCutParams : public o2::conf::ConfigurableParamHelper { bool stepFiltering = true; // if we activate the step filtering in O2BaseMCApplication + bool stepTrackRefHook = false; // if we create track references during generic stepping + std::string stepTrackRefHookFile = "${O2_ROOT}/share/Detectors/gconfig/StandardSteppingTrackRefHook.macro"; // the standard code holding the TrackRef callback + bool trackSeed = false; // per track seeding for track-reproducible mode double maxRTracking = 1E20; // max R tracking cut in cm (in the VMC sense) -- applied in addition to cutting in the stepping function @@ -33,16 +36,19 @@ struct SimCutParams : public o2::conf::ConfigurableParamHelper { float maxRTrackingZDC = 50; // R-cut applied in the tunnel leading to ZDC when z > beampipeZ (custom stepping function) float tunnelZ = 1900; // Z-value from where we apply maxRTrackingZDC (default value taken from standard "hall" dimensions) - float globalDensityFactor = 1.f; // global factor that scales all material densities for systematic studies - + bool lowneut = false; O2ParamDef(SimCutParams, "SimCutParams"); }; // parameter influencing material manager struct SimMaterialParams : public o2::conf::ConfigurableParamHelper { // Local density value takes precedence over global density value, i.e. local values overwrite the global value. - float globalDensityFactor = 1.f; - std::string localDensityFactor; // Expected format: "SimMaterialParams.localDensityFactor=:,:,..." + float globalDensityFactor = 1.f; // global factor that scales all material densities for systematic studies + // String to set densities on module or material level. Expected format: + // "SimMaterialParams.localDensityFactor=:,:,..." + // Example: "SimMaterialParams.localDensityFactor=TPC/Air:1.2,ITS:5." will scale the density of the Air in TPC + // with 1.2 and to 5.0 for all materials in ITS". + std::string localDensityFactor; O2ParamDef(SimMaterialParams, "SimMaterialParams"); }; diff --git a/Common/SimConfig/src/DetectorLists.cxx b/Common/SimConfig/src/DetectorLists.cxx new file mode 100644 index 0000000000000..c9132a3cb84a5 --- /dev/null +++ b/Common/SimConfig/src/DetectorLists.cxx @@ -0,0 +1,78 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include "SimConfig/DetectorLists.h" +#include +#include +#include +#include +#include + +namespace o2::conf +{ + +bool parseDetectorMapfromJSON(const std::string& path, DetectorMap_t& map) +{ + // Parse JSON file to build map + std::ifstream fileStream(path, std::ios::in); + if (!fileStream.is_open()) { + LOGP(error, "Cannot open '{}'!", path); + return false; + } + rapidjson::IStreamWrapper isw(fileStream); + rapidjson::Document doc; + doc.ParseStream(isw); + if (doc.HasParseError()) { + LOGP(error, "Error parsing provided json file '{}':", path); + LOGP(error, " - Error -> {}", rapidjson::GetParseError_En(doc.GetParseError())); + LOGP(error, " - Offset -> {}", doc.GetErrorOffset()); + return false; + } + + // Clear and rebuild map + map.clear(); + try { + for (auto verItr = doc.MemberBegin(); verItr != doc.MemberEnd(); ++verItr) { + const auto& version = verItr->name.GetString(); + DetectorList_t list; + const auto& elements = doc[version]; + for (const auto& ele : elements.GetArray()) { + list.emplace_back(ele.GetString()); + } + map.emplace(version, list); + } + } catch (const std::exception& e) { + LOGP(error, "Failed to build detector map from file '{}' with '{}'", path, e.what()); + return false; + } + + return true; +} + +void printDetMap(const DetectorMap_t& map, const std::string& list) +{ + if (list.empty()) { + LOGP(error, "List of all available versions including their detectors:"); + for (int i{0}; const auto& [version, elements] : map) { + LOGP(error, " - {: >2d}. {}:", i++, version); + for (int j{0}; const auto& element : elements) { + LOGP(error, "\t\t* {: >2d}.\t{}", j++, element); + } + } + } else { + LOGP(error, "List of available modules for version {}:", list); + for (int j{0}; const auto& element : map.at(list)) { + LOGP(error, "\t* {: >2d}.\t{}", j++, element); + } + } +} + +} // namespace o2::conf diff --git a/Common/SimConfig/src/G4Params.cxx b/Common/SimConfig/src/G4Params.cxx index 5c5fe26751cd3..37625d914a85b 100644 --- a/Common/SimConfig/src/G4Params.cxx +++ b/Common/SimConfig/src/G4Params.cxx @@ -19,11 +19,21 @@ namespace conf namespace { -static const std::string confstrings[3] = {"FTFP_BERT_EMV+optical", "FTFP_BERT_EMV+optical+biasing", "FTFP_INCLXX_EMV+optical"}; +static const std::string confstrings[8] = {"FTFP_BERT+optical", + "FTFP_BERT+optical+biasing", + "FTFP_INCLXX+optical", + "FTFP_BERT_HP+optical", + "FTFP_BERT_EMV+optical", + "FTFP_BERT_EMV+optical+biasing", + "FTFP_INCLXX_EMV+optical", + "FTFP_BERT_HP_EMV+optical"}; } std::string const& G4Params::getPhysicsConfigString() const { + if (physicsmode == o2::conf::EG4Physics::kUSER) { + return userPhysicsList; + } return confstrings[(int)physicsmode]; } diff --git a/Detectors/gconfig/src/GlobalProcessCutSimParam.cxx b/Common/SimConfig/src/GlobalProcessCutSimParam.cxx similarity index 92% rename from Detectors/gconfig/src/GlobalProcessCutSimParam.cxx rename to Common/SimConfig/src/GlobalProcessCutSimParam.cxx index 4f30020133996..5ae72164bb81d 100644 --- a/Detectors/gconfig/src/GlobalProcessCutSimParam.cxx +++ b/Common/SimConfig/src/GlobalProcessCutSimParam.cxx @@ -9,5 +9,5 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. -#include "SimSetup/GlobalProcessCutSimParam.h" +#include "SimConfig/GlobalProcessCutSimParam.h" O2ParamImpl(o2::GlobalProcessCutSimParam); diff --git a/Generators/src/InteractionDiamondParam.cxx b/Common/SimConfig/src/InteractionDiamondParam.cxx similarity index 93% rename from Generators/src/InteractionDiamondParam.cxx rename to Common/SimConfig/src/InteractionDiamondParam.cxx index 36f5ae1a4cecf..70d5c4c6b23d6 100644 --- a/Generators/src/InteractionDiamondParam.cxx +++ b/Common/SimConfig/src/InteractionDiamondParam.cxx @@ -11,5 +11,5 @@ /// \author R+Preghenella - October 2018 -#include "Generators/InteractionDiamondParam.h" +#include "SimConfig/InteractionDiamondParam.h" O2ParamImpl(o2::eventgen::InteractionDiamondParam); diff --git a/Common/SimConfig/src/SimConfig.cxx b/Common/SimConfig/src/SimConfig.cxx index 69ebdc24f3b7b..15879687872d5 100644 --- a/Common/SimConfig/src/SimConfig.cxx +++ b/Common/SimConfig/src/SimConfig.cxx @@ -10,10 +10,12 @@ // or submit itself to any jurisdiction. #include +#include #include +#include #include #include -#include +#include #include #include #include @@ -22,8 +24,15 @@ using namespace o2::conf; namespace bpo = boost::program_options; -void SimConfig::initOptions(boost::program_options::options_description& options) +void SimConfig::initOptions(boost::program_options::options_description& options, bool isUpgrade) { + // some default args might depend on whether Run3 or Run5 + // can be updated here: + std::string defaultGeomList{"ALICE2"}; + if (isUpgrade == true) { + defaultGeomList = "ALICE3"; + } + int nsimworkersdefault = std::max(1u, std::thread::hardware_concurrency() / 2); options.add_options()( "mcEngine,e", bpo::value()->default_value("TGeant4"), "VMC backend to be used.")( @@ -33,7 +42,15 @@ void SimConfig::initOptions(boost::program_options::options_description& options "skipModules", bpo::value>()->multitoken()->default_value(std::vector({""}), ""), "list of modules excluded in geometry (precendence over -m")( "readoutDetectors", bpo::value>()->multitoken()->default_value(std::vector(), ""), "list of detectors creating hits, all if not given; added to to active modules")( "skipReadoutDetectors", bpo::value>()->multitoken()->default_value(std::vector(), ""), "list of detectors to skip hit creation (precendence over --readoutDetectors")( - "nEvents,n", bpo::value()->default_value(1), "number of events")( + "detectorList", bpo::value()->default_value(defaultGeomList), + "Use a specific version of ALICE, e.g., a predefined list." + "There is an 'official' list provided with:" + "\nALICE2 : The default configuration for Run 3" + "\nALICE2.1: The future configuration for Run 4" + "\nALICE3 : The far-future configuration for Run 5-6" + "\nAdditionally one can provide their own custom list of modules which should be included in the geometry." + "\nBy specifiying LIST:JSONFILE where LIST is a list present in JSONFILE.")( + "nEvents,n", bpo::value()->default_value(0), "number of events")( "startEvent", bpo::value()->default_value(0), "index of first event to be used (when applicable)")( "extKinFile", bpo::value()->default_value("Kinematics.root"), "name of kinematics file for event generator from file (when applicable)")( @@ -48,52 +65,168 @@ void SimConfig::initOptions(boost::program_options::options_description& options "configFile", bpo::value()->default_value(""), "Path to an INI or JSON configuration file")( "chunkSize", bpo::value()->default_value(500), "max size of primary chunk (subevent) distributed by server")( "chunkSizeI", bpo::value()->default_value(-1), "internalChunkSize")( - "seed", bpo::value()->default_value(-1), "initial seed (default: -1 random)")( - "field", bpo::value()->default_value("-5"), "L3 field rounded to kGauss, allowed values +-2,+-5 and 0; +-U for uniform field; \"ccdb\" for taking it from CCDB ")( + "seed", bpo::value()->default_value(0), "initial seed as ULong_t (default: 0 == random)")( + "field", bpo::value()->default_value("-5"), "L3 field rounded to kGauss, allowed values +-2,+-5 and 0; +-U for uniform field; \"ccdb\" for taking it from CCDB ")("vertexMode", bpo::value()->default_value("kDiamondParam"), "Where the beam-spot vertex should come from. Must be one of kNoVertex, kDiamondParam, kCCDB")( "nworkers,j", bpo::value()->default_value(nsimworkersdefault), "number of parallel simulation workers (only for parallel mode)")( "noemptyevents", "only writes events with at least one hit")( "CCDBUrl", bpo::value()->default_value("http://alice-ccdb.cern.ch"), "URL for CCDB to be used.")( "timestamp", bpo::value(), "global timestamp value in ms (for anchoring) - default is now ... or beginning of run if ALICE run number was given")( "run", bpo::value()->default_value(-1), "ALICE run number")( "asservice", bpo::value()->default_value(false), "run in service/server mode")( - "noGeant", bpo::bool_switch(), "prohibits any Geant transport/physics (by using tight cuts)"); + "noGeant", bpo::bool_switch(), "prohibits any Geant transport/physics (by using tight cuts)")( + "forwardKine", bpo::bool_switch(), "forward kinematics on a FairMQ channel")( + "noDiscOutput", bpo::bool_switch(), "switch off writing sim results to disc (useful in combination with forwardKine)"); + options.add_options()("fromCollContext", bpo::value()->default_value(""), "Use a pregenerated collision context to infer number of events to simulate, how to embedd them, the vertex position etc. Takes precedence of other options such as \"--nEvents\". The format is COLLISIONCONTEXTFILE.root[:SIGNALNAME] where SIGNALNAME is the event part in the context which is relevant."); } -void SimConfig::determineActiveModules(std::vector const& inputargs, std::vector const& skippedModules, std::vector& activeModules) +void SimConfig::determineActiveModules(std::vector const& inputargs, std::vector const& skippedModules, std::vector& activeModules, bool isUpgrade) { using o2::detectors::DetID; // input args is a vector of module strings as obtained from the -m,--modules options // of SimConfig activeModules = inputargs; +#ifdef ENABLE_UPGRADES + if (activeModules[0] != "all") { + if (isUpgrade) { + for (int i = 0; i < activeModules.size(); ++i) { + if (activeModules[i] != "A3IP" && + activeModules[i] != "IT3" && + activeModules[i] != "TRK" && + activeModules[i] != "FT3" && + activeModules[i] != "FCT" && + activeModules[i] != "TF3" && + activeModules[i] != "RCH" && + activeModules[i] != "MI3" && + activeModules[i] != "ECL" && + activeModules[i] != "FD3") { + LOGP(fatal, "List of active modules contains {}, which is not a module from the upgrades.", activeModules[i]); + } + } + } + if (!isUpgrade) { + for (int i = 0; i < activeModules.size(); ++i) { + if (activeModules[i] == "A3IP" || + activeModules[i] == "TRK" || + activeModules[i] == "FT3" || + activeModules[i] == "FCT" || + activeModules[i] == "TF3" || + activeModules[i] == "RCH" || + activeModules[i] == "MI3" || + activeModules[i] == "ECL" || + activeModules[i] == "FD3") { + LOGP(fatal, "List of active modules contains {}, which is not a run 3 module", activeModules[i]); + } + } + } + } +#endif if (activeModules.size() == 1 && activeModules[0] == "all") { activeModules.clear(); - for (int d = DetID::First; d <= DetID::Last; ++d) { #ifdef ENABLE_UPGRADES - if (d != DetID::IT3 && d != DetID::TRK && d != DetID::FT3 && d != DetID::FCT) { - activeModules.emplace_back(DetID::getName(d)); + if (isUpgrade) { + for (int d = DetID::First; d <= DetID::Last; ++d) { + if (d == DetID::TRK || + d == DetID::FT3 || + d == DetID::FCT || + d == DetID::TF3 || + d == DetID::RCH || + d == DetID::ECL || + d == DetID::FD3 || + d == DetID::MI3) { + activeModules.emplace_back(DetID::getName(d)); + } + } + activeModules.emplace_back("A3IP"); + activeModules.emplace_back("A3ABSO"); + activeModules.emplace_back("A3MAG"); + } else { +#endif + // add passive components manually (make a PassiveDetID for them!) + activeModules.emplace_back("HALL"); + activeModules.emplace_back("MAG"); + activeModules.emplace_back("DIPO"); + activeModules.emplace_back("COMP"); + activeModules.emplace_back("PIPE"); + activeModules.emplace_back("ABSO"); + activeModules.emplace_back("SHIL"); + for (int d = DetID::First; d <= DetID::Last; ++d) { +#ifdef ENABLE_UPGRADES + if (d != DetID::IT3 && d != DetID::TRK && d != DetID::FT3 && d != DetID::FCT && d != DetID::TF3 && d != DetID::RCH && d != DetID::ECL && d != DetID::FD3 && d != DetID::MI3) { + activeModules.emplace_back(DetID::getName(d)); + } } #else activeModules.emplace_back(DetID::getName(d)); #endif } - // add passive components manually (make a PassiveDetID for them!) - activeModules.emplace_back("HALL"); - activeModules.emplace_back("MAG"); - activeModules.emplace_back("DIPO"); - activeModules.emplace_back("COMP"); - activeModules.emplace_back("PIPE"); - activeModules.emplace_back("ABSO"); - activeModules.emplace_back("SHIL"); } - // now we take out detectors listed as skipped - for (auto& s : skippedModules) { - auto iter = std::find(activeModules.begin(), activeModules.end(), s); - if (iter != activeModules.end()) { - // take it out - activeModules.erase(iter); + filterSkippedElements(activeModules, skippedModules); +} + +bool SimConfig::determineActiveModulesList(const std::string& version, std::vector const& inputargs, std::vector const& skippedModules, std::vector& activeModules) +{ + DetectorList_t modules; + DetectorMap_t map; + if (auto pos = version.find(':'); pos != std::string::npos) { + auto pversion = version.substr(0, pos); + auto ppath = version.substr(pos + 1); + if (!parseDetectorMapfromJSON(ppath, map)) { + LOGP(error, "Could not parse {}; check errors above!", ppath); + return false; + } + if (map.find(pversion) == map.end()) { + LOGP(error, "List {} is not defined in custom JSON file!", pversion); + printDetMap(map); + return false; + } + modules = map[pversion]; + LOGP(info, "Running with version {} from custom detector list '{}'", pversion, ppath); + } else { + // Otherwise check 'official' versions which provided in config + auto o2env = std::getenv("O2_ROOT"); + if (!o2env) { + LOGP(error, "O2_ROOT environment not defined"); + return false; + } + const std::string rootpath(fmt::format("{}/share/config/o2simdefaultdetectorlist.json", o2env)); + if (!parseDetectorMapfromJSON(rootpath, map)) { + LOGP(error, "Could not parse {} -> check errors above!", rootpath); + return false; + } + if (map.find(version) == map.end()) { + LOGP(error, "List {} is not defined in 'official' JSON file!", version); + printDetMap(map); + return false; + } + modules = map[version]; + static std::string last_version{}; // prevent multiple printouts of same message + if (last_version != version) { + LOGP(info, "Running with official detector version '{}'", version); + last_version = version; + } + } + // check if specified modules are in list + if (inputargs.size() != 1 || inputargs[0] != "all") { + std::vector diff; + for (const auto& in : inputargs) { + if (std::find(modules.begin(), modules.end(), in) == std::end(modules)) { + diff.emplace_back(in); + } + } + if (!diff.empty()) { + LOGP(error, "Modules specified that are not present in detector list {}", version); + for (int j{0}; const auto& m : diff) { + LOGP(info, " - {: <2}. {}", j++, m); + } + printDetMap(map, version); + return false; } } + // Insert into active modules if module is built buy -m or insert all if default for -m is used ("all") + std::copy_if(modules.begin(), modules.end(), std::back_inserter(activeModules), + [&inputargs](const auto& e) { return (inputargs.size() == 1 && inputargs[0] == "all") || (std::find(inputargs.begin(), inputargs.end(), e) != inputargs.end()); }); + return filterSkippedElements(activeModules, skippedModules); } void SimConfig::determineReadoutDetectors(std::vector const& activeModules, std::vector const& enableReadout, std::vector const& disableReadout, std::vector& readoutDetectors) @@ -144,13 +277,46 @@ void SimConfig::determineReadoutDetectors(std::vector const& active } } +std::pair SimConfig::getCollContextFilenameAndEventPrefix() const +{ + // we decompose the argument to fetch + // (a) collision contextfilename + // (b) sim prefix to use from the context + auto pos = mConfigData.mFromCollisionContext.find(':'); + std::string collcontextfile{mConfigData.mFromCollisionContext}; + std::string simprefix{mConfigData.mOutputPrefix}; + if (pos != std::string::npos) { + collcontextfile = mConfigData.mFromCollisionContext.substr(0, pos); + simprefix = mConfigData.mFromCollisionContext.substr(pos + 1); + } + return std::make_pair(collcontextfile, simprefix); +} + bool SimConfig::resetFromParsedMap(boost::program_options::variables_map const& vm) { using o2::detectors::DetID; mConfigData.mMCEngine = vm["mcEngine"].as(); + mConfigData.mNoGeant = vm["noGeant"].as(); + + // Reset modules and detectors as they are anyway re-parsed + mConfigData.mReadoutDetectors.clear(); + mConfigData.mActiveModules.clear(); + + // Get final set of active Modules + if (!determineActiveModulesList(vm["detectorList"].as(), vm["modules"].as>(), vm["skipModules"].as>(), mConfigData.mActiveModules)) { + return false; + } + + if (mConfigData.mNoGeant) { + // CAVE is all that's needed (and that will be built either way), so clear all modules and set the O2TrivialMCEngine + mConfigData.mActiveModules.clear(); + // force usage of O2TrivialMCEngine, no overhead from actual transport engine initialisation + mConfigData.mMCEngine = "O2TrivialMCEngine"; + } else if (mConfigData.mMCEngine.compare("O2TrivialMCEngine") == 0) { + LOG(error) << "The O2TrivialMCEngine engine can only be used with --noGeant option"; + return false; + } - // get final set of active Modules - determineActiveModules(vm["modules"].as>(), vm["skipModules"].as>(), mConfigData.mActiveModules); const auto& activeModules = mConfigData.mActiveModules; // get final set of detectors which are readout @@ -171,29 +337,38 @@ bool SimConfig::resetFromParsedMap(boost::program_options::variables_map const& mConfigData.mConfigFile = vm["configFile"].as(); mConfigData.mPrimaryChunkSize = vm["chunkSize"].as(); mConfigData.mInternalChunkSize = vm["chunkSizeI"].as(); - mConfigData.mStartSeed = vm["seed"].as(); + mConfigData.mStartSeed = vm["seed"].as(); mConfigData.mSimWorkers = vm["nworkers"].as(); if (vm.count("timestamp")) { mConfigData.mTimestamp = vm["timestamp"].as(); - mConfigData.mTimestampMode = kManual; + mConfigData.mTimestampMode = TimeStampMode::kManual; } else { mConfigData.mTimestamp = std::chrono::time_point_cast(std::chrono::system_clock::now()).time_since_epoch().count(); - mConfigData.mTimestampMode = kNow; + mConfigData.mTimestampMode = TimeStampMode::kNow; } mConfigData.mRunNumber = vm["run"].as(); mConfigData.mCCDBUrl = vm["CCDBUrl"].as(); mConfigData.mAsService = vm["asservice"].as(); - mConfigData.mNoGeant = vm["noGeant"].as(); + mConfigData.mForwardKine = vm["forwardKine"].as(); + mConfigData.mWriteToDisc = !vm["noDiscOutput"].as(); if (vm.count("noemptyevents")) { mConfigData.mFilterNoHitEvents = true; } + mConfigData.mFromCollisionContext = vm["fromCollContext"].as(); + auto collcontext_simprefix = getCollContextFilenameAndEventPrefix(); + adjustFromCollContext(collcontext_simprefix.first, collcontext_simprefix.second); + + // analyse vertex options + if (!parseVertexModeString(vm["vertexMode"].as(), mConfigData.mVertexMode)) { + return false; + } // analyse field options // either: "ccdb" or +-2[U],+-5[U] and 0[U]; +-U auto& fieldstring = vm["field"].as(); - std::regex re("(ccdb)|([+-]?[250]U?)"); + std::regex re("(ccdb)|([+-]?(0|[2-9]|[12][0-9]|20)U?)"); if (!std::regex_match(fieldstring, re)) { - LOG(error) << "Invalid field option"; + LOG(error) << "Invalid field option " << fieldstring; return false; } if (fieldstring == "ccdb") { @@ -211,13 +386,33 @@ bool SimConfig::resetFromParsedMap(boost::program_options::variables_map const& return true; } +bool SimConfig::parseVertexModeString(std::string const& vertexstring, VertexMode& mode) +{ + // vertexstring must be either kNoVertex, kDiamondParam, kCCDB + if (vertexstring == "kNoVertex") { + mode = VertexMode::kNoVertex; + return true; + } else if (vertexstring == "kDiamondParam") { + mode = VertexMode::kDiamondParam; + return true; + } else if (vertexstring == "kCCDB") { + mode = VertexMode::kCCDB; + return true; + } else if (vertexstring == "kCollContext") { + mode = VertexMode::kCollCxt; + return true; + } + LOG(error) << "Vertex mode must be one of kNoVertex, kDiamondParam, kCCDB, kCollContext"; + return false; +} + bool SimConfig::parseFieldString(std::string const& fieldstring, int& fieldvalue, SimFieldMode& mode) { // analyse field options // either: "ccdb" or +-2[U],+-5[U] and 0[U]; +-U - std::regex re("(ccdb)|([+-]?[250]U?)"); + std::regex re("(ccdb)|([+-]?(0|[2-9]|[12][0-9]|20)U?)"); if (!std::regex_match(fieldstring, re)) { - LOG(error) << "Invalid field option"; + LOG(error) << "Invalid field option " << fieldstring; return false; } if (fieldstring == "ccdb") { @@ -231,15 +426,94 @@ bool SimConfig::parseFieldString(std::string const& fieldstring, int& fieldvalue return true; } -bool SimConfig::resetFromArguments(int argc, char* argv[]) +bool SimConfig::filterSkippedElements(std::vector& elements, std::vector const& skipped) { - namespace bpo = boost::program_options; + for (auto& s : skipped) { + if (s.empty()) { // nothing to skip here + continue; + } + auto iter = std::find(elements.begin(), elements.end(), s); + if (iter != elements.end()) { + // take it out + elements.erase(iter); + } else { + LOGP(error, "Skipped modules specified that are not present in built modules!"); + LOGP(error, "Built modules:"); + for (int j{0}; const auto& m : elements) { + LOGP(error, " + {: <2}. {}", j++, m); + } + std::vector diff; + for (const auto& skip : skipped) { + if (std::find(elements.begin(), elements.end(), skip) == std::end(elements)) { + diff.emplace_back(skip); + } + } + LOGP(error, "Specified skipped modules not present in built modules:"); + for (int j{0}; const auto& m : diff) { + LOGP(error, " - {: <2}. {}", j++, m); + } + return false; + } + } + return true; +} + +void SimConfig::adjustFromCollContext(std::string const& collcontextfile, std::string const& prefix) +{ + // When we use pregenerated collision contexts, some options + // need to be auto-adjusted. Do so and inform about this in the logs. + if (collcontextfile == "") { + return; + } + + auto context = o2::steer::DigitizationContext::loadFromFile(collcontextfile); + if (context) { + // find the events belonging to a source that corresponds to a sim prefix + LOG(info) << "Looking up simprefixes " << prefix; + int sourceid = context->findSimPrefix(prefix); + if (sourceid == -1) { + LOG(error) << "Could not find collisions with sim prefix " << prefix << " in the collision context. The collision context specifies the following prefixes:"; + for (auto& sp : context->getSimPrefixes()) { + LOG(info) << sp; + } + LOG(fatal) << "Aborting due to prefix error"; + } else { + auto collisionmap = context->getCollisionIndicesForSource(sourceid); + LOG(info) << "Found " << collisionmap.size() << " events in the collisioncontext for prefix " << prefix; + + // check if collisionmap is dense (otherwise it will get screwed up with order/indexing in ROOT output) + bool good = true; + for (auto index = 0; index < collisionmap.size(); ++index) { + if (collisionmap.find(index) == collisionmap.end()) { + good = false; + } + } + if (!good) { + LOG(fatal) << "events in collisioncontext are non-compact "; + } + + // do some adjustments based on the number of events to be simulated + if (mConfigData.mNEvents == 0 || mConfigData.mNEvents == collisionmap.size()) { + // we take what is specified in the context + mConfigData.mNEvents = collisionmap.size(); + } else { + LOG(warning) << "The number of events on the command line " << mConfigData.mNEvents << " and in the collision context differ. We take the one from collision context " << collisionmap.size(); + mConfigData.mNEvents = collisionmap.size(); + } + LOG(info) << "Setting number of events to simulate to " << mConfigData.mNEvents; + } + } else { + LOG(fatal) << "Could not open collision context file " << collcontextfile; + } +} +bool SimConfig::resetFromArguments(int argc, char* argv[]) +{ // Arguments parsing bpo::variables_map vm; bpo::options_description desc("Allowed options"); desc.add_options()("help,h", "Produce help message."); - initOptions(desc); + initOptions(desc, mConfigData.mIsUpgrade); try { bpo::store(parse_command_line(argc, argv, desc), vm); @@ -272,7 +546,7 @@ bool parseSimReconfigFromString(std::string const& argumentstring, SimReconfigDa bpo::options_description options("Allowed options"); options.add_options()( - "nEvents,n", bpo::value(&data.nEvents)->default_value(1), "number of events")( + "nEvents,n", bpo::value(&data.nEvents)->default_value(0), "number of events")( "generator,g", bpo::value(&data.generator)->default_value("boxgen"), "Event generator to be used.")( "trigger,t", bpo::value(&data.trigger)->default_value(""), "Event generator trigger to be used.")( "startEvent", bpo::value(&data.startEvent)->default_value(0), "index of first event to be used (when applicable)")( @@ -286,7 +560,7 @@ bool parseSimReconfigFromString(std::string const& argumentstring, SimReconfigDa "configKeyValues", bpo::value(&data.keyValueTokens)->default_value(""), "semicolon separated key=value strings (e.g.: 'TPC.gasDensity=1;...")( "configFile", bpo::value(&data.configFile)->default_value(""), "Path to an INI or JSON configuration file")( "chunkSize", bpo::value(&data.primaryChunkSize)->default_value(500), "max size of primary chunk (subevent) distributed by server")( - "seed", bpo::value(&data.startSeed)->default_value(-1), "initial seed (default: -1 random)")( + "seed", bpo::value(&data.startSeed)->default_value(0L), "initial seed as ULong_t (default: 0 == random)")( "stop", bpo::value(&data.stop)->default_value(false), "control command to shut down daemon"); bpo::variables_map vm; diff --git a/Common/SimConfig/src/SimConfigLinkDef.h b/Common/SimConfig/src/SimConfigLinkDef.h index 26434e555f2ad..a1315e24ffedd 100644 --- a/Common/SimConfig/src/SimConfigLinkDef.h +++ b/Common/SimConfig/src/SimConfigLinkDef.h @@ -29,6 +29,7 @@ #pragma link C++ class o2::conf::ConfigurableParamHelper < o2::conf::DigiParams> + ; #pragma link C++ enum o2::conf::EG4Physics; +#pragma link C++ enum o2::conf::EG4Nav; #pragma link C++ enum o2::conf::SimFieldMode; #pragma link C++ struct o2::conf::G4Params + ; #pragma link C++ class o2::conf::ConfigurableParamHelper < o2::conf::G4Params> + ; @@ -36,4 +37,10 @@ #pragma link C++ struct o2::conf::MatMapParams + ; #pragma link C++ class o2::conf::ConfigurableParamHelper < o2::conf::MatMapParams> + ; +#pragma link C++ class o2::eventgen::InteractionDiamondParam + ; +#pragma link C++ class o2::conf::ConfigurableParamHelper < o2::eventgen::InteractionDiamondParam> + ; + +#pragma link C++ class o2::GlobalProcessCutSimParam + ; +#pragma link C++ class o2::conf::ConfigurableParamHelper < o2::GlobalProcessCutSimParam> + ; + #endif diff --git a/Common/SimConfig/src/SimDLLoader.cxx b/Common/SimConfig/src/SimDLLoader.cxx new file mode 100644 index 0000000000000..ed15a8b0cef6a --- /dev/null +++ b/Common/SimConfig/src/SimDLLoader.cxx @@ -0,0 +1,13 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include "SimConfig/SimDLLoader.h" +O2DLLoaderImpl(o2::conf::SimDLLoader) diff --git a/Common/SimConfig/test/testSimCutParam.cxx b/Common/SimConfig/test/testSimCutParam.cxx index 7a17ae9d1f2cf..468d76429cbba 100644 --- a/Common/SimConfig/test/testSimCutParam.cxx +++ b/Common/SimConfig/test/testSimCutParam.cxx @@ -15,6 +15,7 @@ #include #include "SimConfig/SimParams.h" #include "CommonUtils/ConfigurableParam.h" +#include using namespace o2::conf; diff --git a/Common/Topologies/o2prototype_topology.xml b/Common/Topologies/o2prototype_topology.xml index 240b8d87d469a..8d53c9eb0127a 100644 --- a/Common/Topologies/o2prototype_topology.xml +++ b/Common/Topologies/o2prototype_topology.xml @@ -74,7 +74,7 @@ The following parameters need adjustment when extending the FLP-EPN configuratio - $ALICEO2_INSTALL_DIR/bin/aliceHLTWrapper Tracker_%collectionIndex%_%taskIndex% 1 --dds --poll-period 100 --input type=pull,size=5000,method=connect,property=EPNReceiverOutputAddress,count=1 --output type=push,size=500,method=bind,property=TrackingOutputAddress,min-port=48000 --library libAliHLTTPC.so --component TPCCATracker --run 167808 --parameter '-GlobalTracking -allowGPU -GPUHelperThreads 4 -loglevel=0x7c' + $ALICEO2_INSTALL_DIR/bin/aliceHLTWrapper Tracker_%collectionIndex%_%taskIndex% 1 --dds --poll-period 100 --input type=pull,size=5000,method=connect,property=EPNReceiverOutputAddress,count=1 --output type=push,size=500,method=bind,property=TrackingOutputAddress,min-port=48000 --library libAliHLTTPC.so --component TPCCATracker --run 167808 --parameter '-GlobalTracking -allowGPU -loglevel=0x7c' EPNReceiverOutputAddress diff --git a/Common/Utils/CMakeLists.txt b/Common/Utils/CMakeLists.txt index a0155ffc61f3d..849a3d70f62e1 100644 --- a/Common/Utils/CMakeLists.txt +++ b/Common/Utils/CMakeLists.txt @@ -14,7 +14,7 @@ o2_add_library(CommonUtils src/RootChain.cxx src/CompStream.cxx src/ShmManager.cxx src/ValueMonitor.cxx src/StringUtils.cxx - src/ConfigurableParamHelper.cxx src/ConfigurableParam.cxx src/RootSerializableKeyValueStore.cxx + src/ConfigurableParamReaders.cxx src/ConfigurableParamHelper.cxx src/ConfigurableParam.cxx src/RootSerializableKeyValueStore.cxx src/KeyValParam.cxx src/FileSystemUtils.cxx src/FIFO.cxx @@ -24,15 +24,15 @@ o2_add_library(CommonUtils src/NameConf.cxx src/IRFrameSelector.cxx src/DebugStreamer.cxx + src/DLLoaderBase.cxx PUBLIC_LINK_LIBRARIES ROOT::Hist ROOT::Tree Boost::iostreams O2::CommonDataFormat O2::Headers - FairLogger::FairLogger O2::MathUtils) + FairLogger::FairLogger O2::MathUtils TBB::tbb O2::GPUCommon) o2_target_root_dictionary(CommonUtils HEADERS include/CommonUtils/TreeStream.h include/CommonUtils/TreeStreamRedirector.h include/CommonUtils/RootChain.h include/CommonUtils/BoostHistogramUtils.h - include/CommonUtils/BoostSerializer.h include/CommonUtils/ShmManager.h include/CommonUtils/RngHelper.h include/CommonUtils/StringUtils.h @@ -40,27 +40,32 @@ o2_target_root_dictionary(CommonUtils include/CommonUtils/MemFileHelper.h include/CommonUtils/ConfigurableParam.h include/CommonUtils/ConfigurableParamHelper.h + include/CommonUtils/ConfigurableParamReaders.h include/CommonUtils/ConfigurationMacroHelper.h include/CommonUtils/RootSerializableKeyValueStore.h include/CommonUtils/KeyValParam.h include/CommonUtils/VerbosityConfig.h include/CommonUtils/FileFetcher.h + include/CommonUtils/DLLoaderBase.h include/CommonUtils/NameConf.h include/CommonUtils/IRFrameSelector.h include/CommonUtils/DebugStreamer.h) +# Extra dictionaries only needed if tests are built +if(BUILD_TESTING) + o2_add_library(CommonUtilsTest + SOURCES src/ConfigurableParamTest.cxx + PUBLIC_LINK_LIBRARIES O2::CommonUtils) + o2_target_root_dictionary(CommonUtilsTest + HEADERS include/CommonUtils/ConfigurableParamTest.h) +endif() + o2_add_test(TreeStream COMPONENT_NAME CommonUtils LABELS utils SOURCES test/testTreeStream.cxx PUBLIC_LINK_LIBRARIES O2::CommonUtils O2::ReconstructionDataFormats) -o2_add_test(BoostSerializer - COMPONENT_NAME CommonUtils - LABELS utils - SOURCES test/testBoostSerializer.cxx - PUBLIC_LINK_LIBRARIES O2::CommonUtils Boost::serialization) - o2_add_test(CompStream COMPONENT_NAME CommonUtils LABELS utils @@ -85,7 +90,22 @@ o2_add_test(MemFileHelper SOURCES test/testMemFileHelper.cxx PUBLIC_LINK_LIBRARIES O2::CommonUtils) +o2_add_test(EnumFlags + COMPONENT_NAME CommonUtils + LABELS utils + SOURCES test/testEnumFlags.cxx + PUBLIC_LINK_LIBRARIES O2::CommonUtils) + +o2_add_test(ConfigurableParam + COMPONENT_NAME CommonUtils + LABELS utils + SOURCES test/testConfigurableParam.cxx + PUBLIC_LINK_LIBRARIES O2::CommonUtilsTest) + o2_add_executable(treemergertool COMPONENT_NAME CommonUtils SOURCES src/TreeMergerTool.cxx PUBLIC_LINK_LIBRARIES O2::CommonUtils Boost::program_options ROOT::Core) + +add_library(fpu_support OBJECT src/fpu.cxx) +add_library(O2::fpu_support ALIAS fpu_support) diff --git a/Common/Utils/include/CommonUtils/BoostHistogramUtils.h b/Common/Utils/include/CommonUtils/BoostHistogramUtils.h index 2bb7262bae108..0d20e97adf7a7 100644 --- a/Common/Utils/include/CommonUtils/BoostHistogramUtils.h +++ b/Common/Utils/include/CommonUtils/BoostHistogramUtils.h @@ -45,10 +45,10 @@ #include using boostHisto2d = boost::histogram::histogram, boost::histogram::axis::regular>, boost::histogram::unlimited_storage>>; -using boostHisto1d = boost::histogram::histogram>, boost::histogram::unlimited_storage>>; +using boostHisto1d = boost::histogram::histogram>>; using boostHisto2d_VarAxis = boost::histogram::histogram>, boost::histogram::axis::variable>>>; -using boostHisto1d_VarAxis = boost::histogram::histogram>>>; +using boostHisto1d_VarAxis = boost::histogram::histogram>>; namespace o2 { @@ -313,7 +313,7 @@ std::vector fitGaus(Iterator first, Iterator last, BinCenterView axisfir if (nbins > 1) { // dont take bins with 0 entries or bins with nan into account // if y-value (*iter) is 1, log(*iter) will be 0. Exclude these cases - if (isnan(*axisiter) || isinf(*axisiter) || *iter <= 0 || *iter == 1) { + if (std::isnan(*axisiter) || std::isinf(*axisiter) || *iter <= 0 || *iter == 1) { continue; } double x = *axisiter; @@ -401,28 +401,124 @@ std::vector fitBoostHistoWithGaus(boost::histogram::histogram& } /// \brief Convert a 1D root histogram to a Boost histogram -boostHisto1d_VarAxis boosthistoFromRoot_1D(TH1D* inHist1D); +template +Hist boosthistoFromRoot_1D(TH1D* inHist1D) +{ + // first setup the proper boost histogram + int nBins = inHist1D->GetNbinsX(); + std::vector binEdges; + for (int i = 0; i < nBins + 1; i++) { + binEdges.push_back(inHist1D->GetBinLowEdge(i + 1)); + } + Hist mHisto; + + if constexpr (std::is_same::value) { + mHisto = boost::histogram::make_histogram(boost::histogram::axis::variable<>(binEdges)); + } else { + mHisto = boost::histogram::make_histogram(boost::histogram::axis::regular<>(nBins, binEdges[0], binEdges.back())); + } + + // trasfer the acutal values + for (Int_t x = 1; x < nBins + 1; x++) { + mHisto.at(x - 1) = inHist1D->GetBinContent(x); + } + return mHisto; +} /// \brief Convert a 2D root histogram to a Boost histogram -boostHisto2d_VarAxis boostHistoFromRoot_2D(TH2D* inHist2D); +template +Hist boostHistoFromRoot_2D(TH2D* inHist2D) +{ + // Get Xaxis binning + const int nBinsX = inHist2D->GetNbinsX(); + std::vector binEdgesX; + for (int i = 0; i < nBinsX + 1; i++) { + binEdgesX.push_back(inHist2D->GetXaxis()->GetBinLowEdge(i + 1)); + } + // Get Yaxis binning + const int nBinsY = inHist2D->GetNbinsY(); + std::vector binEdgesY; + for (int i = 0; i < nBinsY + 1; i++) { + binEdgesY.push_back(inHist2D->GetYaxis()->GetBinLowEdge(i + 1)); + } + + Hist mHisto; + + if constexpr (std::is_same::value) { + mHisto = boost::histogram::make_histogram(boost::histogram::axis::variable<>(binEdgesX), boost::histogram::axis::variable<>(binEdgesY)); + } else { + mHisto = boost::histogram::make_histogram(boost::histogram::axis::regular<>(nBinsX, binEdgesX[0], binEdgesX.back()), boost::histogram::axis::regular<>(nBinsY, binEdgesY[0], binEdgesY.back())); + } + + // trasfer the acutal values + for (Int_t x = 1; x < nBinsX + 1; x++) { + for (Int_t y = 1; y < nBinsY + 1; y++) { + mHisto.at(x - 1, y - 1) = inHist2D->GetBinContent(x, y); + } + } + return mHisto; +} /// \brief Get the mean of a 1D boost histogram +/// \param inHist1D input boost histogram +/// \param rangeLow minimum range considered for the mean calculation (if rangeLow == rangeHigh, no cut will be performed) +/// \param rangeHigh maximum range considered for the mean calculation (if rangeLow == rangeHigh, no cut will be performed) +/// \return mean value of boost histogram in specified range template -double getMeanBoost1D(boost::histogram::histogram& inHist1D) +double getMeanBoost1D(boost::histogram::histogram& inHist1D, const double rangeLow = 0, const double rangeHigh = 0) { // LOG(info) << "Entering the mean function for hist with rank " << inHist1D.rank() << " with " << inHist1D.axis(0).size() << " bins"; o2::math_utils::detail::StatAccumulator stats; - int mynbins = 0; + bool restrictRange = rangeLow < rangeHigh ? true : false; auto histiter = inHist1D.begin() + 1; const auto& axis = inHist1D.axis(0); for (auto bincenter = BinCenterView(axis.begin()); bincenter != BinCenterView(axis.end()); ++bincenter, ++histiter) { // std::cout << "bin center bin " << mynbins << ": " << *bincenter << " <-> value: " << *histiter << std::endl; - ++mynbins; + if (restrictRange) { + if (*bincenter < rangeLow || *bincenter > rangeHigh) { + continue; + } + } stats.add(*bincenter, *histiter); } return stats.getMean(); } +/// \brief Get the variance of a 1D boost histogram +/// \param inHist1D input boost histogram +/// \param mean mean mean of the histogram, if set to -999999, mean will be caluclated +/// \param weight weight of the entries in the histogram. Per default set to 1 +/// \param rangeLow minimum range considered for the mean calculation (if rangeLow == rangeHigh, no cut will be performed) +/// \param rangeHigh maximum range considered for the mean calculation (if rangeLow == rangeHigh, no cut will be performed) +/// \return variance of the distribution with respect to the mean +template +double getVarianceBoost1D(boost::histogram::histogram& inHist1D, double mean = -999999, const double rangeLow = 0, const double rangeHigh = 0, const double weight = 1) +{ + if (std::abs(mean + 999999) < 0.00001) { + mean = getMeanBoost1D(inHist1D, rangeLow, rangeHigh); + } + bool restrictRange = rangeLow < rangeHigh ? true : false; + unsigned int nMeas = 0; // counter for the number of data points + auto histiter = inHist1D.begin() + 1; + const auto& axis = inHist1D.axis(0); + double variance = 0; + for (auto bincenter = BinCenterView(axis.begin()); bincenter != BinCenterView(axis.end()); ++bincenter, ++histiter) { + if (restrictRange) { + LOG(debug) << " *bincenter " << *bincenter << " rangeLow " << rangeLow << " rangeHigh " << rangeHigh; + if (*bincenter < rangeLow || *bincenter > rangeHigh) { + continue; + } + } + nMeas += *histiter / weight; // to get the number of entries, for weighted histograms we need to divide by the weight to get back to the number of entries + variance += *histiter * (*bincenter - mean) * (*bincenter - mean); + } + if (nMeas <= 1) { + return 0; + } + variance /= (nMeas - 1); + return variance; +} + /// \brief Convert a 2D boost histogram to a root histogram template TH1F TH1FFromBoost(BoostHist hist, const char* name = "hist") @@ -468,7 +564,7 @@ TH2F TH2FFromBoost(BoostHist hist, const char* name = "hist") /// \return result /// 1d boost histogram from projection of the input 2d boost histogram template -auto ProjectBoostHistoX(boost::histogram::histogram& hist2d, const int binLow, const int binHigh) +auto ProjectBoostHistoX(const boost::histogram::histogram& hist2d, const int binLow, const int binHigh) { using namespace boost::histogram::literals; // enables _c suffix needed for projection @@ -494,7 +590,7 @@ auto ProjectBoostHistoX(boost::histogram::histogram& hist2d, const int /// \return result /// 1d boost histogram from projection of the input 2d boost histogram template -auto ProjectBoostHistoXFast(boost::histogram::histogram& hist2d, const int binLow, const int binHigh) +auto ProjectBoostHistoXFast(const boost::histogram::histogram& hist2d, const int binLow, const int binHigh) { unsigned int nbins = hist2d.axis(0).size(); double binStartX = hist2d.axis(0).bin(0).lower(); @@ -522,7 +618,7 @@ auto ProjectBoostHistoXFast(boost::histogram::histogram& hist2d, const /// \return result /// 1d boost histogram from projection of the input 2d boost histogram template -auto ReduceBoostHistoFastSlice(boost::histogram::histogram& hist2d, int binXLow, int binXHigh, int binYLow, int binYHigh, bool includeOverflowUnderflow) +auto ReduceBoostHistoFastSlice(const boost::histogram::histogram& hist2d, int binXLow, int binXHigh, int binYLow, int binYHigh, bool includeOverflowUnderflow) { int nXbins = binXHigh - binXLow + 1; int nYbins = binYHigh - binYLow + 1; @@ -618,6 +714,29 @@ auto ReduceBoostHistoFastSliceByValue(boost::histogram::histogram& hist return ReduceBoostHistoFastSlice(hist2d, binXLow, binXHigh, binYLow, binYHigh, includeOverflowUnderflow); } +/// \brief Function to integrate 1d boost histogram in specified range +/// \param hist 1d boost histogram +/// \param min lower integration range +/// \param max upper integration range +/// \return sum of bin contents in specified range +template +double getIntegralBoostHist(boost::histogram::histogram& hist, double min, double max) +{ + // find bins for min and max values + std::array axisLimitsIndex = {hist.axis(0).index(min), hist.axis(0).index(max)}; + // over/underflow bin have to be protected + for (auto& bin : axisLimitsIndex) { + if (bin < 0) { + bin = 0; + } else if (bin >= hist.axis(0).size()) { + bin = hist.axis(0).size() - 1; + } + } + // Reduce histogram to desired range + auto slicedHist = ReduceBoostHistoFastSlice1D(hist, axisLimitsIndex[0], axisLimitsIndex[1], false); + return boost::histogram::algorithm::sum(slicedHist); +} + } // namespace utils } // end namespace o2 diff --git a/Common/Utils/include/CommonUtils/BoostSerializer.h b/Common/Utils/include/CommonUtils/BoostSerializer.h deleted file mode 100644 index 4c388a3daa8cc..0000000000000 --- a/Common/Utils/include/CommonUtils/BoostSerializer.h +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// \file BoostSerializer.h -/// \brief Templatised boost serializer/deserializer for containers and base types -/// \author Gabriele G. Fronzé -/// \date 17 July 2018 - -#ifndef ALICEO2_BOOSTSERIALIZER_H -#define ALICEO2_BOOSTSERIALIZER_H - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace o2 -{ -namespace utils -{ -template -std::ostringstream BoostSerialize(const ContT& dataSet) -{ - /// Serialises a container (vector, array or list) using boost serialisation routines. - /// Requires the contained type to be either trivial or provided with an overried of boost::serialise method. - std::ostringstream buffer; - boost::archive::binary_oarchive outputArchive(buffer); - outputArchive << dataSet; - return buffer; -} - -template -ContT BoostDeserialize(std::string& msgStr) -{ - /// Deserialises a msg contained in a string in a container type (vector, array or list) of the provided type. - ContT output; - std::istringstream buffer(msgStr); - boost::archive::binary_iarchive inputArchive(buffer); - inputArchive >> output; - return std::move(output); -} -} // namespace utils -} // namespace o2 - -#endif //ALICEO2_BOOSTSERIALIZER_H diff --git a/Common/Utils/include/CommonUtils/ConfigurableParam.h b/Common/Utils/include/CommonUtils/ConfigurableParam.h index 8a992c8288253..39b24bbbbd57c 100644 --- a/Common/Utils/include/CommonUtils/ConfigurableParam.h +++ b/Common/Utils/include/CommonUtils/ConfigurableParam.h @@ -15,11 +15,13 @@ #define COMMON_SIMCONFIG_INCLUDE_SIMCONFIG_CONFIGURABLEPARAM_H_ #include +#include #include #include -#include +#include #include #include +#include class TFile; class TRootIOCtor; @@ -160,7 +162,11 @@ class ConfigurableParam virtual std::string getName() const = 0; // print the current keys and values to screen (optionally with provenance information) - virtual void printKeyValues(bool showprov = true) const = 0; + virtual void printKeyValues(bool showprov = true, bool useLogger = false, bool withPadding = false, bool showHash = false) const = 0; + + // get a single size_t hash_value of this parameter (can be used as a checksum to see + // if object changed or different) + virtual size_t getHash() const = 0; // return the provenance of the member key virtual EParamProvenance getMemberProvenance(const std::string& key) const = 0; @@ -168,17 +174,12 @@ class ConfigurableParam static EParamProvenance getProvenance(const std::string& key); static void printAllRegisteredParamNames(); - static void printAllKeyValuePairs(); + static void printAllKeyValuePairs(bool useLogger = false); - static const std::string& getInputDir() { return sInputDir; } static const std::string& getOutputDir() { return sOutputDir; } - static void setInputDir(const std::string& d) { sInputDir = d; } static void setOutputDir(const std::string& d) { sOutputDir = d; } - static boost::property_tree::ptree readINI(std::string const& filepath); - static boost::property_tree::ptree readJSON(std::string const& filepath); - static boost::property_tree::ptree readConfigFile(std::string const& filepath); static bool configFileExists(std::string const& filepath); // writes a human readable JSON file of all parameters @@ -190,10 +191,12 @@ class ConfigurableParam template static T getValueAs(std::string key) { - if (!sIsFullyInitialized) { - initialize(); - } - return sPtree->get(key); + return [](auto* tree, const std::string& key) -> T { + if (!sIsFullyInitialized) { + initialize(); + } + return tree->template get(key); + }(sPtree, key); } template @@ -202,42 +205,43 @@ class ConfigurableParam if (!sIsFullyInitialized) { initialize(); } - assert(sPtree); - try { - auto key = mainkey + "." + subkey; - if (sPtree->get_optional(key).is_initialized()) { - sPtree->put(key, x); - auto changed = updateThroughStorageMap(mainkey, subkey, typeid(T), (void*)&x); - if (changed != EParamUpdateStatus::Failed) { - sValueProvenanceMap->find(key)->second = kRT; // set to runtime + return [&subkey, &x, &mainkey](auto* tree) -> void { + assert(tree); + try { + auto key = mainkey + "." + subkey; + if (tree->template get_optional(key).is_initialized()) { + tree->put(key, x); + auto changed = updateThroughStorageMap(mainkey, subkey, typeid(T), (void*)&x); + if (changed != EParamUpdateStatus::Failed) { + sValueProvenanceMap->find(key)->second = kRT; // set to runtime + } } + } catch (std::exception const& e) { + std::cerr << "Error in setValue (T) " << e.what() << "\n"; } - } catch (std::exception const& e) { - std::cerr << "Error in setValue (T) " << e.what() << "\n"; - } + }(sPtree); } - // specialized for std::string - // which means that the type will be converted internally - static void setValue(std::string const& key, std::string const& valuestring) + static void setProvenance(std::string const& mainkey, std::string const& subkey, EParamProvenance p) { if (!sIsFullyInitialized) { - initialize(); + std::cerr << "setProvenance was called on non-initialized ConfigurableParam\n"; + return; } - assert(sPtree); try { - if (sPtree->get_optional(key).is_initialized()) { - sPtree->put(key, valuestring); - auto changed = updateThroughStorageMapWithConversion(key, valuestring); - if (changed != EParamUpdateStatus::Failed) { - sValueProvenanceMap->find(key)->second = kRT; // set to runtime - } + auto key = mainkey + "." + subkey; + auto keyProv = sValueProvenanceMap->find(key); + if (keyProv != sValueProvenanceMap->end()) { + keyProv->second = p; } } catch (std::exception const& e) { - std::cerr << "Error in setValue (string) " << e.what() << "\n"; + std::cerr << "Error in setProvenance (T) " << e.what() << "\n"; } } + // specialized for std::string + // which means that the type will be converted internally + static void setValue(std::string const& key, std::string const& valuestring); static void setEnumValue(const std::string&, const std::string&); static void setArrayValue(const std::string&, const std::string&); @@ -298,7 +302,6 @@ class ConfigurableParam // (stored as a vector of pairs ) static EnumRegistry* sEnumRegistry; - static std::string sInputDir; static std::string sOutputDir; void setRegisterMode(bool b) { sRegisterMode = b; } @@ -318,17 +321,19 @@ class ConfigurableParam } // end namespace o2 // a helper macro for boilerplate code in parameter classes -#define O2ParamDef(classname, key) \ - public: \ - classname(TRootIOCtor*) {} \ - classname(classname const&) = delete; \ - \ - private: \ - static constexpr char const* const sKey = key; \ - static classname sInstance; \ - classname() = default; \ - template \ - friend class o2::conf::ConfigurableParamHelper; +#define O2ParamDef(classname, key) \ + public: \ + classname(TRootIOCtor*) {} \ + classname(classname const&) = delete; \ + \ + private: \ + static constexpr char const* const sKey = key; \ + static classname sInstance; \ + classname() = default; \ + template \ + friend class o2::conf::ConfigurableParamHelper; \ + template \ + friend class o2::conf::ConfigurableParamPromoter; // a helper macro to implement necessary symbols in source #define O2ParamImpl(classname) classname classname::sInstance; diff --git a/Common/Utils/include/CommonUtils/ConfigurableParamHelper.h b/Common/Utils/include/CommonUtils/ConfigurableParamHelper.h index f52a9e0675363..6e69fae03e6c3 100644 --- a/Common/Utils/include/CommonUtils/ConfigurableParamHelper.h +++ b/Common/Utils/include/CommonUtils/ConfigurableParamHelper.h @@ -34,7 +34,7 @@ struct ParamDataMember { std::string value; std::string provenance; - std::string toString(std::string const& prefix, bool showProv) const; + std::string toString(std::string const& prefix, bool showProv, size_t padding = 0) const; }; // ---------------------------------------------------------------- @@ -45,24 +45,29 @@ class _ParamHelper { private: static std::vector* getDataMembersImpl(std::string const& mainkey, TClass* cl, void*, - std::map const* provmap); + std::map const* provmap, size_t virtualoffset); static void fillKeyValuesImpl(std::string const& mainkey, TClass* cl, void*, boost::property_tree::ptree*, std::map>*, - EnumRegistry*); + EnumRegistry*, size_t offset); static void printWarning(std::type_info const&); static void assignmentImpl(std::string const& mainkey, TClass* cl, void* to, void* from, - std::map* provmap); + std::map* provmap, size_t offset); static void syncCCDBandRegistry(std::string const& mainkey, TClass* cl, void* to, void* from, - std::map* provmap); + std::map* provmap, size_t offset); - static void outputMembersImpl(std::ostream& out, std::string const& mainkey, std::vector const* members, bool showProv); - static void printMembersImpl(std::string const& mainkey, std::vector const* members, bool showProv); + static void outputMembersImpl(std::ostream& out, std::string const& mainkey, std::vector const* members, bool showProv, bool useLogger, bool withPadding = false, bool showHash = false); + static void printMembersImpl(std::string const& mainkey, std::vector const* members, bool showProv, bool useLogger, bool withPadding, bool showHash); + + static size_t getHashImpl(std::string const& mainkey, std::vector const* members); template friend class ConfigurableParamHelper; + + template + friend class ConfigurableParamPromoter; }; // ---------------------------------------------------------------- @@ -95,13 +100,19 @@ class ConfigurableParamHelper : virtual public ConfigurableParam // ---------------------------------------------------------------- // one of the key methods, using introspection to print itself - void printKeyValues(bool showProv = true) const final + void printKeyValues(bool showProv = true, bool useLogger = false, bool withPadding = true, bool showHash = true) const final { if (!isInitialized()) { initialize(); } auto members = getDataMembers(); - _ParamHelper::printMembersImpl(getName(), members, showProv); + _ParamHelper::printMembersImpl(getName(), members, showProv, useLogger, withPadding, showHash); + } + + // + size_t getHash() const final + { + return _ParamHelper::getHashImpl(getName(), getDataMembers()); } // ---------------------------------------------------------------- @@ -109,7 +120,7 @@ class ConfigurableParamHelper : virtual public ConfigurableParam void output(std::ostream& out) const final { auto members = getDataMembers(); - _ParamHelper::outputMembersImpl(out, getName(), members, true); + _ParamHelper::outputMembersImpl(out, getName(), members, true, false); } // ---------------------------------------------------------------- @@ -132,7 +143,7 @@ class ConfigurableParamHelper : virtual public ConfigurableParam return nullptr; } - return _ParamHelper::getDataMembersImpl(getName(), cl, (void*)this, sValueProvenanceMap); + return _ParamHelper::getDataMembersImpl(getName(), cl, (void*)this, sValueProvenanceMap, 0); } // ---------------------------------------------------------------- @@ -145,7 +156,7 @@ class ConfigurableParamHelper : virtual public ConfigurableParam _ParamHelper::printWarning(typeid(P)); return; } - _ParamHelper::fillKeyValuesImpl(getName(), cl, (void*)this, tree, sKeyToStorageMap, sEnumRegistry); + _ParamHelper::fillKeyValuesImpl(getName(), cl, (void*)this, tree, sKeyToStorageMap, sEnumRegistry, 0); } // ---------------------------------------------------------------- @@ -159,7 +170,7 @@ class ConfigurableParamHelper : virtual public ConfigurableParam file->GetObject(getName().c_str(), readback); if (readback != nullptr) { _ParamHelper::assignmentImpl(getName(), TClass::GetClass(typeid(P)), (void*)this, (void*)readback, - sValueProvenanceMap); + sValueProvenanceMap, 0); delete readback; } setRegisterMode(true); @@ -177,7 +188,146 @@ class ConfigurableParamHelper : virtual public ConfigurableParam // setRegisterMode(false); _ParamHelper::syncCCDBandRegistry(getName(), TClass::GetClass(typeid(P)), (void*)this, (void*)externalobj, - sValueProvenanceMap); + sValueProvenanceMap, 0); + setRegisterMode(true); + } + + // ---------------------------------------------------------------- + + void serializeTo(TFile* file) const final + { + file->WriteObjectAny((void*)this, TClass::GetClass(typeid(P)), getName().c_str()); + } +}; + +// Promotes a simple struct Base to a configurable parameter class +// Aka implements all interfaces for a ConfigurableParam P, which shares or +// takes the fields from a Base struct +template +class ConfigurableParamPromoter : public Base, virtual public ConfigurableParam +{ + public: + using ConfigurableParam::ConfigurableParam; + + static const P& Instance() + { + return P::sInstance; + } + + // extracts a copy of the underlying data struct + Base detach() const + { + static_assert(std::copyable, "Base type must be copyable."); + return static_cast(*this); + } + + // ---------------------------------------------------------------- + std::string getName() const final + { + return P::sKey; + } + + // ---------------------------------------------------------------- + // get the provenace of the member with given key + EParamProvenance getMemberProvenance(const std::string& key) const final + { + return getProvenance(getName() + '.' + key); + } + + // ---------------------------------------------------------------- + + // one of the key methods, using introspection to print itself + void printKeyValues(bool showProv = true, bool useLogger = false, bool withPadding = true, bool showHash = true) const final + { + if (!isInitialized()) { + initialize(); + } + auto members = getDataMembers(); + _ParamHelper::printMembersImpl(getName(), members, showProv, useLogger, withPadding, showHash); + } + + // + size_t getHash() const final + { + return _ParamHelper::getHashImpl(getName(), getDataMembers()); + } + + // ---------------------------------------------------------------- + + void output(std::ostream& out) const final + { + auto members = getDataMembers(); + _ParamHelper::outputMembersImpl(out, getName(), members, true, false); + } + + // ---------------------------------------------------------------- + + // Grab the list of ConfigurableParam data members + // Returns a nullptr if the TClass of the P template class cannot be created. + std::vector* getDataMembers() const + { + // just a helper line to make sure P::sInstance is looked-up + // and that compiler complains about missing static sInstance of type P + // volatile void* ptr = (void*)&P::sInstance; + // static assert on type of sInstance: + static_assert(std::is_same::value, + "static instance must of same type as class"); + + // obtain the TClass for the Base type and delegate further + auto cl = TClass::GetClass(typeid(Base)); + if (!cl) { + _ParamHelper::printWarning(typeid(Base)); + return nullptr; + } + + // we need to put an offset of 8 bytes since internally this is using data members of the Base class + // which doesn't account for the virtual table of P + return _ParamHelper::getDataMembersImpl(getName(), cl, (void*)this, sValueProvenanceMap, 8); + } + + // ---------------------------------------------------------------- + + // fills the data structures with the initial default values + void putKeyValues(boost::property_tree::ptree* tree) final + { + auto cl = TClass::GetClass(typeid(Base)); + if (!cl) { + _ParamHelper::printWarning(typeid(Base)); + return; + } + _ParamHelper::fillKeyValuesImpl(getName(), cl, (void*)this, tree, sKeyToStorageMap, sEnumRegistry, 8); + } + + // ---------------------------------------------------------------- + + void initFrom(TFile* file) final + { + // switch off auto registering since the readback object is + // only a "temporary" singleton + setRegisterMode(false); + P* readback = nullptr; + file->GetObject(getName().c_str(), readback); + if (readback != nullptr) { + _ParamHelper::assignmentImpl(getName(), TClass::GetClass(typeid(Base)), (void*)this, (void*)readback, + sValueProvenanceMap, 8); + delete readback; + } + setRegisterMode(true); + } + + // ---------------------------------------------------------------- + + void syncCCDBandRegistry(void* externalobj) final + { + // We may be getting an external copy from CCDB which is passed as externalobj. + // The task of this function is to + // a) update the internal registry with fields coming from CCDB + // but only if keys have not been modified via RT == command line / ini file + // b) update the external object with with fields having RT provenance + // + setRegisterMode(false); + _ParamHelper::syncCCDBandRegistry(getName(), TClass::GetClass(typeid(Base)), (void*)this, (void*)externalobj, + sValueProvenanceMap, 8); setRegisterMode(true); } diff --git a/Common/Utils/include/CommonUtils/ConfigurableParamReaders.h b/Common/Utils/include/CommonUtils/ConfigurableParamReaders.h new file mode 100644 index 0000000000000..d5ecd6cb97f7a --- /dev/null +++ b/Common/Utils/include/CommonUtils/ConfigurableParamReaders.h @@ -0,0 +1,36 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +#ifndef O2_COMMON_UTILS_CONFIGURABLEPARAMREADERS_H_ +#define O2_COMMON_UTILS_CONFIGURABLEPARAMREADERS_H_ + +#include +#include + +namespace o2::conf +{ + +// Helpers to read ConfigurableParam from different file formats +class ConfigurableParamReaders +{ + public: + static void setInputDir(const std::string& d) { sInputDir = d; } + static const std::string& getInputDir() { return sInputDir; } + + static boost::property_tree::ptree readINI(std::string const& filepath); + static boost::property_tree::ptree readJSON(std::string const& filepath); + static boost::property_tree::ptree readConfigFile(std::string const& filepath); + + private: + static std::string sInputDir; +}; + +} // namespace o2::conf +#endif // O2_COMMON_UTILS_CONF_CONFIGURABLEPARAMREADERS_H_ diff --git a/Common/Utils/include/CommonUtils/ConfigurableParamTest.h b/Common/Utils/include/CommonUtils/ConfigurableParamTest.h new file mode 100644 index 0000000000000..547bbf9ba8c38 --- /dev/null +++ b/Common/Utils/include/CommonUtils/ConfigurableParamTest.h @@ -0,0 +1,45 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef COMMON_CONFIGURABLE_PARAM_TEST_H_ +#define COMMON_CONFIGURABLE_PARAM_TEST_H_ + +#include "CommonUtils/ConfigurableParam.h" +#include "CommonUtils/ConfigurableParamHelper.h" + +namespace o2::conf::test +{ +struct TestParam : public o2::conf::ConfigurableParamHelper { + enum TestEnum : uint8_t { + A, + B, + C + }; + + int iValue{42}; + float fValue{3.14}; + double dValue{3.14}; + bool bValue{true}; + unsigned uValue{1}; + long lValue{1}; + unsigned long ulValue{1}; + long long llValue{1}; + unsigned long long ullValue{1}; + std::string sValue = "default"; + int iValueProvenanceTest{0}; + TestEnum eValue = TestEnum::C; + int caValue[3] = {0, 1, 2}; + + O2ParamDef(TestParam, "TestParam"); +}; +} // namespace o2::conf::test + +#endif diff --git a/Common/Utils/include/CommonUtils/ConfigurationMacroHelper.h b/Common/Utils/include/CommonUtils/ConfigurationMacroHelper.h index 79e8b63872c37..be9bbcad25772 100644 --- a/Common/Utils/include/CommonUtils/ConfigurationMacroHelper.h +++ b/Common/Utils/include/CommonUtils/ConfigurationMacroHelper.h @@ -14,11 +14,11 @@ #ifndef ALICEO2_CONF_CONFIGURATIONMACRO_H_ #define ALICEO2_CONF_CONFIGURATIONMACRO_H_ -#include "FairLogger.h" #include "TROOT.h" #include "TSystem.h" #include "TGlobal.h" #include "TFunction.h" +#include #include namespace o2 @@ -55,8 +55,9 @@ T GetFromMacro(const std::string& file, const std::string& funcname, const std:: } /** check the return type matches the required one **/ - if (strcmp(gROOT->GetGlobalFunction(gfunc.c_str())->GetReturnTypeName(), type.c_str())) { - LOG(info) << "Global function '" << gfunc << "' does not return a '" << type << "' type"; + auto returnedtype = gROOT->GetGlobalFunction(gfunc.c_str())->GetReturnTypeName(); + if (strcmp(returnedtype, type.c_str())) { + LOG(info) << "Global function '" << gfunc << "' does not return a '" << type << "' type ( but " << returnedtype << " )"; return nullptr; } @@ -68,6 +69,21 @@ T GetFromMacro(const std::string& file, const std::string& funcname, const std:: return *ptr; } +// just-in-time interpret some C++ function using ROOT and make result available to runtime +// functiondecl: A string coding the function to call (example "bool foo(){ return true; }") +// funcname: The name of the function to call (example "foo()") +// type: Return type of function (example "bool") +// unique: Some unique string identifier under which the result will be stored in gROOT global variable space +template +T JITAndEvalFunction(const std::string& functiondecl, const std::string& funcname, const std::string& type, const std::string& unique) +{ + /** interpret and execute a function and retrieve pointer to the returned type **/ + auto line = Form("%s; %s __%s__ = %s;", functiondecl.c_str(), type.c_str(), unique.c_str(), funcname.c_str()); + gROOT->ProcessLine(line); + auto ptr = (T*)gROOT->GetGlobal(Form("__%s__", unique.c_str()))->GetAddress(); + return *ptr; +} + } // namespace conf } // namespace o2 diff --git a/Common/Utils/include/CommonUtils/DLLoaderBase.h b/Common/Utils/include/CommonUtils/DLLoaderBase.h new file mode 100644 index 0000000000000..61d349876c92e --- /dev/null +++ b/Common/Utils/include/CommonUtils/DLLoaderBase.h @@ -0,0 +1,266 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +// \brief A thin wrapper to manage dynamic library loading, based around boost::dll + +#ifndef DLLOADER_H_ +#define DLLOADER_H_ + +#include +#include +#include +#include +#include +#include +#include +#include "dlfcn.h" + +#if defined(__APPLE__) +#define DLLOADER_MAC_LINUX(mac, linux) mac +#else +#define DLLOADER_MAC_LINUX(mac, linux) linux +#endif + +#include "Framework/Logger.h" + +namespace o2::utils +{ + +// Manages dynamic loading and unloading (or rather ensuring they are not +// unloaded for the duration of the program) of libraries and symbol lookups. It +// ensures thread-safety through the use of a mutex and implements the Meyers +// Singleton pattern to provide a single instance of the manager. +template +class DLLoaderBase +{ + public: + struct filename_decorations { + static constexpr const char* prefix = "lib"; // same prefix on mac and linux + static constexpr const char* suffix = DLLOADER_MAC_LINUX(".dylib", ".so"); + }; + enum Options { + none = 0, + global = RTLD_GLOBAL, + local = RTLD_LOCAL, + no_delete = RTLD_NODELETE, + no_load = RTLD_NOLOAD, + lazy = RTLD_LAZY, + }; + using handle_t = void; + using handle_ptr_t = handle_t*; + struct HandleDeleter { + void operator()(handle_ptr_t p) + { + if (p != nullptr) { + dlclose(p); + } + } + }; + using library_t = std::unique_ptr; + + // Returns the singleton instance of the manager. Any function should only be + // accessed through an instance. This being a singleton serves two purposes: + // 1. Libraries are loaded only once. + // 2. Once loaded they are not again unloaded until the end of the program. + static DerivedType& Instance() + { + return DerivedType::sInstance; + } + + // Loads a dynamic library by its name and stores its handle. Returns true + // if the library is successfully loaded or already loaded. + bool addLibrary(const std::string& library) + { + const std::lock_guard lock(mLock); + + if (mLibraries.find(library) != mLibraries.end()) { + return true; // Library already loaded + } + + if (mO2Path.empty()) { + if (const auto* path = std::getenv("O2_ROOT")) { + mO2Path = path; + } else { + LOGP(error, "$O2_ROOT not set!"); + return false; + } + } + + auto path = getO2Path(library); + if (!std::filesystem::exists(path)) { + LOGP(error, "Library under '{}' does not exist!", path); + return false; + } + + try { + auto lib = std::unique_ptr(dlopen(path.c_str(), mLoadPolicy)); + if (lib == nullptr) { + throw std::runtime_error("Library handle is nullptr!"); + } + mLibraries[library] = std::move(lib); + LOGP(info, "Loaded dynamic library '{}' from '{}'", library, path); + return true; + } catch (std::exception& e) { + LOGP(error, "Failed to load library (path='{}'), failed reason: '{}'", path, e.what()); + return false; + } catch (...) { + LOGP(error, "Failed to load library (path='{}') for unknown reason!", path.c_str()); + return false; + } + } + + // Unloads a given library returns true if this succeeded. + // + // Nota bene: Actually, we have very little control here when the unloading + // hapens since the linkder decides this based on if there is any reference + // left. And even if the reference counter goes to zero the linker is free to + // leave the library loaded and clean up whenever it wants. + bool unloadLibrary(const std::string& library) + { + const std::lock_guard lock(mLock); + + if (auto it = mLibraries.find(library); it != mLibraries.end()) { + mLibraries.erase(it); + return true; + } + + LOGP(error, "No '{}' library found, cannot unload it!", library); + return false; + } + + // Resets all loaded libraries and O2Path, this invalidates all outside kept + // references. + void reset() + { + mO2Path.clear(); + mLibraries.clear(); + } + + // Checks if a library contains a specific symbol. + bool hasSymbol(const std::string& library, const std::string& symbol) + { + const std::lock_guard lock(mLock); + + if (mLibraries.find(library) == mLibraries.end()) { + // Library not loaded, attempt to load it + if (!addLibrary(library)) { + return false; + } + } + + dlerror(); // clear previous error + + // Checks if the symbol exists but does not load it. + handle_ptr_t ptr = dlsym(mLibraries[library].get(), symbol.c_str()); + if (const auto* err = dlerror(); err != nullptr) { + LOGP(error, "Did not get {} from {}; error: {}", symbol, library, err); + } + + return ptr != nullptr; + } + + // Executes a function from a loaded library or return nullopt + template + std::optional executeFunction(const std::string& library, const std::string& fname, Args... args) + { + using Func_t = Ret (*)(Args...); + + const std::lock_guard lock(mLock); + + if (fname.empty()) { + LOGP(error, "Function name cannot be empty!"); + return std::nullopt; + } + + if (mLibraries.find(library) == mLibraries.end()) { + // Library not loaded, attempt to load it + if (!addLibrary(library)) { + return std::nullopt; + } + } + + const auto& lib = mLibraries[library].get(); + if (!hasSymbol(library, fname)) { + LOGP(error, "Library '{}' does not have a symbol '{}'", library, fname); + return std::nullopt; + } + + dlerror(); // Clear previous error + + auto func = (Func_t)dlsym(lib, fname.c_str()); + if (const auto* err = dlerror(); err != nullptr) { + LOGP(error, "Did not get {} from {}; error: {}", fname, library, err); + return std::nullopt; + } + + if (func == nullptr) { + LOGP(error, "Library '{}' does not have a symbol '{}' with {}", library, fname, getTypeName()); + return std::nullopt; + } + + // Execute function and return its return value + return func(args...); + } + + // Wrapper for function execution which fatals if execution fails + template + Ret executeFunctionAlias(const std::string& library, const std::string& fname, Args... args) + { + if (auto opt = executeFunction(library, fname, args...)) { + return *opt; + } + + LOGP(fatal, "Execution of '{}' from '{}' failed spectaculary!", fname, library); + __builtin_unreachable(); // is this safe, AFACIT only gcc and clang are supported anyway + } + + // Delete copy and move constructors to enforce singleton pattern. + DLLoaderBase(const DLLoaderBase&) = delete; + DLLoaderBase& operator=(const DLLoaderBase&) = delete; + DLLoaderBase(DLLoaderBase&&) = delete; + DLLoaderBase& operator=(DLLoaderBase&&) = delete; + + protected: + // Constructor and destructor are protected to enforce singleton pattern. + DLLoaderBase() = default; + ~DLLoaderBase() = default; + + private: + // Returns the full path to a O2 shared library.. + [[nodiscard]] std::string getO2Path(const std::string& library) const + { + return mO2Path + "/lib/" + filename_decorations::prefix + library + filename_decorations::suffix; + } + + // Returns the demangled type name of a prototype, e.g., for pretty printing. + template + [[nodiscard]] auto getTypeName() -> std::string + { + return typeid(ProtoType).name(); // TODO + } + + std::unordered_map mLibraries{}; // Pointers to loaded libraries, calls `unload()' for each library, e.g., correctly destroy this object + std::recursive_mutex mLock{}; // While a recursive mutex is more expansive it makes locking easier + std::string mO2Path{}; // Holds the path O2 dynamic library determined by $O2_ROOT + Options mLoadPolicy{Options::lazy}; // load resolution policy +}; + +} // namespace o2::utils + +#define O2DLLoaderDef(classname) \ + private: \ + static classname sInstance; \ + classname() = default; \ + friend class o2::utils::DLLoaderBase; + +#define O2DLLoaderImpl(classname) classname classname::sInstance; + +#endif // DLLOADER_H_ diff --git a/Common/Utils/include/CommonUtils/DebugStreamer.h b/Common/Utils/include/CommonUtils/DebugStreamer.h index ccf663f042a54..741afaabfdd60 100644 --- a/Common/Utils/include/CommonUtils/DebugStreamer.h +++ b/Common/Utils/include/CommonUtils/DebugStreamer.h @@ -19,8 +19,10 @@ #include "GPUCommonDef.h" #if !defined(GPUCA_GPUCODE) && !defined(GPUCA_STANDALONE) #include "CommonUtils/ConfigurableParamHelper.h" +#include #if defined(DEBUG_STREAMER) #include "CommonUtils/TreeStreamRedirector.h" +#include #endif #endif @@ -29,7 +31,26 @@ namespace o2::utils /// struct defining the flags which can be used to check if a certain debug streamer is used enum StreamFlags { - streamdEdx = 1 << 0, ///< stream corrections and cluster properties used for the dE/dx + streamdEdx = 1 << 0, ///< stream corrections and cluster properties used for the dE/dx + streamDigitFolding = 1 << 1, ///< stream ion tail and saturatio information + streamDigits = 1 << 2, ///< stream digit information + streamFastTransform = 1 << 3, ///< stream tpc fast transform + streamITCorr = 1 << 4, ///< stream ion tail correction information + streamDistortionsSC = 1 << 5, ///< stream distortions applied in the TPC space-charge class (used for example in the tpc digitizer) + streamUpdateTrack = 1 << 6, ///< stream update track informations + streamRejectCluster = 1 << 7, ///< stream cluster rejection informations + streamMergeBorderTracksBest = 1 << 8, ///< stream MergeBorderTracks best track + streamMergeBorderTracksAll = 1 << 9, ///< stream MergeBorderTracks all tracks + streamFlagsCount = 10 ///< total number of streamers +}; + +enum SamplingTypes { + sampleAll = 0, ///< use all data (default) + sampleRandom = 1, ///< sample randomly every n points + sampleID = 2, ///< sample every n IDs (per example track) + sampleIDGlobal = 3, ///< in case different streamers have access to the same IDs use this gloabl ID + sampleWeights = 4, ///< perform sampling on weights, defined where the streamer is called + sampleTsallis = 5, ///< perform sampling on tsallis pdf }; #if !defined(GPUCA_GPUCODE) && !defined(GPUCA_STANDALONE) @@ -43,7 +64,10 @@ inline StreamFlags operator~(StreamFlags a) { return static_cast(~s /// struct for setting and storing the streamer level struct ParameterDebugStreamer : public o2::conf::ConfigurableParamHelper { - StreamFlags StreamLevel{}; /// flag to store what will be streamed + int streamLevel{}; /// flag to store what will be streamed + SamplingTypes samplingType[StreamFlags::streamFlagsCount]{}; ///< sampling type for each streamer (default = SamplingTypes::sampleAll) + float samplingFrequency[StreamFlags::streamFlagsCount]{0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1}; ///< frequency which is used for the sampling (0.1 -> 10% is written if sampling is used) + int sampleIDGlobal[StreamFlags::streamFlagsCount]{}; ///< storage of reference streamer used for sampleIDFromOtherStreamer O2ParamDef(ParameterDebugStreamer, "DebugStreamerParam"); }; @@ -56,32 +80,69 @@ class DebugStreamer // CPU implementation of the class #if !defined(GPUCA_GPUCODE) && !defined(GPUCA_STANDALONE) && defined(DEBUG_STREAMER) public: + /// default constructor + DebugStreamer(); + + static DebugStreamer* instance() + { + static DebugStreamer streamer; + return &streamer; + } + /// set the streamer i.e. create the output file and set up the streamer /// \param outFile output file name without .root suffix /// \param option RECREATE or UPDATE - void setStreamer(const char* outFile, const char* option); + /// \param id unique id for given streamer (i.e. for defining unique streamers for each thread) + void setStreamer(const char* outFile, const char* option, const size_t id = getCPUID()); /// \return returns if the streamer is set - bool isStreamerSet() const { return mTreeStreamer ? true : false; } + /// \param id unique id of streamer + bool isStreamerSet(const size_t id = getCPUID()) const { return getStreamerPtr(id); } + + /// flush TTree for given ID to disc + /// \param id unique id of streamer + void flush(const size_t id); + + /// flush all TTrees to disc + void flush(); + + /// \return returns streamer object for given id + /// \param id unique id of streamer + o2::utils::TreeStreamRedirector& getStreamer(const size_t id = getCPUID()) { return *(mTreeStreamer[id]); } + + /// \return returns streamer object for given id + /// \param outFile output file name without .root suffix + /// \param option RECREATE or UPDATE + /// \param id unique id of streamer + o2::utils::TreeStreamRedirector& getStreamer(const char* outFile, const char* option, const size_t id = getCPUID()); /// \return returns streamer object - o2::utils::TreeStreamRedirector& getStreamer() { return *mTreeStreamer; } + /// \param id unique id of streamer + o2::utils::TreeStreamRedirector* getStreamerPtr(const size_t id = getCPUID()) const; /// \return returns streamer level i.e. what will be written to file - static StreamFlags getStreamFlags() { return ParameterDebugStreamer::Instance().StreamLevel; } + static StreamFlags getStreamFlags() { return static_cast(ParameterDebugStreamer::Instance().streamLevel); } + + /// \return returns sampling type and sampling frequency for given streamer + static std::pair getSamplingTypeFrequency(const StreamFlags streamFlag); + + /// \return returns sampling type and sampling frequency for given streamer + static float getSamplingFrequency(const StreamFlags streamFlag) { return getSamplingTypeFrequency(streamFlag).second; } ///< return returns unique ID for each CPU thread to give each thread an own output file static size_t getCPUID(); /// \return returns number of trees in the streamer - int getNTrees() const; + /// \param id unique id of streamer + int getNTrees(const size_t id = getCPUID()) const; /// \return returns an unique branch name which is not already written in the file /// \param tree name of the tree for which to get a unique tree name - std::string getUniqueTreeName(const char* tree) const; + /// \param id unique id of streamer + std::string getUniqueTreeName(const char* tree, const size_t id = getCPUID()) const; /// set directly the debug level - static void setStreamFlags(const StreamFlags streamFlags) { o2::conf::ConfigurableParam::setValue("DebugStreamerParam", "StreamLevel", static_cast(streamFlags)); } + static void setStreamFlags(const StreamFlags streamFlags) { o2::conf::ConfigurableParam::setValue("DebugStreamerParam", "streamLevel", static_cast(streamFlags)); } /// enable specific streamer flag static void enableStream(const StreamFlags streamFlag); @@ -90,7 +151,9 @@ class DebugStreamer static void disableStream(const StreamFlags streamFlag); /// check if streamer for specific flag is enabled - static bool checkStream(const StreamFlags streamFlag) { return ((getStreamFlags() & streamFlag) == streamFlag); } + /// \param samplingID optional index of the data which is streamed in to perform sampling on this index + /// \param weight weight which can be used to perform some weightes sampling + static bool checkStream(const StreamFlags streamFlag, const size_t samplingID = -1, const float weight = 1); /// merge trees with the same content structure, but different naming /// \param inpFile input file containing several trees with the same content @@ -98,8 +161,15 @@ class DebugStreamer /// \param option setting which is used for the merging static void mergeTrees(const char* inpFile, const char* outFile, const char* option = "fast"); + /// \return returns integer index for given streamer flag + static int getIndex(const StreamFlags streamFlag); + + /// get random value between min and max + static float getRandom(float min = 0, float max = 1); + private: - std::unique_ptr mTreeStreamer; ///< streamer which is used for the debugging + using StreamersPerFlag = tbb::concurrent_unordered_map>; + StreamersPerFlag mTreeStreamer; ///< streamer which is used for the debugging #else // empty implementation of the class for GPU or when the debug streamer is not build for CPU @@ -109,7 +179,9 @@ class DebugStreamer GPUd() void setStreamer(Args... args){}; /// always false for GPU - GPUd() static bool checkStream(const StreamFlags) { return false; } + GPUd() static bool checkStream(const StreamFlags, const int samplingID = 0) { return false; } + + GPUd() static DebugStreamer* instance() { return nullptr; } class StreamerDummy { @@ -123,14 +195,23 @@ class DebugStreamer } }; - GPUd() StreamerDummy getStreamer() const { return StreamerDummy{}; }; + GPUd() StreamerDummy getStreamer(const int id = 0) const { return StreamerDummy{}; }; + + /// empty for GPU + template + GPUd() StreamerDummy getStreamer(Args... args) const + { + return StreamerDummy{}; + }; template - GPUd() StreamerDummy getUniqueTreeName(Type) const + GPUd() StreamerDummy getUniqueTreeName(Type, const int id = 0) const { return StreamerDummy{}; } + GPUd() void flush() const {}; + #endif }; diff --git a/Common/Utils/include/CommonUtils/EnumFlags.h b/Common/Utils/include/CommonUtils/EnumFlags.h new file mode 100644 index 0000000000000..e7481c903e666 --- /dev/null +++ b/Common/Utils/include/CommonUtils/EnumFlags.h @@ -0,0 +1,796 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +#ifndef O2_FRAMEWORK_FLAGS_H_ +#define O2_FRAMEWORK_FLAGS_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "CommonUtils/StringUtils.h" + +namespace o2::utils +{ + +namespace details::enum_flags +{ + +// Require that an enum with an underlying unsigned type. +template +concept EnumFlagHelper = requires { + requires std::is_enum_v; + requires std::is_unsigned_v>; + requires std::same_as>; +}; + +// Static constexpr only helper struct to implement modicum of enum reflection +// functions and also check via concepts expected properties of the enum. +// This is very much inspired by much more extensive libraries like magic_enum. +// Inspiration by its c++20 version (https://github.com/fix8mt/conjure_enum). +// NOTE: Cannot detect if bit values past the underlying type are defined. +template +struct FlagsHelper final { + using U = std::underlying_type_t; + using UMax = uint64_t; // max represetable type + static_assert(std::numeric_limits::digits <= std::numeric_limits::digits, "Underlying type has more digits than max supported digits"); + + static constexpr bool isScoped() noexcept + { + return std::is_enum_v && !std::is_convertible_v>; + } + + // Return line at given position. + template + static consteval const char* tpeek() noexcept + { + return std::source_location::current().function_name(); + } + // string_view value of function above + template + static constexpr std::string_view tpeek_v{tpeek()}; + + // Compiler Specifics + static constexpr auto CSpecifics{std::to_array< + std::tuple>({ +#if defined __clang__ + {"e = ", ']', "(anonymous namespace)", '('}, + {"T = ", ']', "(anonymous namespace)", '('}, +#else // assuming __GNUC__ + {"e = ", ';', "", '<'}, + {"T = ", ']', "{anonymous}", '{'}, +#endif + })}; + enum class SVal : uint8_t { Start, + End, + AnonStr, + AnonStart }; + enum class SType : uint8_t { Enum_t, + Type_t, + eT0, + eT1, + eT2, + eT3 }; + // Extract a compiler specification. + template + static constexpr auto getSpec() noexcept + { + return std::get(v)>(CSpecifics[static_cast(t)]); + } + + // Range that is scanned by the compiler + static constexpr size_t MinScan{0}; + static constexpr size_t MarginScan{1}; // Scan one past to check for overpopulation + static constexpr size_t MaxUnderScan{std::numeric_limits::digits}; // Maximum digits the underlying type has + static constexpr size_t MaxScan{MaxUnderScan + MarginScan}; + + // Checks if a given 'location' contains an enum. + template + static constexpr bool isValid() noexcept + { + constexpr auto tp{tpeek_v.rfind(getSpec())}; + if constexpr (tp == std::string_view::npos) { + return false; + } +#if defined __clang__ + else if constexpr (tpeek_v[tp + getSpec().size()] == '(') { + if constexpr (tpeek_v[tp + getSpec().size() + 1] == '(') { + return false; + } + if constexpr (tpeek_v.find(getSpec(), tp + getSpec().size()) != std::string_view::npos) { + return true; + } + } else if constexpr (tpeek_v.find_first_of(getSpec(), tp + getSpec().size()) != std::string_view::npos) { + // check if this is an anonymous enum + return true; + } +#elif __GNUC__ + else if constexpr (tpeek_v[tp + getSpec().size()] != '(' && tpeek_v.find_first_of(getSpec(), tp + getSpec().size()) != std::string_view::npos) { + return true; + } +#else +#error Unsupported compiler +#endif + return false; + } + + // Extract which values are present in the enum by checking all values in + // the min-max-range above. + template + static constexpr auto getValues(std::index_sequence /*unused*/) noexcept + { + constexpr std::array valid{isValid(MinScan + I)>()...}; + constexpr auto count{std::count_if(valid.cbegin(), valid.cend(), [](bool v) noexcept { return v; })}; + static_assert(count > 0, "EnumFlag requires at least one enum value. Check that your enum has consecutive values starting from 0."); + static_assert(count <= MaxUnderScan, "Too many enum values for underlying type. Consider using a larger underlying type or fewer enum values."); + std::array values{}; + for (size_t idx{}, n{}; n < count; ++idx) { + if (valid[idx]) { + values[n++] = static_cast(MinScan + idx); + } + } + return values; + } + static constexpr auto Values{getValues(std::make_index_sequence())}; // Enum Values + static constexpr auto count() noexcept { return Values.size(); } // Number of enum members + static constexpr auto Min_v{Values.front()}; // Enum first entry + static constexpr auto Max_v{Values.back()}; // Enum last entry + static constexpr auto Min_u_v{static_cast(Min_v)}; // Enum first entry as size_t + static constexpr auto Max_u_v{static_cast(Max_v)}; // Enum last entry as size_t + static_assert(Max_u_v < std::numeric_limits::digits, "Max Bit is beyond allow range deferred from underlying type"); + static constexpr bool isContinuous() noexcept { return (Max_u_v - Min_u_v + 1) == count(); } // Is the enum continuous + static constexpr UMax makeMaxRep(size_t min, size_t max) + { + const size_t width = max - min + 1; + if (width >= std::numeric_limits::digits) { + return std::numeric_limits::max(); + } + return ((UMax(1) << width) - 1) << min; + } + static constexpr auto MaxRep{makeMaxRep(Min_u_v, Max_u_v)}; // largest representable value + + template + static constexpr std::string_view getName() + { + constexpr auto tp{tpeek_v.rfind(getSpec())}; + if constexpr (tp == std::string_view::npos) { + return {}; + } + if constexpr (tpeek_v[tp + getSpec().size()] == getSpec()) { +#if defined __clang__ + if constexpr (tpeek_v[tp + getSpec().size() + 1] == getSpec()) { + return {}; + } +#endif + if (constexpr auto lstr{tpeek_v.substr(tp + getSpec().size())}; lstr.find(getSpec()) != std::string_view::npos) { // is anon + if constexpr (constexpr auto lc{lstr.find_first_of(getSpec())}; lc != std::string_view::npos) { + return lstr.substr(getSpec().size() + 2, lc - (getSpec().size() + 2)); + } + } + } + constexpr std::string_view result{tpeek_v.substr(tp + getSpec().size())}; + if constexpr (constexpr auto lc{result.find_first_of(getSpec())}; lc != std::string_view::npos) { + return result.substr(0, lc); + } else { + return {}; + } + } + + static constexpr std::string_view removeScope(std::string_view s) + { + if (const auto lc{s.find_last_of(':')}; lc != std::string_view::npos) { + return s.substr(lc + 1); + } + return s; + } + + static constexpr std::string_view findScope(std::string_view s) + { + const auto pos1 = s.rfind("::"); + if (pos1 == std::string_view::npos) { + return s; + } + const auto pos2 = s.rfind("::", pos1 - 1); + if (pos2 == std::string_view::npos) { + return s.substr(0, pos1); + } + return s.substr(pos2 + 2, pos1 - pos2 - 2); + } + + template + static constexpr auto getNameValue{getName()}; + + template + static constexpr auto getNames(std::index_sequence /*unused*/) + { + if constexpr (with_scope) { + return std::array{getNameValue...}; + } else { + return std::array{removeScope(getNameValue)...}; + } + } + + static constexpr auto Names{getNames(std::make_index_sequence())}; // Enum names without scope + static constexpr auto NamesScoped{getNames(std::make_index_sequence())}; // Enum names with scope + static constexpr auto Scope{findScope(NamesScoped.front())}; // Enum scope + + static constexpr auto getLongestName() noexcept + { + size_t max{0}; + for (size_t i{0}; i < count(); ++i) { + max = std::max(max, Names[i].size()); + } + return max; + } + + static constexpr auto NamesLongest{getLongestName()}; // Size of longest name + + template + static constexpr std::string_view toString() noexcept + { + return getNameValue(); + } + + static constexpr std::optional fromString(std::string_view str) noexcept + { + for (size_t i{0}; i < count(); ++i) { + if (isIEqual(Names[i], str) || isIEqual(NamesScoped[i], str)) { + return Values[i]; + } + } + return std::nullopt; + } + + // Convert char to lower. + static constexpr unsigned char toLower(const unsigned char c) noexcept + { + return (c >= 'A' && c <= 'Z') ? (c - 'A' + 'a') : c; + } + + // Are these chars equal (case-insensitive). + static constexpr bool isIEqual(const unsigned char a, const unsigned char b) noexcept + { + return toLower(a) == toLower(b); + } + + // Case-insensitive comparison for string_view. + static constexpr bool isIEqual(std::string_view s1, std::string_view s2) noexcept + { + if (s1.size() != s2.size()) { + return false; + } + for (size_t i{0}; i < s1.size(); ++i) { + if (!isIEqual(s1[i], s2[i])) { + return false; + } + } + return true; + } + + static constexpr std::string_view None{"none"}; + static constexpr bool hasNone() noexcept + { + // check that enum does not contain member named 'none' + for (size_t i{0}; i < count(); ++i) { + if (isIEqual(Names[i], None)) { + return true; + } + } + return false; + } + + static constexpr std::string_view All{"all"}; + static constexpr bool hasAll() noexcept + { + // check that enum does not contain member named 'all' + for (size_t i{0}; i < count(); ++i) { + if (isIEqual(Names[i], All)) { + return true; + } + } + return false; + } +}; + +} // namespace details::enum_flags + +// Require an enum to fullfil what one would except from a bitset. +template +concept EnumFlag = requires { + // range checks + requires details::enum_flags::FlagsHelper::Min_u_v == 0; // the first bit should be at position 0 + requires details::enum_flags::FlagsHelper::Max_u_v < details::enum_flags::FlagsHelper::count(); // the maximum is less than the total + requires details::enum_flags::FlagsHelper::isContinuous(); // do not allow missing bits + + // type checks + requires !details::enum_flags::FlagsHelper::hasNone(); // added automatically + requires !details::enum_flags::FlagsHelper::hasAll(); // added automatically +}; + +/** + * \brief Class to aggregate and manage enum-based on-off flags. + * + * This class manages flags as bits in the underlying type of an enum (upto 64 bits), allowing + * manipulation via enum member names. It supports operations akin to std::bitset + * but is fully constexpr and is ideal for aggregating multiple on-off booleans, + * e.g., enabling/disabling algorithm features. + * + * Example: + * enum class AlgoOptions { + * Feature1, + * Feature2, + * Feature3, + * }; + * ... + * EnumFlags opts; + * opts.set("Feature1 | Feature3"); // Set Feature1 and Feature3. + * if (opts[AlgoOptions::Feature1]) { // Do some work. } // Check if Feature1 is set. + * + * Additional examples of how to use this class are in testEnumFlags.cxx. + */ +template +class EnumFlags +{ + static constexpr int DefaultBase{2}; + using H = details::enum_flags::FlagsHelper; + using U = std::underlying_type_t; + U mBits{0}; + + // Converts enum to its underlying type. + constexpr auto to_underlying(E e) const noexcept + { + return static_cast(e); + } + + // Returns the bit representation of a flag. + constexpr auto to_bit(E e) const noexcept + { + return U(1) << to_underlying(e); + } + + public: + // Default constructor. + constexpr explicit EnumFlags() = default; + // Constructor to initialize with a single flag. + constexpr explicit EnumFlags(E e) : mBits(to_bit(e)) {} + // Copy constructor. + constexpr EnumFlags(const EnumFlags&) = default; + // Move constructor. + constexpr EnumFlags(EnumFlags&&) = default; + // Constructor to initialize with the underlying type. + constexpr explicit EnumFlags(U u) : mBits(u) {} + // Initialize with a list of flags. + constexpr EnumFlags(std::initializer_list flags) noexcept + { + std::for_each(flags.begin(), flags.end(), [this](const E f) noexcept { mBits |= to_bit(f); }); + } + // Init from a string. + // + explicit EnumFlags(const std::string& str, int base = DefaultBase) + { + set(str, base); + } + // Destructor. + constexpr ~EnumFlags() = default; + + static constexpr U None{0}; // Represents no flags set. + static constexpr U All{H::MaxRep}; // Represents all flags set. + + // Return list of all enum values + static constexpr auto getValues() noexcept + { + return H::Values; + } + + // Return list of all enum Names + static constexpr auto getNames() noexcept + { + return H::Names; + } + + // Sets flags from a string representation. + // This can be either from a number representation (binary or digits) or + // a concatenation of the enums members name e.g., 'Enum1|Enum2|...' + void set(const std::string& s, int base = DefaultBase) + { + if (s.empty()) { // no-op + return; + } + // on throw restore previous state and rethrow + const U prev = mBits; + reset(); + try { + setImpl(s, base); + } catch (const std::exception& e) { + mBits = prev; + throw; + } + } + // Returns the raw bitset value. + [[nodiscard]] constexpr auto value() const noexcept + { + return mBits; + } + + // Resets all flags. + constexpr void reset() noexcept + { + mBits = U(0); + } + + // Resets a specific flag. + template T> + constexpr void reset(T t) + { + mBits &= ~to_bit(t); + } + + // Tests if a specific flag is set. + template T> + [[nodiscard]] constexpr bool test(T t) const noexcept + { + return (mBits & to_bit(t)) != None; + } + + // Tests if all specified flags are set. + template ... Ts> + [[nodiscard]] constexpr bool test(Ts... flags) const noexcept + { + return ((test(flags) && ...)); + } + + // Sets a specific flag. + template T> + constexpr void set(T t) noexcept + { + mBits |= to_bit(t); + } + + // Sets multiple specific flags. + template ... Ts> + constexpr void set(Ts... flags) noexcept + { + (set(flags), ...); + } + + // Toggles a specific flag. + template T> + constexpr void toggle(T t) noexcept + { + mBits ^= to_bit(t); + } + + // Checks if any flag is set. + [[nodiscard]] constexpr bool any() const noexcept + { + return mBits != None; + } + + // Checks if all flags are set. + [[nodiscard]] constexpr bool all() const noexcept + { + return mBits == All; + } + + // Returns the bitset as a binary string. + [[nodiscard]] std::string string() const + { + std::ostringstream oss; + oss << std::bitset(mBits); + return oss.str(); + } + + // Returns the bitset as a pretty multiline binary string. + [[nodiscard]] std::string pstring(bool withNewline = false) const + { + std::ostringstream oss; + if (withNewline) { + oss << '\n'; + } + oss << "0b"; + const std::bitset bits(mBits); + oss << bits; + if constexpr (H::isScoped()) { + oss << " " << H::Scope; + } + oss << '\n'; + for (size_t i = 0; i < H::count(); ++i) { + oss << " "; + for (size_t j = 0; j < H::count() - i - 1; ++j) { + oss << "┃"; + } + oss << "┗"; + for (size_t a{2 + i}; --a != 0U;) { + oss << "━"; + } + oss << " " << std::setw(H::NamesLongest) << std::left + << H::Names[i] << " " << (bits[i] ? "[Active]" : "[Inactive]"); + if (i != H::count() - 1) { + oss << "\n"; + } + } + return oss.str(); + } + + // Checks if any flag is set (Boolean context). + [[nodiscard]] constexpr explicit operator bool() const noexcept + { + return any(); + } + + // Check if given flag is set. + template T> + [[nodiscard]] constexpr bool operator[](const T t) const noexcept + { + return test(t); + } + + // Checks if two flag sets are equal. + [[nodiscard]] constexpr bool operator==(const EnumFlags& o) const noexcept + { + return mBits == o.mBits; + } + + // Checks if two flag sets are not equal. + [[nodiscard]] constexpr bool operator!=(const EnumFlags& o) const noexcept + { + return mBits != o.mBits; + } + + // Copy assignment operator + constexpr EnumFlags& operator=(const EnumFlags& o) = default; + + // Move assignment operator + constexpr EnumFlags& operator=(EnumFlags&& o) = default; + + // Performs a bitwise OR with a flag. + template T> + constexpr EnumFlags& operator|=(T t) noexcept + { + mBits |= to_bit(t); + return *this; + } + + // Performs a bitwise AND with a flag. + template T> + constexpr EnumFlags& operator&=(T t) noexcept + { + mBits &= to_bit(t); + return *this; + } + + // Returns a flag set with a bitwise AND. + template T> + constexpr EnumFlags operator&(T t) const noexcept + { + return EnumFlags(mBits & to_bit(t)); + } + + // Returns a flag set with all bits inverted. + constexpr EnumFlags operator~() const noexcept + { + return EnumFlags(~mBits); + } + + // Performs a bitwise OR with another flag set. + constexpr EnumFlags operator|(const EnumFlags& o) const noexcept + { + return EnumFlags(mBits | o.mBits); + } + + // Performs a bitwise OR assignment. + constexpr EnumFlags& operator|=(const EnumFlags& o) noexcept + { + mBits |= o.mBits; + return *this; + } + + // Performs a bitwise XOR with another flag set. + constexpr EnumFlags operator^(const EnumFlags& o) const noexcept + { + return EnumFlags(mBits ^ o.mBits); + } + + // Performs a bitwise and with another flag set. + constexpr EnumFlags operator&(const EnumFlags& o) const noexcept + { + return EnumFlags(mBits & o.mBits); + } + + // Performs a bitwise XOR assignment. + constexpr EnumFlags& operator^=(const EnumFlags& o) noexcept + { + mBits ^= o.mBits; + return *this; + } + + // Checks if all specified flags are set. + template + [[nodiscard]] constexpr bool all_of(Ts... flags) const noexcept + { + return test(flags...); + } + + // Checks if none of the specified flags are set. + template + [[nodiscard]] constexpr bool none_of(Ts... flags) const noexcept + { + return (!(test(flags) || ...)); + } + + // Serializes the flag set to a string. + [[nodiscard]] std::string serialize() const + { + return std::to_string(mBits); + } + + // Deserializes a string into the flag set. + void deserialize(const std::string& data) + { + typename H::UMax v = std::stoul(data); + if (v > H::MaxRep) { + throw std::out_of_range("Values exceeds enum range."); + } + mBits = static_cast(v); + } + + // Counts the number of set bits (active flags). + [[nodiscard]] constexpr size_t count() const noexcept + { + return std::popcount(mBits); + } + + // Returns the union of two flag sets. + [[nodiscard]] constexpr EnumFlags union_with(const EnumFlags& o) const noexcept + { + return EnumFlags(mBits | o.mBits); + } + + // Returns the intersection of two flag sets. + [[nodiscard]] constexpr EnumFlags intersection_with(const EnumFlags& o) const noexcept + { + return EnumFlags(mBits & o.mBits); + } + + // Checks if all flags in another Flags object are present in the current object. + [[nodiscard]] constexpr bool contains(const EnumFlags& other) const noexcept + { + return (mBits & other.mBits) == other.mBits; + } + + private: + // Set implementation, bits was zeroed before. + void setImpl(const std::string& s, int base = 2) + { + // Helper to check if character is valid for given base + auto isValidForBase = [](unsigned char c, int base) -> bool { + if (base == 2) { + return c == '0' || c == '1'; + } + if (base == 10) { + return std::isdigit(c); + } + if (base == 16) { + return std::isdigit(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); + } + return false; + }; + + // hex + if (base == 16) { + std::string_view hex_str{s}; + // Strip optional 0x or 0X prefix + if (s.size() >= 2 && s[0] == '0' && (s[1] == 'x' || s[1] == 'X')) { + hex_str.remove_prefix(2); + } + if (hex_str.empty()) { + throw std::invalid_argument("Empty hexadecimal string."); + } + if (!std::all_of(hex_str.begin(), hex_str.end(), [&](unsigned char c) { return isValidForBase(c, 16); })) { + throw std::invalid_argument("Invalid hexadecimal string."); + } + typename H::UMax v = std::stoul(std::string(hex_str), nullptr, 16); + if (v > H::MaxRep) { + throw std::out_of_range("Value exceeds enum range."); + } + mBits = static_cast(v); + return; + } + + // decimal and binary + if (std::all_of(s.begin(), s.end(), [](unsigned char c) { return std::isdigit(c); })) { + if (base == 2) { + // Binary: check only 0 and 1 + if (!std::all_of(s.begin(), s.end(), [&](unsigned char c) { return isValidForBase(c, 2); })) { + throw std::invalid_argument("Invalid binary string."); + } + } + typename H::UMax v = std::stoul(std::string(s), nullptr, base); + if (v > H::MaxRep) { + throw std::out_of_range("Value exceeds enum range."); + } + mBits = static_cast(v); + } + // enum name strings + else if (std::all_of(s.begin(), s.end(), [](unsigned char c) { return std::isalnum(c) != 0 || c == '|' || c == ' ' || c == ':' || c == ',' || c == ';'; })) { + std::string cs{s}; + std::transform(cs.begin(), cs.end(), cs.begin(), [](unsigned char c) { return std::tolower(c); }); + + if (cs == H::All) { + mBits = All; + } else if (cs == H::None) { + mBits = None; + } else { + // Detect delimiter and ensure only one type is used + char token = ' '; + size_t pipePos = s.find('|'); + size_t commaPos = s.find(','); + size_t semiPos = s.find(';'); + + // Count how many different delimiters exist + int delimiterCount = (pipePos != std::string_view::npos ? 1 : 0) + + (commaPos != std::string_view::npos ? 1 : 0) + + (semiPos != std::string_view::npos ? 1 : 0); + + if (delimiterCount > 1) { + throw std::invalid_argument("Mixed delimiters not allowed!"); + } + + if (pipePos != std::string_view::npos) { + token = '|'; + } else if (commaPos != std::string_view::npos) { + token = ','; + } else if (semiPos != std::string_view::npos) { + token = ';'; + } + + for (const auto& tok : Str::tokenize(std::string(s), token)) { + if (auto e = H::fromString(tok)) { + mBits |= to_bit(*e); + } else { + throw std::invalid_argument(tok + " is not a valid enum value!"); + } + } + } + } else { + throw std::invalid_argument("Cannot parse string!"); + } + } +}; + +template +std::ostream& operator<<(std::ostream& os, const EnumFlags& f) +{ + os << f.pstring(true); + return os; +} + +} // namespace o2::utils + +#endif diff --git a/Common/Utils/include/CommonUtils/FIFO.h b/Common/Utils/include/CommonUtils/FIFO.h index 94942277daa0a..83afa0b2206c9 100644 --- a/Common/Utils/include/CommonUtils/FIFO.h +++ b/Common/Utils/include/CommonUtils/FIFO.h @@ -17,6 +17,7 @@ #include #include +#include namespace o2 { diff --git a/Common/Utils/include/CommonUtils/FileFetcher.h b/Common/Utils/include/CommonUtils/FileFetcher.h index 83d77635eef7c..b1c1ff6399a65 100644 --- a/Common/Utils/include/CommonUtils/FileFetcher.h +++ b/Common/Utils/include/CommonUtils/FileFetcher.h @@ -58,10 +58,12 @@ class FileFetcher ~FileFetcher(); const auto& getFileRef(size_t i) const { return mInputFiles[i]; } - + void setFailThreshold(float f) { mFailThreshold = f; } + float getFailThreshold() const { return mFailThreshold; } void setMaxFilesInQueue(size_t s) { mMaxInQueue = s > 0 ? s : 1; } void setMaxLoops(size_t v) { mMaxLoops = v; } bool isRunning() const { return mRunning; } + bool isFailed() const { return mFailure; } void start(); void stop(); void cleanup(); @@ -100,10 +102,12 @@ class FileFetcher size_t mMaxInQueue{5}; bool mRunning = false; bool mNoRemoteCopy = false; + bool mFailure = false; size_t mMaxLoops = 0; size_t mNLoops = 0; size_t mNFilesProc = 0; size_t mNFilesProcOK = 0; + float mFailThreshold = 0.f; // throw if too many failed fetches (>0 : fraction to total, <0 abs number) mutable std::mutex mMtx; std::mutex mMtxStop; std::thread mFetcherThread{}; diff --git a/Common/Utils/include/CommonUtils/FileSystemUtils.h b/Common/Utils/include/CommonUtils/FileSystemUtils.h index e6852749d311e..9f0c486820350 100644 --- a/Common/Utils/include/CommonUtils/FileSystemUtils.h +++ b/Common/Utils/include/CommonUtils/FileSystemUtils.h @@ -32,6 +32,13 @@ std::vector listFiles(std::string const& searchpattern); // create path if absent, account for eventual concurrent creation void createDirectoriesIfAbsent(std::string const& path); +// A function to expand a string containing shell variables +// to a string in which these vars have been substituted. +// Motivation:: filesystem::exists() does not do this by default +// and I couldn't find information on this. Potentially there is an +// existing solution. +std::string expandShellVarsInFileName(std::string const& input); + } // namespace o2::utils #endif //O2_FILEITERATOR_H diff --git a/Common/Utils/include/CommonUtils/IRFrameSelector.h b/Common/Utils/include/CommonUtils/IRFrameSelector.h index 4509964ccc6ca..a4365030b6a12 100644 --- a/Common/Utils/include/CommonUtils/IRFrameSelector.h +++ b/Common/Utils/include/CommonUtils/IRFrameSelector.h @@ -13,6 +13,9 @@ /// \brief Class to check if give InteractionRecord or IRFrame is selected by the external IRFrame vector /// \author ruben.shahoyan@cern.ch +#ifndef O2_UTILS_IRFRAMESELECTOR_H +#define O2_UTILS_IRFRAMESELECTOR_H + #include "CommonDataFormat/IRFrame.h" #include @@ -21,28 +24,39 @@ namespace o2::utils class IRFrameSelector { public: - long check(const o2::dataformats::IRFrame& fr); - long check(const o2::InteractionRecord& ir) { return check(o2::dataformats::IRFrame{ir, ir}); } + long check(o2::dataformats::IRFrame fr, size_t bwd = 0, size_t fwd = 0); + long check(const o2::InteractionRecord& ir, size_t bwd = 0, size_t fwd = 0) { return check(o2::dataformats::IRFrame{ir, ir}, bwd, fwd); } + gsl::span getMatchingFrames(const o2::dataformats::IRFrame& fr); template - void setSelectedIRFrames(const SPAN& sp) + void setSelectedIRFrames(const SPAN& sp, size_t bwd = 0, size_t fwd = 0, long shift = 0, bool removeOverlaps = true) { mFrames = gsl::span(sp.data(), sp.size()); - mLastBoundID = -1; + mIsSet = true; + applyMargins(bwd, fwd, shift, removeOverlaps); mLastIRFrameChecked.getMin().clear(); // invalidate + mLastBoundID = -1; } + void clear(); size_t loadIRFrames(const std::string& fname); + void applyMargins(size_t bwd, size_t fwd, long shift, bool removeOverlaps = true); void print(bool lst = false) const; auto getIRFrames() const { return mFrames; } + bool isSet() const { return mIsSet; } + + void setOwnList(const std::vector& lst, bool toBeSorted); private: gsl::span mFrames{}; // externally provided span of IRFrames, must be sorted in IRFrame.getMin() o2::dataformats::IRFrame mLastIRFrameChecked{}; // last frame which was checked long mLastBoundID = -1; // id of the last checked entry >= mLastIRFrameChecked + bool mIsSet = false; // flag that something was set (even if empty) std::vector mOwnList; // list loaded from the file ClassDefNV(IRFrameSelector, 1); }; } // namespace o2::utils + +#endif diff --git a/Common/Utils/include/CommonUtils/RngHelper.h b/Common/Utils/include/CommonUtils/RngHelper.h index 162554138ad19..4b0f4169cfad5 100644 --- a/Common/Utils/include/CommonUtils/RngHelper.h +++ b/Common/Utils/include/CommonUtils/RngHelper.h @@ -32,11 +32,11 @@ class RngHelper { public: // sets the state of the currently active ROOT gRandom Instance - // if -1 (or negative) is given ... we will init with a random seed + // if 0 is given ... we will init with a random seed // returns seed set to TRandom - static unsigned int setGRandomSeed(int seed = -1) + static ULong_t setGRandomSeed(ULong_t seed = 0) { - unsigned int s = seed < 0 ? readURandom() : seed; + const auto s = seed == 0 ? readURandom() : seed; gRandom->SetSeed(s); return s; } diff --git a/Common/Utils/include/CommonUtils/RootSerializableKeyValueStore.h b/Common/Utils/include/CommonUtils/RootSerializableKeyValueStore.h index 80545997af159..25205ce05ab40 100644 --- a/Common/Utils/include/CommonUtils/RootSerializableKeyValueStore.h +++ b/Common/Utils/include/CommonUtils/RootSerializableKeyValueStore.h @@ -21,6 +21,7 @@ #include #include #include +#include namespace o2 { @@ -39,14 +40,19 @@ class RootSerializableKeyValueStore /// Structure encapsulating the stored information: raw buffers and attached type information (combination of type_index_hash and TClass information) struct SerializedInfo { SerializedInfo() = default; - SerializedInfo(void* o, int N, char* b, TClass* cl, std::string const& s) : objptr(o), N(N), bufferptr(b), cl(cl), typeinfo_name(s) {} + SerializedInfo(int N, + std::unique_ptr buffer, TClass const* cl, std::string const& s) : N(N), cl(cl), typeinfo_name(s) + { + bufferptr = buffer.get(); + buffer.release(); + } SerializedInfo(SerializedInfo const& other) { // we do a deep copy N = other.N; bufferptr = new char[N]; + std::memcpy(bufferptr, other.bufferptr, sizeof(char) * N); - objptr = nullptr; cl = other.cl; typeinfo_name = other.typeinfo_name; } @@ -56,13 +62,18 @@ class RootSerializableKeyValueStore std::swap(*this, temp); return *this; } + ~SerializedInfo() + { + // we are the owner of this ... so delete it + delete bufferptr; + } - void* objptr = nullptr; //! pointer to existing object in memory + void* objptr = nullptr; //! pointer for "caching" Int_t N = 0; char* bufferptr = nullptr; //[N] pointer to serialized buffer // we use the TClass and/or the type_index_hash for type idendification - TClass* cl = nullptr; + TClass const* cl = nullptr; std::string typeinfo_name; // typeinfo name that can be used to store type if TClass not available (for PODs!) ClassDefNV(SerializedInfo, 1); }; @@ -141,8 +152,15 @@ class RootSerializableKeyValueStore mStore.clear(); } - /// print list of keys and type information - void print() const; + /// print list of keys, values (and optionally type information) + void print(bool includetypeinfo = false) const; + + /// resets store to the store of another object + void copyFrom(RootSerializableKeyValueStore const& other) + { + mStore.clear(); + mStore = other.mStore; + } private: std::map mStore; @@ -153,21 +171,21 @@ class RootSerializableKeyValueStore { // make sure we have a TClass for this // if there is a TClass, we'll use ROOT serialization to encode into the buffer - auto ptr = new T(value); + auto ptr = std::make_unique(T{value}); auto cl = TClass::GetClass(typeid(value)); if (!cl) { state = GetState::kNOTCLASS; return; } - char* bufferptr = nullptr; + std::unique_ptr bufferptr(nullptr); TBufferFile buff(TBuffer::kWrite); - buff.WriteObjectAny(ptr, cl); + buff.WriteObjectAny(ptr.get(), cl); int N = buff.Length(); - bufferptr = new char[N]; - memcpy(bufferptr, buff.Buffer(), sizeof(char) * N); + bufferptr.reset(new char[N]); + memcpy(bufferptr.get(), buff.Buffer(), sizeof(char) * N); auto name = std::type_index(typeid(value)).name(); - mStore.insert(std::pair(key, SerializedInfo((void*)ptr, N, (char*)bufferptr, cl, name))); + mStore.insert(std::pair(key, SerializedInfo(N, std::move(bufferptr), cl, name))); } // implementation for put for trivial types @@ -177,13 +195,13 @@ class RootSerializableKeyValueStore // we forbid pointers static_assert(!std::is_pointer::value); // serialization of trivial types is easy (not based on ROOT) - auto ptr = new T(value); + auto ptr = std::make_unique(T{value}); int N = sizeof(T); - auto bufferptr = new char[N]; - memcpy(bufferptr, (char*)ptr, sizeof(char) * N); + std::unique_ptr bufferptr(new char[N]); + memcpy(bufferptr.get(), (char*)ptr.get(), sizeof(char) * N); auto name = std::type_index(typeid(value)).name(); - mStore.insert(std::pair(key, SerializedInfo((void*)ptr, N, (char*)bufferptr, nullptr, name))); + mStore.insert(std::pair(key, SerializedInfo(N, std::move(bufferptr), nullptr, name))); } // generic implementation for get relying on TClass diff --git a/Common/Utils/include/CommonUtils/ShmManager.h b/Common/Utils/include/CommonUtils/ShmManager.h index 06dba283fec82..e7174b909b82c 100644 --- a/Common/Utils/include/CommonUtils/ShmManager.h +++ b/Common/Utils/include/CommonUtils/ShmManager.h @@ -23,10 +23,15 @@ #include #include +#if !defined(__CINT__) && !defined(__MAKECINT__) && !defined(__ROOTCLING__) && !defined(__CLING__) #include #include +#endif +#if !defined(__CINT__) && !defined(__MAKECINT__) && !defined(__ROOTCLING__) && !defined(__CLING__) && !defined(__APPLE__) +// this shared mem mode is meant for compiled stuff in o2-sim; not for ROOT sessions #define USESHM 1 +#endif namespace o2 { @@ -113,8 +118,10 @@ class ShmManager void* tryAttach(bool& success); size_t getPointerOffset(void* ptr) const { return (size_t)((char*)ptr - (char*)mBufferPtr); } +#if !defined(__CINT__) && !defined(__MAKECINT__) && !defined(__ROOTCLING__) && !defined(__CLING__) boost::interprocess::wmanaged_external_buffer* boostmanagedbuffer; boost::interprocess::allocator* boostallocator; +#endif }; } // namespace utils diff --git a/Common/Utils/include/CommonUtils/StringUtils.h b/Common/Utils/include/CommonUtils/StringUtils.h index 2402f3faa975c..710632fc7dbfe 100644 --- a/Common/Utils/include/CommonUtils/StringUtils.h +++ b/Common/Utils/include/CommonUtils/StringUtils.h @@ -20,8 +20,7 @@ #include #include #include -#include -#include +#include "GPUCommonRtypes.h" namespace o2 { @@ -41,6 +40,13 @@ struct Str { s.erase(s.begin(), std::find_if(s.begin(), s.end(), [](int ch) { return !std::isspace(ch); })); } + static inline void ltrim(std::string& s, const std::string& start) + { + if (beginsWith(s, start)) { + s.erase(0, start.size()); + } + } + /** Trim from end (in place) * * @param s @@ -50,6 +56,13 @@ struct Str { s.erase(std::find_if(s.rbegin(), s.rend(), [](int ch) { return !std::isspace(ch); }).base(), s.end()); } + static inline void rtrim(std::string& s, const std::string& ending) + { + if (endsWith(s, ending)) { + s.erase(s.size() - ending.size(), ending.size()); + } + } + /** * Trim from both ends (in place) * @param s @@ -72,6 +85,13 @@ struct Str { return ss; } + static inline std::string ltrim_copy(const std::string& s, const std::string& start) + { + std::string ss = s; + ltrim(ss, start); + return ss; + } + /** * Trim from end (copying) * @param s @@ -84,6 +104,13 @@ struct Str { return ss; } + static inline std::string rtrim_copy(const std::string& s, const std::string& ending) + { + std::string ss = s; + rtrim(ss, ending); + return ss; + } + /** * Trim from both sides (copying) * @param s @@ -118,6 +145,9 @@ struct Str { return s.str(); } + // replace all occurencies of from by to, return count + static int replaceAll(std::string& s, const std::string& from, const std::string& to); + // generate random string of given length, suitable for file names static std::string getRandomString(int length); diff --git a/Common/Utils/include/CommonUtils/TreeStream.h b/Common/Utils/include/CommonUtils/TreeStream.h index a5adeec810865..d1d4527ffc99d 100644 --- a/Common/Utils/include/CommonUtils/TreeStream.h +++ b/Common/Utils/include/CommonUtils/TreeStream.h @@ -11,7 +11,7 @@ /// @brief Class for creating debug root trees with std::cout like intervace /// @author Marian Ivanov, marian.ivanov@cern.ch (original code in AliRoot) -/// Ruben Shahoyan, ruben.shahoyan@cern.ch (porting to O2) +/// Ruben Shahoyan, ruben.shahoyan@cern.ch (porting to O2). #ifndef ALICEO2_TREESTREAM_H #define ALICEO2_TREESTREAM_H @@ -19,6 +19,9 @@ #include #include #include +#include +#include +#include "GPUCommonDef.h" class TBranch; class TClass; @@ -38,13 +41,82 @@ namespace utils /// /// See testTreeStream.cxx for functional example /// +namespace details +{ +template +struct IsTrivialRootType { + static constexpr bool value = + std::is_same_v || // Float_t + std::is_same_v || // Double_t + std::is_same_v || std::is_same_v || // ULong64_t or ULong_t + std::is_same_v || std::is_same_v || // Long64_t or Long_t + std::is_same_v || // UInt_t + std::is_same_v || // Int_t + std::is_same_v || // UShort_t + std::is_same_v || // Short_t + std::is_same_v || // UChar_t + std::is_same_v || std::is_same_v || std::is_same_v; // Char_t, int8_t, or Bool_t +}; + +template +struct IsTrivialRootType { + static constexpr bool value = IsTrivialRootType::value; +}; + +template +struct IsTrivialRootType { + static constexpr bool value = IsTrivialRootType::value; +}; + +template +concept TrivialRootType = IsTrivialRootType::value; + +template +concept ComplexRootType = !IsTrivialRootType::value; + +template +static constexpr char getRootTypeCode() +{ + if constexpr (std::is_array_v) { + return getRootTypeCode>(); + } else if constexpr (std::is_same_v) { + return 'F'; + } else if constexpr (std::is_same_v) { + return 'D'; + } else if constexpr (std::is_same_v || + std::is_same_v) { + return 'l'; + } else if constexpr (std::is_same_v || + std::is_same_v) { + return 'L'; + } else if constexpr (std::is_same_v) { + return 'i'; + } else if constexpr (std::is_same_v) { + return 'I'; + } else if constexpr (std::is_same_v) { + return 's'; + } else if constexpr (std::is_same_v) { + return 'S'; + } else if constexpr (std::is_same_v) { + return 'b'; + } else if constexpr (std::is_same_v || + std::is_same_v || + std::is_same_v) { + return 'B'; + } else { + static_assert(false, "unsupported type!"); + } +} +} // namespace details + class TreeStream { public: struct TreeDataElement { + int arsize = 1; ///< size of array char type = 0; ///< type of data element const TClass* cls = nullptr; ///< data type pointer - void* ptr = nullptr; ///< pointer to element + const void* ptr = nullptr; ///< pointer to element std::string name; ///< name of the element }; @@ -52,7 +124,7 @@ class TreeStream TreeStream() = default; virtual ~TreeStream() = default; void Close() { mTree.Write(); } - Int_t CheckIn(Char_t type, void* pointer); + Int_t CheckIn(Char_t type, const void* pointer); void BuildTree(); void Fill(); Double_t getSize() { return mTree.GetZipBytes(); } @@ -62,102 +134,32 @@ class TreeStream const char* getName() const { return mTree.GetName(); } void setID(int id) { mID = id; } int getID() const { return mID; } - TreeStream& operator<<(Bool_t& b) - { - CheckIn('B', &b); - return *this; - } - - TreeStream& operator<<(Char_t& c) - { - CheckIn('B', &c); - return *this; - } - - TreeStream& operator<<(UChar_t& c) - { - CheckIn('b', &c); - return *this; - } - - TreeStream& operator<<(Short_t& h) - { - CheckIn('S', &h); - return *this; - } - - TreeStream& operator<<(UShort_t& h) - { - CheckIn('s', &h); - return *this; - } - - TreeStream& operator<<(Int_t& i) - { - CheckIn('I', &i); - return *this; - } - - TreeStream& operator<<(UInt_t& i) - { - CheckIn('i', &i); - return *this; - } - - TreeStream& operator<<(Long_t& l) - { - CheckIn('L', &l); - return *this; - } - TreeStream& operator<<(ULong_t& l) + template + TreeStream& operator<<(const T& t) { - CheckIn('l', &l); - return *this; - } - - TreeStream& operator<<(Long64_t& l) - { - CheckIn('L', &l); - return *this; - } - - TreeStream& operator<<(ULong64_t& l) - { - CheckIn('l', &l); - return *this; - } - - TreeStream& operator<<(Float_t& f) - { - CheckIn('F', &f); - return *this; - } - - TreeStream& operator<<(Double_t& d) - { - CheckIn('D', &d); + CheckIn(details::getRootTypeCode(), &t); return *this; } TreeStream& operator<<(const Char_t* name); template - TreeStream& operator<<(T* obj) + TreeStream& operator<<(const T* obj) { CheckIn(obj); return *this; } - template - TreeStream& operator<<(T& obj) + template ::value, bool>::type* = nullptr> + TreeStream& operator<<(const T& obj) { CheckIn(&obj); return *this; } template - Int_t CheckIn(T* obj); + Int_t CheckIn(const T* obj); private: // @@ -167,6 +169,7 @@ class TreeStream int mCurrentIndex = 0; ///< index of current element int mID = -1; ///< identifier of layout int mNextNameCounter = 0; ///< next name counter + int mNextArraySize = 0; ///< next array size int mStatus = 0; ///< status of the layout TString mNextName; ///< name for next entry @@ -174,7 +177,7 @@ class TreeStream }; template -Int_t TreeStream::CheckIn(T* obj) +Int_t TreeStream::CheckIn(const T* obj) { // check in arbitrary class having dictionary TClass* pClass = nullptr; @@ -183,8 +186,7 @@ Int_t TreeStream::CheckIn(T* obj) } if (mCurrentIndex >= static_cast(mElements.size())) { - mElements.emplace_back(); - auto& element = mElements.back(); + auto& element = mElements.emplace_back(); element.cls = pClass; TString name = mNextName; if (name.Length()) { @@ -196,6 +198,8 @@ Int_t TreeStream::CheckIn(T* obj) } element.name = name.Data(); element.ptr = obj; + element.arsize = mNextArraySize; + mNextArraySize = 1; // reset } else { auto& element = mElements[mCurrentIndex]; if (!element.cls) { diff --git a/Common/Utils/include/CommonUtils/TreeStreamRedirector.h b/Common/Utils/include/CommonUtils/TreeStreamRedirector.h index 8199009df400d..80858fecea87b 100644 --- a/Common/Utils/include/CommonUtils/TreeStreamRedirector.h +++ b/Common/Utils/include/CommonUtils/TreeStreamRedirector.h @@ -41,7 +41,7 @@ namespace utils class TreeStreamRedirector { public: - TreeStreamRedirector(const char* fname = "", const char* option = "update"); + TreeStreamRedirector(const char* fname = "", const char* option = "recreate"); virtual ~TreeStreamRedirector(); void Close(); TFile* GetFile() { return mDirectory->GetFile(); } diff --git a/Common/Utils/src/BoostHistogramUtils.cxx b/Common/Utils/src/BoostHistogramUtils.cxx index c5c9d97742976..279a709b97586 100644 --- a/Common/Utils/src/BoostHistogramUtils.cxx +++ b/Common/Utils/src/BoostHistogramUtils.cxx @@ -40,48 +40,6 @@ std::string createErrorMessageFitGaus(o2::utils::FitGausError_t errorcode) return "Gaus fit failed! Unknown error code"; } -boostHisto1d_VarAxis boosthistoFromRoot_1D(TH1D* inHist1D) -{ - // first setup the proper boost histogram - int nBins = inHist1D->GetNbinsX(); - std::vector binEdges; - for (int i = 0; i < nBins + 1; i++) { - binEdges.push_back(inHist1D->GetBinLowEdge(i + 1)); - } - boostHisto1d_VarAxis mHisto = boost::histogram::make_histogram(boost::histogram::axis::variable<>(binEdges)); - - // trasfer the acutal values - for (Int_t x = 1; x < nBins + 1; x++) { - mHisto.at(x - 1) = inHist1D->GetBinContent(x); - } - return mHisto; -} - -boostHisto2d_VarAxis boostHistoFromRoot_2D(TH2D* inHist2D) -{ - // Get Xaxis binning - const int nBinsX = inHist2D->GetNbinsX(); - std::vector binEdgesX; - for (int i = 0; i < nBinsX + 1; i++) { - binEdgesX.push_back(inHist2D->GetXaxis()->GetBinLowEdge(i + 1)); - } - // Get Yaxis binning - const int nBinsY = inHist2D->GetNbinsY(); - std::vector binEdgesY; - for (int i = 0; i < nBinsY + 1; i++) { - binEdgesY.push_back(inHist2D->GetYaxis()->GetBinLowEdge(i + 1)); - } - - boostHisto2d_VarAxis mHisto = boost::histogram::make_histogram(boost::histogram::axis::variable<>(binEdgesX), boost::histogram::axis::variable<>(binEdgesY)); - - // trasfer the acutal values - for (Int_t x = 1; x < nBinsX + 1; x++) { - for (Int_t y = 1; y < nBinsY + 1; y++) { - mHisto.at(x - 1, y - 1) = inHist2D->GetBinContent(x, y); - } - } - return mHisto; -} /// \brief Printing an error message when then fit returns an invalid result /// \param errorcode Error of the type FitGausError_t, thrown when fit result is invalid. std::string createErrorMessage(o2::utils::FitGausError_t errorcode) diff --git a/Common/Utils/src/CommonUtilsTestLinkDef.h b/Common/Utils/src/CommonUtilsTestLinkDef.h new file mode 100644 index 0000000000000..9ee67f62fd7d0 --- /dev/null +++ b/Common/Utils/src/CommonUtilsTestLinkDef.h @@ -0,0 +1,21 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifdef __CLING__ + +#pragma link off all globals; +#pragma link off all classes; +#pragma link off all functions; + +#pragma link C++ class o2::conf::test::TestParam + ; +#pragma link C++ class o2::conf::ConfigurableParamHelper < o2::conf::test::TestParam> + ; + +#endif diff --git a/Common/Utils/src/ConfigurableParam.cxx b/Common/Utils/src/ConfigurableParam.cxx index 6fdb8611ae7a3..8497a485fca39 100644 --- a/Common/Utils/src/ConfigurableParam.cxx +++ b/Common/Utils/src/ConfigurableParam.cxx @@ -9,11 +9,13 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. -//first version 8/2018, Sandro Wenzel +// first version 8/2018, Sandro Wenzel #include "CommonUtils/ConfigurableParam.h" #include "CommonUtils/StringUtils.h" #include "CommonUtils/KeyValParam.h" +#include "CommonUtils/ConfigurableParamReaders.h" +#define BOOST_BIND_GLOBAL_PLACEHOLDERS #include #include #include @@ -29,7 +31,7 @@ #include #include #include -#include +#include #include #include "TDataMember.h" #include "TDataType.h" @@ -46,7 +48,6 @@ std::vector* ConfigurableParam::sRegisteredParamClasses = nu boost::property_tree::ptree* ConfigurableParam::sPtree = nullptr; std::map>* ConfigurableParam::sKeyToStorageMap = nullptr; std::map* ConfigurableParam::sValueProvenanceMap = nullptr; -std::string ConfigurableParam::sInputDir = ""; std::string ConfigurableParam::sOutputDir = ""; EnumRegistry* ConfigurableParam::sEnumRegistry = nullptr; @@ -76,6 +77,30 @@ bool keyInTree(boost::property_tree::ptree* pt, const std::string& key) return reply; } +// Convert a type info to the appropiate literal suffix +std::string getLiteralSuffixFromType(const std::type_info& type) +{ + if (type == typeid(float)) { + return "f"; + } + if (type == typeid(long double)) { + return "l"; + } + if (type == typeid(unsigned int)) { + return "u"; + } + if (type == typeid(unsigned long)) { + return "ul"; + } + if (type == typeid(long long)) { + return "ll"; + } + if (type == typeid(unsigned long long)) { + return "ull"; + } + return ""; +} + // ------------------------------------------------------------------ void EnumRegistry::add(const std::string& key, const TDataMember* dm) @@ -121,7 +146,6 @@ std::string EnumRegistry::toString() const out.append("\n"); } - LOG(info) << out; return out; } @@ -193,60 +217,58 @@ void ConfigurableParam::writeINI(std::string const& filename, std::string const& bool ConfigurableParam::configFileExists(std::string const& filepath) { - return std::filesystem::exists(o2::utils::Str::concat_string(sInputDir, filepath)); -} - -// ------------------------------------------------------------------ - -boost::property_tree::ptree ConfigurableParam::readConfigFile(std::string const& filepath) -{ - auto inpfilename = o2::utils::Str::concat_string(sInputDir, filepath); - if (!std::filesystem::exists(inpfilename)) { - LOG(fatal) << inpfilename << " : config file does not exist!"; - } - - boost::property_tree::ptree pt; - - if (boost::iends_with(inpfilename, ".ini")) { - pt = readINI(inpfilename); - } else if (boost::iends_with(inpfilename, ".json")) { - pt = readJSON(inpfilename); - } else { - LOG(fatal) << "Configuration file must have either .ini or .json extension"; - } - - return pt; + return std::filesystem::exists(o2::utils::Str::concat_string(ConfigurableParamReaders::getInputDir(), filepath)); } // ------------------------------------------------------------------ -boost::property_tree::ptree ConfigurableParam::readINI(std::string const& filepath) +void ConfigurableParam::setValue(std::string const& key, std::string const& valuestring) { - boost::property_tree::ptree pt; - try { - boost::property_tree::read_ini(filepath, pt); - } catch (const boost::property_tree::ptree_error& e) { - LOG(fatal) << "Failed to read INI config file " << filepath << " (" << e.what() << ")"; - } catch (...) { - LOG(fatal) << "Unknown error when reading INI config file "; + if (!sIsFullyInitialized) { + initialize(); } - - return pt; -} - -// ------------------------------------------------------------------ - -boost::property_tree::ptree ConfigurableParam::readJSON(std::string const& filepath) -{ - boost::property_tree::ptree pt; - + assert(sPtree); + auto setValueImpl = [&](std::string const& value) { + sPtree->put(key, value); + auto changed = updateThroughStorageMapWithConversion(key, value); + if (changed != EParamUpdateStatus::Failed) { + sValueProvenanceMap->find(key)->second = kRT; // set to runtime + } + }; try { - boost::property_tree::read_json(filepath, pt); - } catch (const boost::property_tree::ptree_error& e) { - LOG(fatal) << "Failed to read JSON config file " << filepath << " (" << e.what() << ")"; + if (sPtree->get_optional(key).is_initialized()) { + try { + // try first setting value without stripping a literal suffix + setValueImpl(valuestring); + } catch (...) { + // try second stripping the expected literal suffix value for fundamental types + auto iter = sKeyToStorageMap->find(key); + if (iter == sKeyToStorageMap->end()) { + std::cerr << "Error in setValue (string) key is not known\n"; + return; + } + const auto expectedSuffix = getLiteralSuffixFromType(iter->second.first); + if (!expectedSuffix.empty()) { + auto valuestringLower = valuestring; + std::transform(valuestring.cbegin(), valuestring.cend(), valuestringLower.begin(), tolower); + if (valuestringLower.ends_with(expectedSuffix)) { + std::string strippedValue = valuestringLower.substr(0, valuestringLower.length() - expectedSuffix.length()); + setValueImpl(strippedValue); + } else { + // check if it has a different suffix and throw + for (const auto& suffix : {"f", "l", "u", "ul", "ll", "ull"}) { + if (valuestringLower.ends_with(suffix) && suffix != expectedSuffix) { + throw std::invalid_argument("Wrong type suffix: expected " + expectedSuffix + " but got " + suffix); + } + } + throw; // just rethrow the original exception + } + } + } + } + } catch (std::exception const& e) { + std::cerr << "Error in setValue (string) " << e.what() << "\n"; } - - return pt; } // ------------------------------------------------------------------ @@ -257,7 +279,7 @@ void ConfigurableParam::writeJSON(std::string const& filename, std::string const LOG(info) << "ignoring writing of json file " << filename; return; } - initPropertyTree(); // update the boost tree before writing + initPropertyTree(); // update the boost tree before writing auto outfilename = o2::utils::Str::concat_string(sOutputDir, filename); if (!keyOnly.empty()) { // write ini for selected key only try { @@ -284,14 +306,14 @@ void ConfigurableParam::initPropertyTree() // ------------------------------------------------------------------ -void ConfigurableParam::printAllKeyValuePairs() +void ConfigurableParam::printAllKeyValuePairs(bool useLogger) { if (!sIsFullyInitialized) { initialize(); } std::cout << "####\n"; for (auto p : *sRegisteredParamClasses) { - p->printKeyValues(true); + p->printKeyValues(true, useLogger); } std::cout << "----\n"; } @@ -408,7 +430,7 @@ void ConfigurableParam::updateFromFile(std::string const& configFile, std::strin return; } - boost::property_tree::ptree pt = readConfigFile(cfgfile); + boost::property_tree::ptree pt = ConfigurableParamReaders::readConfigFile(cfgfile); std::vector> keyValPairs; auto request = o2::utils::Str::tokenize(paramsList, ',', true); @@ -479,14 +501,12 @@ void ConfigurableParam::updateFromString(std::string const& configString) std::vector> pairs; for (auto& token : tokens) { - auto keyval = o2::utils::Str::tokenize(token, '='); - if (keyval.size() != 2) { + auto s = token.find('='); + if (s == 0 || s == std::string::npos || s == token.size() - 1) { LOG(fatal) << "Illegal command-line key/value string: " << token; continue; } - - std::pair pair = std::make_pair(keyval[0], o2::utils::Str::trim_copy(keyval[1])); - pairs.push_back(pair); + pairs.emplace_back(token.substr(0, s), token.substr(s + 1, token.size())); } return pairs; @@ -521,7 +541,7 @@ void ConfigurableParam::updateFromString(std::string const& configString) const auto& kv = o2::conf::KeyValParam::Instance(); if (getProvenance("keyval.input_dir") != kCODE) { - sInputDir = o2::utils::Str::concat_string(o2::utils::Str::rectifyDirectory(kv.input_dir)); + ConfigurableParamReaders::setInputDir(o2::utils::Str::concat_string(o2::utils::Str::rectifyDirectory(kv.input_dir))); } if (getProvenance("keyval.output_dir") != kCODE) { if (kv.output_dir == "/dev/null") { @@ -540,6 +560,8 @@ void ConfigurableParam::setValues(std::vector 0 && (el.at(0) == '[') && (el.at(el.size() - 1) == ']'); }; + bool nonFatal = getenv("ALICEO2_CONFIGURABLEPARAM_WRONGKEYISNONFATAL") != nullptr; + // Take a vector of param key/value pairs // and update the storage map for each of them by calling setValue. // 1. For string/scalar types this is simple. @@ -552,6 +574,10 @@ void ConfigurableParam::setValues(std::vector #include #include -#include "FairLogger.h" +#include #include +#include #include +#include #ifdef NDEBUG #undef NDEBUG #endif @@ -34,25 +36,32 @@ using namespace o2::conf; // ---------------------------------------------------------------------- -std::string ParamDataMember::toString(std::string const& prefix, bool showProv) const +std::string ParamDataMember::toString(std::string const& prefix, bool showProv, size_t padding) const { - std::string nil = ""; - + const std::string label = prefix + "." + name + " : " + value; std::ostringstream out; - out << prefix << "." << name << " : " << value; + out << label; if (showProv) { - std::string prov = (provenance.compare("") == 0 ? nil : provenance); - out << "\t\t[ " + prov + " ]"; + std::string prov = (provenance.compare("") == 0 ? "" : provenance); + if (padding) { + size_t len = label.size() - prefix.size() - 5; // 4 four the extra chars + 1 for the maxpad + if (len < padding) { + out << std::string(padding - len, ' '); + } else { + out << ' '; + } + out << "[ " + prov + " ]"; + } else { + out << "\t\t[ " + prov + " ]"; + } } - - out << "\n"; return out.str(); } std::ostream& operator<<(std::ostream& out, const ParamDataMember& pdm) { - out << pdm.toString("", false); + out << pdm.toString("", false) << "\n"; return out; } @@ -183,19 +192,19 @@ std::string asString(TDataMember const& dm, char* pointer) // potentially other cases to be added here LOG(error) << "COULD NOT REPRESENT AS STRING"; - return nullptr; + return std::string(); } // ---------------------------------------------------------------------- std::vector* _ParamHelper::getDataMembersImpl(std::string const& mainkey, TClass* cl, void* obj, - std::map const* provmap) + std::map const* provmap, size_t globaloffset) { std::vector* members = new std::vector; - auto toDataMember = [&members, obj, mainkey, provmap](const TDataMember* dm, int index, int size) { + auto toDataMember = [&members, obj, mainkey, provmap, globaloffset](const TDataMember* dm, int index, int size) { auto TS = getSizeOfUnderlyingType(*dm); - char* pointer = ((char*)obj) + dm->GetOffset() + index * TS; + char* pointer = ((char*)obj) + dm->GetOffset() + index * TS + globaloffset; const std::string name = getName(dm, index, size); auto value = asString(*dm, pointer); @@ -281,14 +290,14 @@ std::type_info const& nameToTypeInfo(const char* tname, TDataType const* dt) void _ParamHelper::fillKeyValuesImpl(std::string const& mainkey, TClass* cl, void* obj, boost::property_tree::ptree* tree, std::map>* keytostoragemap, - EnumRegistry* enumRegistry) + EnumRegistry* enumRegistry, size_t globaloffset) { boost::property_tree::ptree localtree; - auto fillMap = [obj, &mainkey, &localtree, &keytostoragemap, &enumRegistry](const TDataMember* dm, int index, int size) { + auto fillMap = [obj, &mainkey, &localtree, &keytostoragemap, &enumRegistry, globaloffset](const TDataMember* dm, int index, int size) { const auto name = getName(dm, index, size); auto dt = dm->GetDataType(); auto TS = getSizeOfUnderlyingType(*dm); - char* pointer = ((char*)obj) + dm->GetOffset() + index * TS; + char* pointer = ((char*)obj) + dm->GetOffset() + index * TS + globaloffset; localtree.put(name, asString(*dm, pointer)); auto key = mainkey + "." + name; @@ -309,20 +318,52 @@ void _ParamHelper::fillKeyValuesImpl(std::string const& mainkey, TClass* cl, voi // ---------------------------------------------------------------------- -void _ParamHelper::printMembersImpl(std::string const& mainkey, std::vector const* members, bool showProv) +void _ParamHelper::printMembersImpl(std::string const& mainkey, std::vector const* members, bool showProv, bool useLogger, bool withPadding, bool showHash) { - _ParamHelper::outputMembersImpl(std::cout, mainkey, members, showProv); + + _ParamHelper::outputMembersImpl(std::cout, mainkey, members, showProv, useLogger, withPadding, showHash); } -void _ParamHelper::outputMembersImpl(std::ostream& out, std::string const& mainkey, std::vector const* members, bool showProv) +void _ParamHelper::outputMembersImpl(std::ostream& out, std::string const& mainkey, std::vector const* members, bool showProv, bool useLogger, bool withPadding, bool showHash) { if (members == nullptr) { return; } + size_t maxpad{0}; + if (withPadding) { + for (auto& member : *members) { + maxpad = std::max(maxpad, member.name.size() + member.value.size()); + } + } + + if (showHash) { + std::string shash = std::format("{:07x}", getHashImpl(mainkey, members)); + shash = shash.substr(0, 7); + if (useLogger) { + LOG(info) << mainkey << " [Hash#" << shash << "]"; + } else { + out << mainkey << " [Hash#" << shash << "]\n"; + } + } + + for (auto& member : *members) { + if (useLogger) { + LOG(info) << member.toString(mainkey, showProv, maxpad); + } else { + out << member.toString(mainkey, showProv, maxpad) << "\n"; + } + } +} + +size_t _ParamHelper::getHashImpl(std::string const& mainkey, std::vector const* members) +{ + size_t hash = 0; + boost::hash_combine(hash, mainkey); for (auto& member : *members) { - out << member.toString(mainkey, showProv); + boost::hash_combine(hash, member.value); } + return hash; } // ---------------------------------------------------------------------- @@ -341,14 +382,14 @@ bool isMemblockDifferent(char const* block1, char const* block2, int sizeinbytes // ---------------------------------------------------------------------- void _ParamHelper::assignmentImpl(std::string const& mainkey, TClass* cl, void* to, void* from, - std::map* provmap) + std::map* provmap, size_t globaloffset) { - auto assignifchanged = [to, from, &mainkey, provmap](const TDataMember* dm, int index, int size) { + auto assignifchanged = [to, from, &mainkey, provmap, globaloffset](const TDataMember* dm, int index, int size) { const auto name = getName(dm, index, size); auto dt = dm->GetDataType(); auto TS = getSizeOfUnderlyingType(*dm); - char* pointerto = ((char*)to) + dm->GetOffset() + index * TS; - char* pointerfrom = ((char*)from) + dm->GetOffset() + index * TS; + char* pointerto = ((char*)to) + dm->GetOffset() + index * TS + globaloffset; + char* pointerfrom = ((char*)from) + dm->GetOffset() + index * TS + globaloffset; // lambda to update the provenance auto updateProv = [&mainkey, name, provmap]() { @@ -388,14 +429,14 @@ void _ParamHelper::assignmentImpl(std::string const& mainkey, TClass* cl, void* // ---------------------------------------------------------------------- void _ParamHelper::syncCCDBandRegistry(const std::string& mainkey, TClass* cl, void* to, void* from, - std::map* provmap) + std::map* provmap, size_t globaloffset) { - auto sync = [to, from, &mainkey, provmap](const TDataMember* dm, int index, int size) { + auto sync = [to, from, &mainkey, provmap, globaloffset](const TDataMember* dm, int index, int size) { const auto name = getName(dm, index, size); auto dt = dm->GetDataType(); auto TS = getSizeOfUnderlyingType(*dm); - char* pointerto = ((char*)to) + dm->GetOffset() + index * TS; - char* pointerfrom = ((char*)from) + dm->GetOffset() + index * TS; + char* pointerto = ((char*)to) + dm->GetOffset() + index * TS + globaloffset; + char* pointerfrom = ((char*)from) + dm->GetOffset() + index * TS + globaloffset; // check current provenance auto key = mainkey + "." + name; diff --git a/Common/Utils/src/ConfigurableParamReaders.cxx b/Common/Utils/src/ConfigurableParamReaders.cxx new file mode 100644 index 0000000000000..2f36f8b27f5fa --- /dev/null +++ b/Common/Utils/src/ConfigurableParamReaders.cxx @@ -0,0 +1,76 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include "CommonUtils/ConfigurableParamReaders.h" +#include "CommonUtils/StringUtils.h" +#include +#include + +#include +#include +#include + +namespace o2::conf +{ +// ------------------------------------------------------------------ + +boost::property_tree::ptree ConfigurableParamReaders::readINI(std::string const& filepath) +{ + boost::property_tree::ptree pt; + try { + boost::property_tree::read_ini(filepath, pt); + } catch (const boost::property_tree::ptree_error& e) { + LOG(fatal) << "Failed to read INI config file " << filepath << " (" << e.what() << ")"; + } catch (...) { + LOG(fatal) << "Unknown error when reading INI config file "; + } + + return pt; +} + +// ------------------------------------------------------------------ + +boost::property_tree::ptree ConfigurableParamReaders::readJSON(std::string const& filepath) +{ + boost::property_tree::ptree pt; + + try { + boost::property_tree::read_json(filepath, pt); + } catch (const boost::property_tree::ptree_error& e) { + LOG(fatal) << "Failed to read JSON config file " << filepath << " (" << e.what() << ")"; + } + + return pt; +} + +boost::property_tree::ptree ConfigurableParamReaders::readConfigFile(std::string const& filepath) +{ + auto inpfilename = o2::utils::Str::concat_string(sInputDir, filepath); + if (!std::filesystem::exists(inpfilename)) { + LOG(fatal) << inpfilename << " : config file does not exist!"; + } + + boost::property_tree::ptree pt; + + if (boost::iends_with(inpfilename, ".ini")) { + pt = ConfigurableParamReaders::readINI(inpfilename); + } else if (boost::iends_with(inpfilename, ".json")) { + pt = ConfigurableParamReaders::readJSON(inpfilename); + } else { + LOG(fatal) << "Configuration file must have either .ini or .json extension"; + } + + return pt; +} + +std::string ConfigurableParamReaders::sInputDir = ""; + +} // namespace o2::conf diff --git a/Common/Utils/src/ConfigurableParamTest.cxx b/Common/Utils/src/ConfigurableParamTest.cxx new file mode 100644 index 0000000000000..5115a8dfe889d --- /dev/null +++ b/Common/Utils/src/ConfigurableParamTest.cxx @@ -0,0 +1,13 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include "CommonUtils/ConfigurableParamTest.h" +O2ParamImpl(o2::conf::test::TestParam); diff --git a/Common/Utils/src/DLLoaderBase.cxx b/Common/Utils/src/DLLoaderBase.cxx new file mode 100644 index 0000000000000..1e4ab97604794 --- /dev/null +++ b/Common/Utils/src/DLLoaderBase.cxx @@ -0,0 +1,12 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include "CommonUtils/DLLoaderBase.h" diff --git a/Common/Utils/src/DebugStreamer.cxx b/Common/Utils/src/DebugStreamer.cxx index 3ff0082144e4e..46ff9be83d415 100644 --- a/Common/Utils/src/DebugStreamer.cxx +++ b/Common/Utils/src/DebugStreamer.cxx @@ -15,38 +15,159 @@ #include #include "TROOT.h" #include "TKey.h" +#include +#include "Framework/Logger.h" #endif O2ParamImpl(o2::utils::ParameterDebugStreamer); #if !defined(GPUCA_GPUCODE) && !defined(GPUCA_STANDALONE) && defined(DEBUG_STREAMER) -void o2::utils::DebugStreamer::setStreamer(const char* outFile, const char* option) +o2::utils::DebugStreamer::DebugStreamer() { - if (!isStreamerSet()) { - ROOT::EnableThreadSafety(); - mTreeStreamer = std::make_unique(fmt::format("{}_{}.root", outFile, getCPUID()).data(), option); + ROOT::EnableThreadSafety(); +} + +void o2::utils::DebugStreamer::setStreamer(const char* outFile, const char* option, const size_t id) +{ + if (!isStreamerSet(id)) { + mTreeStreamer[id] = std::make_unique(fmt::format("{}_{}.root", outFile, id).data(), option); + } +} + +o2::utils::TreeStreamRedirector& o2::utils::DebugStreamer::getStreamer(const char* outFile, const char* option, const size_t id) +{ + setStreamer(outFile, option, id); + return getStreamer(id); +} + +void o2::utils::DebugStreamer::flush(const size_t id) +{ + if (isStreamerSet(id)) { + mTreeStreamer[id].reset(); + } +} + +void o2::utils::DebugStreamer::flush() +{ + for (const auto& pair : mTreeStreamer) { + flush(pair.first); + } +} + +bool o2::utils::DebugStreamer::checkStream(const StreamFlags streamFlag, const size_t samplingID, const float weight) +{ + const bool isStreamerSet = ((getStreamFlags() & streamFlag) == streamFlag); + if (!isStreamerSet) { + return false; } + + // check sampling frequency + const auto sampling = getSamplingTypeFrequency(streamFlag); + if (sampling.first != SamplingTypes::sampleAll) { + auto sampleTrack = [&]() { + if (samplingID == -1) { + LOGP(fatal, "Sampling type sampleID not supported for stream flag {}", (int)streamFlag); + } + // sample on samplingID (e.g. track level) + static thread_local std::unordered_map> idMap; + // in case of first call samplingID in idMap is 0 and always false and first ID rejected + if (idMap[streamFlag].first != samplingID) { + idMap[streamFlag] = std::pair{samplingID, (getRandom() < sampling.second)}; + } + return idMap[streamFlag].second; + }; + + if (sampling.first == SamplingTypes::sampleRandom) { + // just sample randomly + return (getRandom() < sampling.second); + } else if (sampling.first == SamplingTypes::sampleID) { + return sampleTrack(); + } else if (sampling.first == SamplingTypes::sampleIDGlobal) { + // this contains for each flag the processed track IDs and stores if it was processed or not + static tbb::concurrent_unordered_map> refIDs; + const int index = ParameterDebugStreamer::Instance().sampleIDGlobal[getIndex(streamFlag)]; + + // check if refIDs contains track ID + auto it = refIDs[index].find(samplingID); + if (it != refIDs[index].end()) { + // in case it is present get stored decission + return it->second; + } else { + // in case it is not present sample random decission + const bool storeTrk = sampleTrack(); + refIDs[index][samplingID] = storeTrk; + return storeTrk; + } + } else if (sampling.first == SamplingTypes::sampleWeights) { + // sample with weight + return (weight * getRandom() < sampling.second); + } + } + return true; +} + +float o2::utils::DebugStreamer::getRandom(float min, float max) +{ + // init random number generator for each thread + static thread_local std::mt19937 generator(std::random_device{}()); + std::uniform_real_distribution<> distr(min, max); + const float rnd = distr(generator); + return rnd; +} + +int o2::utils::DebugStreamer::getIndex(const StreamFlags streamFlag) +{ + // see: https://stackoverflow.com/a/71539401 + uint32_t v = streamFlag; + v -= 1; + v = v - ((v >> 1) & 0x55555555); + v = (v & 0x33333333) + ((v >> 2) & 0x33333333); + const uint32_t ind = (((v + (v >> 4) & 0xF0F0F0F) * 0x1010101) >> 24); + return ind; } -std::string o2::utils::DebugStreamer::getUniqueTreeName(const char* tree) const { return fmt::format("{}_{}", tree, getNTrees()); } +std::pair o2::utils::DebugStreamer::getSamplingTypeFrequency(const StreamFlags streamFlag) +{ + const int ind = getIndex(streamFlag); + return std::pair{ParameterDebugStreamer::Instance().samplingType[ind], ParameterDebugStreamer::Instance().samplingFrequency[ind]}; +} + +std::string o2::utils::DebugStreamer::getUniqueTreeName(const char* tree, const size_t id) const { return fmt::format("{}_{}", tree, getNTrees(id)); } size_t o2::utils::DebugStreamer::getCPUID() { return std::hash{}(std::this_thread::get_id()); } -int o2::utils::DebugStreamer::getNTrees() const { return mTreeStreamer->GetFile()->GetListOfKeys()->GetEntries(); } +o2::utils::TreeStreamRedirector* o2::utils::DebugStreamer::getStreamerPtr(const size_t id) const +{ + auto it = mTreeStreamer.find(id); + if (it != mTreeStreamer.end()) { + return (it->second).get(); + } else { + return nullptr; + } +} + +int o2::utils::DebugStreamer::getNTrees(const size_t id) const { return isStreamerSet(id) ? getStreamerPtr(id)->GetFile()->GetListOfKeys()->GetEntries() : -1; } void o2::utils::DebugStreamer::mergeTrees(const char* inpFile, const char* outFile, const char* option) { TFile fInp(inpFile, "READ"); - TList list; + std::unordered_map lists; for (TObject* keyAsObj : *fInp.GetListOfKeys()) { const auto key = dynamic_cast(keyAsObj); - list.Add((TTree*)fInp.Get(key->GetName())); + TTree* tree = (TTree*)fInp.Get(key->GetName()); + // perform simple check on the number of entries to merge only TTree with same content (ToDo: Do check on name of branches) + const int entries = tree->GetListOfBranches()->GetEntries(); + const std::string brName = key->GetName(); + const std::string nameBr = brName.substr(0, brName.find_last_of("_")); + lists[nameBr].Add(tree); } TFile fOut(outFile, "RECREATE"); - auto tree = TTree::MergeTrees(&list, option); - fOut.WriteObject(tree, "tree"); + for (auto& list : lists) { + auto tree = TTree::MergeTrees(&list.second, option); + fOut.WriteObject(tree, list.first.data()); + } } void o2::utils::DebugStreamer::enableStream(const StreamFlags streamFlag) diff --git a/Common/Utils/src/FileFetcher.cxx b/Common/Utils/src/FileFetcher.cxx index d852a97b76298..32c51ac704d4b 100644 --- a/Common/Utils/src/FileFetcher.cxx +++ b/Common/Utils/src/FileFetcher.cxx @@ -84,10 +84,17 @@ void FileFetcher::processInput(const std::vector& input) { for (auto inp : input) { o2::utils::Str::trim(inp); - if (fs::is_directory(inp)) { processDirectory(inp); } else if (mSelRegex && !std::regex_match(inp, *mSelRegex.get())) { // provided selector does not match, treat as a txt file with list + // Avoid reading a multigiB data file as a list of inputs + // bringing down the system. + std::filesystem::path p(inp); + if (std::filesystem::file_size(p) > 10000000) { + LOGP(error, "file list {} larger than 10MB. Is this a data file?", inp); + continue; + } + std::ifstream listFile(inp); if (!listFile.good()) { LOGP(error, "file {} pretends to be a list of inputs but does not exist", inp); @@ -97,7 +104,7 @@ void FileFetcher::processInput(const std::vector& input) std::vector newInput; while (getline(listFile, line)) { o2::utils::Str::trim(line); - if (line[0] == '#') { // ignore commented file + if (line[0] == '#' || line.empty()) { // ignore commented file or empty line continue; } newInput.push_back(line); @@ -215,6 +222,9 @@ void FileFetcher::stop() if (mFetcherThread.joinable()) { mFetcherThread.join(); } + if (mFailure) { + LOGP(fatal, "too many failures in file fetching: {} in {} attempts for {} files in {} loops, abort", mNFilesProc - mNFilesProcOK, mNFilesProc, getNFiles(), mNLoops); + } } //____________________________________________________________ @@ -282,6 +292,19 @@ void FileFetcher::fetcher() fileRef.copied = true; mQueue.push(fileEntry); mNFilesProcOK++; + } else { + if (mFailThreshold < 0.f) { // cut on abs number of failures + if (mNFilesProc - mNFilesProcOK > -mNFilesProcOK) { + mFailure = true; + } + } else if (mFailThreshold > 0.f) { + float fracFail = mNLoops ? (mNFilesProc - mNFilesProcOK) / float(mNFilesProc) : (mNFilesProc - mNFilesProcOK) / float(getNFiles()); + mFailure = fracFail > mFailThreshold; + } + if (mFailure) { + mRunning = false; + break; + } } } } @@ -303,19 +326,46 @@ void FileFetcher::discardFile(const std::string& fname) bool FileFetcher::copyFile(size_t id) { // copy remote file to local setCopyDirName. Adaptation for Gvozden's code from SubTimeFrameFileSource::DataFetcherThread() + bool aliencpMode = false; + std::string uuid{}; + std::vector logsToClean; + std::string dbgset{}; if (mCopyCmd.find("alien") != std::string::npos) { if (!gGrid && !TGrid::Connect("alien://")) { LOG(error) << "Copy command refers to alien but connection to Grid failed"; } + uuid = mInputFiles[id].getOrigName(); + for (auto& c : uuid) { + if (!std::isalnum(c) && c != '-') { + c = '_'; + } + } + if (!(getenv("ALIENPY_DEBUG") && std::stoi(getenv("ALIENPY_DEBUG")) == 1)) { + logsToClean.push_back(fmt::format("log_alienpy_{}.txt", uuid)); + dbgset += fmt::format("ALIENPY_DEBUG=1 ALIENPY_DEBUG_FILE={} ", logsToClean.back()); + } + if (!(getenv("XRD_LOGLEVEL") && strcmp(getenv("XRD_LOGLEVEL"), "Dump") == 0)) { + logsToClean.push_back(fmt::format("log_xrd_{}.txt", uuid)); + dbgset += fmt::format("XRD_LOGLEVEL=Dump XRD_LOGFILE={} ", logsToClean.back()); + } + LOGP(debug, "debug setting for for {}: {}", mInputFiles[id].getOrigName(), dbgset); } - auto realCmd = std::regex_replace(std::regex_replace(mCopyCmd, std::regex("\\?src"), mInputFiles[id].getOrigName()), std::regex("\\?dst"), mInputFiles[id].getLocalName()); - auto fullCmd = fmt::format("sh -c \"{}\" >> {} 2>&1", realCmd, mCopyCmdLogFile); + auto realCmd = std::regex_replace(std::regex_replace(mCopyCmd, std::regex(R"(\?src)"), mInputFiles[id].getOrigName()), std::regex(R"(\?dst)"), mInputFiles[id].getLocalName()); + auto fullCmd = fmt::format(R"(sh -c "{}{}" >> {} 2>&1)", dbgset, realCmd, mCopyCmdLogFile); LOG(info) << "Executing " << fullCmd; const auto sysRet = gSystem->Exec(fullCmd.c_str()); if (sysRet != 0) { LOGP(warning, "FileFetcher: non-zero exit code {} for cmd={}", sysRet, realCmd); + std::string logCmd = fmt::format(R"(sh -c "cp {} log_aliencp_{}.txt")", mCopyCmdLogFile, uuid); + gSystem->Exec(logCmd.c_str()); + } else { // on success cleanup debug log files + for (const auto& log : logsToClean) { + if (fs::exists(log)) { + fs::remove(log); + } + } } - if (!fs::is_regular_file(mInputFiles[id].getLocalName()) || fs::is_empty(mInputFiles[id].getLocalName())) { + if (!fs::is_regular_file(mInputFiles[id].getLocalName()) || fs::is_empty(mInputFiles[id].getLocalName()) || sysRet != 0) { LOGP(alarm, "FileFetcher: failed for copy command {}", realCmd); return false; } diff --git a/Common/Utils/src/FileSystemUtils.cxx b/Common/Utils/src/FileSystemUtils.cxx index 7c69d2b8c25af..44c0b06b4b03a 100644 --- a/Common/Utils/src/FileSystemUtils.cxx +++ b/Common/Utils/src/FileSystemUtils.cxx @@ -61,5 +61,45 @@ void createDirectoriesIfAbsent(std::string const& path) throw std::runtime_error(fmt::format("Failed to create {} directory", path)); } } +// A function to expand string containing shell variables +// to a string in which these vars have been substituted. +// Motivation:: filesystem::exists() does not do this by default +// and I couldn't find information on this. Potentially there is an +// existing solution. +std::string expandShellVarsInFileName(std::string const& input) +{ + std::regex e(R"(\$\{?[a-zA-Z0-9_]*\}?)"); + std::regex e3("[a-zA-Z0-9_]+"); + std::string finalstr; + std::sregex_iterator iter; + auto words_end = std::sregex_iterator(); // the end iterator (default) + auto words_begin = std::sregex_iterator(input.begin(), input.end(), e); + + // check first of all if there is shell variable inside + if (words_end == words_begin) { + return input; + } + + std::string tail; + for (auto i = words_begin; i != words_end; ++i) { + std::smatch match = *i; + // remove ${ and } + std::smatch m; + std::string s(match.str()); + + if (std::regex_search(s, m, e3)) { + auto envlookup = getenv(m[0].str().c_str()); + if (envlookup) { + finalstr += match.prefix().str() + std::string(envlookup); + } else { + // in case of non existance we keep the env part unreplaced + finalstr += match.prefix().str() + "${" + m[0].str().c_str() + "}"; + } + tail = match.suffix().str(); + } + } + finalstr += tail; + return finalstr; +} } // namespace o2::utils diff --git a/Common/Utils/src/IRFrameSelector.cxx b/Common/Utils/src/IRFrameSelector.cxx index aa5245f86e4f5..abc0ee1ee6ce3 100644 --- a/Common/Utils/src/IRFrameSelector.cxx +++ b/Common/Utils/src/IRFrameSelector.cxx @@ -21,10 +21,27 @@ using namespace o2::utils; -long IRFrameSelector::check(const o2::dataformats::IRFrame& fr) +gsl::span IRFrameSelector::getMatchingFrames(const o2::dataformats::IRFrame& fr) { - long ans = -1; + // extract span of IRFrames matching to fr + auto lower = std::lower_bound(mFrames.begin(), mFrames.end(), o2::dataformats::IRFrame{fr.getMin(), fr.getMin()}); + if (lower == mFrames.end() || *lower > fr) { + return {}; + } + auto upper = std::upper_bound(lower, mFrames.end(), o2::dataformats::IRFrame{fr.getMax(), fr.getMax()}); + return {&*lower, size_t(std::distance(lower, upper))}; +} +long IRFrameSelector::check(o2::dataformats::IRFrame fr, size_t bwd, size_t fwd) +{ + // check if fr overlaps with at least 1 entry in the frames container, if needed expand fr by -bwd and fwd BCs from 2 sides + long ans = -1; + if (bwd) { + fr.setMin(fr.getMin().toLong() > bwd ? fr.getMin() - bwd : o2::InteractionRecord{0, 0}); + } + if (fwd) { + fr.setMax(o2::InteractionRecord::MaxGlobalBCs - fr.getMax().toLong() > fwd ? fr.getMax() + fwd : o2::InteractionRecord::getIRMaxBC()); + } // find entry which overlaps or above fr auto fullcheck = [&fr, this]() -> long { auto lower = std::lower_bound(this->mFrames.begin(), this->mFrames.end(), fr); @@ -88,19 +105,20 @@ size_t IRFrameSelector::loadIRFrames(const std::string& fname) // read IRFrames to filter from the file std::unique_ptr tfl(TFile::Open(fname.c_str())); if (!tfl || tfl->IsZombie()) { - LOGP(fatal, "Cannot open file {}", fname); + LOGP(fatal, "Cannot open selected IRFrames file {}", fname); } auto klst = gDirectory->GetListOfKeys(); TIter nextkey(klst); TKey* key; std::string clVec{TClass::GetClass("std::vector")->GetName()}; bool done = false; + bool toBeSorted = false; while ((key = (TKey*)nextkey())) { std::string kcl(key->GetClassName()); if (kcl == clVec) { auto* v = (std::vector*)tfl->GetObjectUnchecked(key->GetName()); if (!v) { - LOGP(fatal, "Failed to extract vector {} from {}", key->GetName(), fname); + LOGP(fatal, "Failed to extract IRFrames vector {} from {}", key->GetName(), fname); } mOwnList.insert(mOwnList.end(), v->begin(), v->end()); LOGP(info, "Loaded {} IRFrames from vector {} of {}", mOwnList.size(), key->GetName(), fname); @@ -120,8 +138,28 @@ size_t IRFrameSelector::loadIRFrames(const std::string& fname) done = true; LOGP(info, "Loaded {} IRFrames from tree {} of {}", mOwnList.size(), key->GetName(), fname); break; + } else if (kcl == "TDirectoryFile") { + TTree* bcRanges = (TTree*)tfl->Get(fmt::format("{}/O2bcranges", key->GetName()).data()); + if (!bcRanges) { + continue; + } + LOGP(info, "Loading BCrange trees in the directory {}", key->GetName()); + ULong64_t minBC, maxBC; + bcRanges->SetBranchAddress("fBCstart", &minBC); + bcRanges->SetBranchAddress("fBCend", &maxBC); + for (int i = 0; i < (int)bcRanges->GetEntries(); i++) { + bcRanges->GetEntry(i); + mOwnList.emplace_back(InteractionRecord::long2IR(minBC), InteractionRecord::long2IR(maxBC)); + } + done = true; + toBeSorted = true; } } + + if (toBeSorted) { + LOGP(info, "Sorting {} IRFrames", mOwnList.size()); + std::sort(mOwnList.begin(), mOwnList.end(), [](const auto& a, const auto& b) { return a.getMin() < b.getMin(); }); + } if (!true) { LOGP(fatal, "Did not find neither tree nor vector of IRFrames in {}", fname); } @@ -129,6 +167,16 @@ size_t IRFrameSelector::loadIRFrames(const std::string& fname) return mOwnList.size(); } +void IRFrameSelector::setOwnList(const std::vector& lst, bool toBeSorted) +{ + clear(); + mOwnList.insert(mOwnList.end(), lst.begin(), lst.end()); + if (toBeSorted) { + std::sort(mOwnList.begin(), mOwnList.end(), [](const auto& a, const auto& b) { return a.getMin() < b.getMin(); }); + } + setSelectedIRFrames(mOwnList, 0, 0, 0, false); +} + void IRFrameSelector::print(bool lst) const { LOGP(info, "Last query stopped at entry {} for IRFrame {}:{}", mLastBoundID, @@ -140,3 +188,47 @@ void IRFrameSelector::print(bool lst) const } } } + +void IRFrameSelector::clear() +{ + mIsSet = false; + mOwnList.clear(); + mLastIRFrameChecked.getMin().clear(); // invalidate + mLastBoundID = -1; + mFrames = {}; +} + +void IRFrameSelector::applyMargins(size_t bwd, size_t fwd, long shift, bool removeOverlaps) +{ + // apply shift and margin to all IRFrames by converting them to IRFrame.getMin() - fwd + shift, IRFrame.getMax() + bwd + shift + LOGP(debug, "applyMargins({},{},{})", bwd, fwd, removeOverlaps); + if ((!fwd && !bwd && !shift) || !mIsSet || !mFrames.size()) { + return; + } + mLastBoundID = -1; + std::vector lst; + long shiftBwd = bwd - shift, shiftFwd = fwd + shift; + + for (const auto& fr : mFrames) { + auto irmin = fr.getMin(); + auto irmax = fr.getMax(); + if (shiftBwd > 0) { + irmin = fr.getMin().toLong() > shiftBwd ? fr.getMin() - shiftBwd : o2::InteractionRecord{0, 0}; + } else { + irmin = (o2::InteractionRecord::MaxGlobalBCs - fr.getMin().toLong()) > -shiftBwd ? fr.getMin() - shiftBwd : o2::InteractionRecord::getIRMaxBC(); + } + if (shiftFwd > 0) { + irmax = (o2::InteractionRecord::MaxGlobalBCs - fr.getMax().toLong()) > shiftFwd ? fr.getMax() + shiftFwd : o2::InteractionRecord::getIRMaxBC(); + } else { + irmax = fr.getMax().toLong() > -shiftFwd ? fr.getMax() + shiftFwd : o2::InteractionRecord{0, 0}; + } + LOGP(debug, "before removerlap: {}:{} -> {}:{}", fr.getMin().toLong(), fr.getMax().toLong(), irmin.toLong(), irmax.toLong()); + if (removeOverlaps && lst.size() && lst.back().getMax() >= irmin) { + lst.back().setMax(irmax); + } else { + lst.emplace_back(irmin, irmax).info = fr.info; + } + } + mOwnList.swap(lst); + mFrames = gsl::span(mOwnList.data(), mOwnList.size()); +} diff --git a/Common/Utils/src/RootSerializableKeyValueStore.cxx b/Common/Utils/src/RootSerializableKeyValueStore.cxx index 7bc424300674e..0e0902f6cc4f2 100644 --- a/Common/Utils/src/RootSerializableKeyValueStore.cxx +++ b/Common/Utils/src/RootSerializableKeyValueStore.cxx @@ -14,11 +14,57 @@ using namespace o2::utils; -void RootSerializableKeyValueStore::print() const +namespace +{ +template +std::string stringFromType(char* buffer) +{ + T value; + std::memcpy(&value, buffer, sizeof(T)); + return std::to_string(value); +} +} // namespace + +void RootSerializableKeyValueStore::print(bool includetypeinfo) const { for (auto& p : mStore) { const auto& key = p.first; const auto info = p.second; - std::cout << "key: " << key << " of-type: " << info.typeinfo_name << "\n"; + auto tinfo = info.typeinfo_name; + + std::string value("unknown-value"); + // let's try to decode the value as a string if we can + if (tinfo == typeid(int).name()) { + value = stringFromType(info.bufferptr); + } + // let's try to decode the value as a string if we can + else if (tinfo == typeid(unsigned int).name()) { + value = stringFromType(info.bufferptr); + } + // let's try to decode the value as a string if we can + else if (tinfo == typeid(short).name()) { + value = stringFromType(info.bufferptr); + } + // let's try to decode the value as a string if we can + else if (tinfo == typeid(unsigned short).name()) { + value = stringFromType(info.bufferptr); + } + // let's try to decode the value as a string if we can + else if (tinfo == typeid(double).name()) { + value = stringFromType(info.bufferptr); + } + // let's try to decode the value as a string if we can + else if (tinfo == typeid(float).name()) { + value = stringFromType(info.bufferptr); + } + // let's try to decode the value as a string if we can + else if (tinfo == typeid(std::string).name()) { + value = *(get(key)); + } + std::cout << "key: " << key << " value: " << value; + if (includetypeinfo) { + std::cout << " type: " << info.typeinfo_name; + } + std::cout << "\n"; } } diff --git a/Common/Utils/src/ShmManager.cxx b/Common/Utils/src/ShmManager.cxx index cb3af5dedf380..26b30be062220 100644 --- a/Common/Utils/src/ShmManager.cxx +++ b/Common/Utils/src/ShmManager.cxx @@ -119,8 +119,8 @@ bool ShmManager::createGlobalSegment(int nsegments) return false; } - LOG(info) << "CREATING SIM SHARED MEM SEGMENT FOR " << nsegments << " WORKERS"; #ifdef USESHM + LOG(info) << "CREATING SIM SHARED MEM SEGMENT FOR " << nsegments << " WORKERS"; // LOG(info) << "SIZEOF ShmMetaInfo " << sizeof(ShmMetaInfo); const auto totalsize = sizeof(ShmMetaInfo) + SHMPOOLSIZE * nsegments; if ((mShmID = shmget(IPC_PRIVATE, totalsize, IPC_CREAT | 0666)) == -1) { diff --git a/Common/Utils/src/StringUtils.cxx b/Common/Utils/src/StringUtils.cxx index f14b82f06a85c..687225d069ed2 100644 --- a/Common/Utils/src/StringUtils.cxx +++ b/Common/Utils/src/StringUtils.cxx @@ -12,6 +12,11 @@ #include "CommonUtils/StringUtils.h" #include #include +#ifndef GPUCA_STANDALONE +#include +#include +#endif +#include using namespace o2::utils; @@ -32,13 +37,28 @@ std::vector Str::tokenize(const std::string& src, char delim, bool return tokens; } +// replace all occurencies of from by to, return count +int Str::replaceAll(std::string& s, const std::string& from, const std::string& to) +{ + int count = 0; + size_t pos = 0; + while ((pos = s.find(from, pos)) != std::string::npos) { + s.replace(pos, from.length(), to); + pos += to.length(); // Handles case where 'to' is a substring of 'from' + count++; + } + return count; +} + // generate random string of given lenght, suitable for file names std::string Str::getRandomString(int lenght) { - auto nextAllowed = []() { + int pid = (int)getpid(); + auto nextAllowed = [pid]() { constexpr char chars[] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; constexpr size_t L = sizeof(chars) - 1; - return chars[std::rand() % L]; + int rn = std::rand() | pid; + return chars[rn % L]; }; std::string str(lenght, 0); std::generate_n(str.begin(), lenght, nextAllowed); @@ -47,7 +67,7 @@ std::string Str::getRandomString(int lenght) bool Str::pathExists(const std::string_view p) { - return std::filesystem::exists(std::string{p}); + return p.compare(0, 8, "alien://") ? std::filesystem::exists(std::string{p}) : true; // we don't validate alien paths } bool Str::pathIsDirectory(const std::string_view p) @@ -60,21 +80,35 @@ std::string Str::getFullPath(const std::string_view p) return std::filesystem::canonical(std::string{p}).string(); } +#ifndef GPUCA_STANDALONE std::string Str::rectifyDirectory(const std::string_view p) { std::string dir(p); if (dir.empty() || dir == "none") { dir = ""; } else { - dir = getFullPath(dir); - if (!pathIsDirectory(dir)) { - throw std::runtime_error(fmt::format("{:s} is not an accessible directory", dir)); + if (p.compare(0, 8, "alien://") == 0) { + if (!gGrid && !TGrid::Connect("alien://")) { + throw std::runtime_error(fmt::format("failed to initialize alien for {}", dir)); + } + // for root or raw files do not treat as directory + if (dir.back() != '/' && !(endsWith(dir, ".root") || endsWith(dir, ".raw") || endsWith(dir, ".tf"))) { + dir += '/'; + } } else { - dir += '/'; + dir = getFullPath(dir); + if (!pathIsDirectory(dir)) { + throw std::runtime_error(fmt::format("{} is not an accessible directory", dir)); + } else { + if (dir.back() != '/') { + dir += '/'; + } + } } } return dir; } +#endif // Create unique non-existing path name starting with prefix. Loose equivalent of boost::filesystem::unique_path() // The prefix can be either existing directory or just a string to add in front of the random part diff --git a/Common/Utils/src/TreeStream.cxx b/Common/Utils/src/TreeStream.cxx index deebc4d3edb9f..cd0641a11d043 100644 --- a/Common/Utils/src/TreeStream.cxx +++ b/Common/Utils/src/TreeStream.cxx @@ -24,13 +24,12 @@ TreeStream::TreeStream(const char* treename) : mTree(treename, treename) } //_________________________________________________ -int TreeStream::CheckIn(Char_t type, void* pointer) +int TreeStream::CheckIn(Char_t type, const void* pointer) { // Insert object if (mCurrentIndex >= static_cast(mElements.size())) { - mElements.emplace_back(); - auto& element = mElements.back(); + auto& element = mElements.emplace_back(); element.type = type; TString name = mNextName; if (name.Length()) { @@ -42,6 +41,8 @@ int TreeStream::CheckIn(Char_t type, void* pointer) } element.name = name.Data(); element.ptr = pointer; + element.arsize = mNextArraySize; + mNextArraySize = 1; // reset } else { auto& element = mElements[mCurrentIndex]; if (element.type != type) { @@ -77,26 +78,32 @@ void TreeStream::BuildTree() name = TString::Format("B%d", i); } if (element.cls) { - br = mTree.Branch(name.Data(), element.cls->GetName(), &(element.ptr)); + br = mTree.Branch(name.Data(), element.cls->GetName(), const_cast(&element.ptr)); mBranches[i] = br; if (entriesFilled) { br->SetAddress(nullptr); for (int ientry = 0; ientry < entriesFilled; ientry++) { br->Fill(); } - br->SetAddress(&(element.ptr)); + br->SetAddress(const_cast(&element.ptr)); } } if (element.type > 0) { - TString nameC = TString::Format("%s/%c", name.Data(), element.type); - br = mTree.Branch(name.Data(), element.ptr, nameC.Data()); + TString nameC; + if (element.arsize > 1) { + nameC = TString::Format("%s[%d]/%c", name.Data(), element.arsize, + element.type); + } else { + nameC = TString::Format("%s/%c", name.Data(), element.type); + } + br = mTree.Branch(name.Data(), const_cast(element.ptr), nameC.Data()); if (entriesFilled) { br->SetAddress(nullptr); for (int ientry = 0; ientry < entriesFilled; ientry++) { br->Fill(); } - br->SetAddress(element.ptr); + br->SetAddress(const_cast(element.ptr)); } mBranches[i] = br; } @@ -120,7 +127,7 @@ void TreeStream::Fill() auto br = mBranches[i]; if (br) { if (element.type) { - br->SetAddress(element.ptr); + br->SetAddress(const_cast(element.ptr)); } } } @@ -148,28 +155,43 @@ TreeStream& TreeStream::Endl() TreeStream& TreeStream::operator<<(const Char_t* name) { // Stream the branch name - // if (name[0] == '\n') { return Endl(); } - // + // if tree was already defined ignore if (mTree.GetEntries() > 0) { return *this; } + + int arsize = 1; + // check branch name if tree was not - // Int_t last = 0; for (last = 0;; last++) { if (name[last] == 0) { break; } } - if (last > 0 && name[last - 1] == '=') { mNextName = name; - mNextName[last - 1] = 0; + mNextName[last - 1] = 0; // remove '=' from string mNextNameCounter = 0; + + TString inName{name}; + auto brkStaPos = inName.Index('['); + + if (brkStaPos != kNPOS) { + auto brkEndPos = inName.Index(']'); + if (brkEndPos != kNPOS && brkEndPos > brkStaPos + 1) { + TString size = inName(brkStaPos + 1, brkEndPos - brkStaPos - 1); + arsize = size.Atoi(); + mNextName = inName(0, brkStaPos); // use parsed name + } + } } + + mNextArraySize = arsize; + return *this; } diff --git a/Common/Utils/src/TreeStreamRedirector.cxx b/Common/Utils/src/TreeStreamRedirector.cxx index 86e0eb5a37552..06fb3d65678c4 100644 --- a/Common/Utils/src/TreeStreamRedirector.cxx +++ b/Common/Utils/src/TreeStreamRedirector.cxx @@ -111,11 +111,13 @@ TreeStream& TreeStreamRedirector::operator<<(const char* name) void TreeStreamRedirector::Close() { // flush and close - + if (!mDirectory) { + return; + } TDirectory* backup = gDirectory; mDirectory->cd(); for (auto& layout : mDataLayouts) { - layout->getTree().Write(layout->getName()); + layout->getTree().Write(layout->getName(), TObject::kOverwrite); } mDataLayouts.clear(); if (backup) { diff --git a/Common/Utils/src/fpu.cxx b/Common/Utils/src/fpu.cxx new file mode 100644 index 0000000000000..505935e15c36e --- /dev/null +++ b/Common/Utils/src/fpu.cxx @@ -0,0 +1,44 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +// To enable check for problematic LIBXML version uncomment this +// #define _PROTECT_LIBXML_ + +#ifdef __linux +#ifdef _PROTECT_LIBXML_ +#include +#if LIBXML_VERSION > 20912 +#define _DUMMY_FEE_TRAP_ +#endif // LIBXML_VERSION > 20912 +#endif // _PROTECT_LIBXML_ +#else // __linux +#define _DUMMY_FEE_TRAP_ +#endif // __linux + +#ifdef _DUMMY_FEE_TRAP_ +void trapfpe() {} +#else // _DUMMY_FEE_TRAP_ +#define _GNU_SOURCE 1 +#include +#include +static void __attribute__((constructor)) trapfpe() +{ + // allows to disable set of particular FE's by setting corresponding bit of the O2_DISABLE_FPE_TRAP, + // i.e. to enable only FE_DIVBYZERO use O2_DISABLE_FPE_TRAP=9 + // const char* ev = std::getenv("O2_DISABLE_FPE_TRAP"); + // int enabledFE = (FE_INVALID | FE_DIVBYZERO | FE_OVERFLOW) & ~(ev ? atoi(ev) : 0); + const char* ev = std::getenv("O2_ENABLE_FPE_TRAP"); + int enabledFE = (FE_INVALID | FE_DIVBYZERO | FE_OVERFLOW) & (ev ? atoi(ev) : 0); + if (enabledFE) { + feenableexcept(enabledFE); + } +} +#endif // _DUMMY_FEE_TRAP_ diff --git a/Common/Utils/test/testBoostSerializer.cxx b/Common/Utils/test/testBoostSerializer.cxx deleted file mode 100644 index 520bc7dda8b08..0000000000000 --- a/Common/Utils/test/testBoostSerializer.cxx +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// -/// @author Gabriele Gaetano Fronzé - -#define BOOST_TEST_DYN_LINK -#define BOOST_TEST_MAIN - -#include -#include -#include -#include -#include "CommonUtils/BoostSerializer.h" -#include - -using namespace o2::utils; - -struct TestCluster { - uint8_t deId; ///< Detection element ID - float xCoor; ///< Local x coordinate - float yCoor; ///< Local y coordinate - float sigmaX2; ///< Square of dispersion along x - float sigmaY2; ///< Square of dispersion along y - - friend class boost::serialization::access; - - /// Serializes the struct - template - void serialize(Archive& ar, const unsigned int version) - { - ar& deId& xCoor& yCoor& sigmaX2& sigmaY2; - } -}; - -BOOST_AUTO_TEST_SUITE(testDPLSerializer) - -// BOOST_AUTO_TEST_CASE(testTrivialTypeVect) -// { -// using contType = std::vector; - -// contType inputV{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; - -// auto msgStr = BoostSerialize(inputV).str(); -// auto inputV2 = BoostDeserialize(msgStr); - -// BOOST_TEST(inputV.size() == inputV2.size()); - -// size_t i = 0; -// for (auto const& test : inputV) { -// BOOST_TEST(test == inputV2[i]); -// i++; -// } -// } - -// BOOST_AUTO_TEST_CASE(testTrivialTypeArray) -// { -// using contType = std::array; - -// contType inputV{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; - -// auto msgStr = BoostSerialize(inputV).str(); -// auto inputV2 = BoostDeserialize(msgStr); - -// BOOST_TEST(inputV.size() == inputV2.size()); - -// size_t i = 0; -// for (auto const& test : inputV) { -// BOOST_TEST(test == inputV2[i]); -// i++; -// } -// } - -// BOOST_AUTO_TEST_CASE(testTrivialTypeList) -// { -// using contType = std::list; - -// contType inputV{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; - -// auto msgStr = BoostSerialize(inputV).str(); -// auto inputV2 = BoostDeserialize(msgStr); - -// BOOST_TEST(inputV.size() == inputV2.size()); - -// size_t i = 0; -// for (auto const& test : inputV) { -// auto value = std::next(std::begin(inputV2), i).operator*(); -// BOOST_TEST(test == value); -// i++; -// } -// } - -BOOST_AUTO_TEST_CASE(testBoostSerialisedType) -{ - using contType = std::vector; - - contType inputV; - - for (size_t i = 0; i < 17; i++) { - float iFloat = (float)i; - inputV.emplace_back(TestCluster{(uint8_t)i, 0.3f * iFloat, 0.5f * iFloat, 0.7f / iFloat, 0.9f / iFloat}); - } - - auto msgStr = BoostSerialize(inputV).str(); - auto inputV2 = BoostDeserialize(msgStr); - - BOOST_TEST(inputV.size() == inputV2.size()); - - size_t i = 0; - for (auto const& test : inputV) { - BOOST_TEST(test.deId == inputV2[i].deId); - BOOST_TEST(test.xCoor == inputV2[i].xCoor); - BOOST_TEST(test.yCoor == inputV2[i].yCoor); - BOOST_TEST(test.sigmaX2 == inputV2[i].sigmaX2); - BOOST_TEST(test.sigmaY2 == inputV2[i].sigmaY2); - i++; - } -} - -BOOST_AUTO_TEST_SUITE_END() diff --git a/Common/Utils/test/testConfigurableParam.cxx b/Common/Utils/test/testConfigurableParam.cxx new file mode 100644 index 0000000000000..3ef177aaca3fe --- /dev/null +++ b/Common/Utils/test/testConfigurableParam.cxx @@ -0,0 +1,145 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#define BOOST_TEST_MODULE Test ConfigurableParams +#define BOOST_TEST_MAIN +#define BOOST_TEST_DYN_LINK + +#include +#include +#include + +#include "CommonUtils/ConfigurableParamTest.h" + +using namespace o2::conf; +using namespace o2::conf::test; + +BOOST_AUTO_TEST_CASE(ConfigurableParam_Basic) +{ + // Tests the default parameters and also getter helpers. + auto& param = TestParam::Instance(); + BOOST_CHECK_EQUAL(param.iValue, 42); + BOOST_CHECK_EQUAL(param.dValue, 3.14); + BOOST_CHECK_EQUAL(param.bValue, true); + BOOST_CHECK_EQUAL(param.sValue, "default"); + BOOST_CHECK_EQUAL(static_cast(param.eValue), 2); + + BOOST_CHECK_EQUAL(ConfigurableParam::getValueAs("TestParam.iValue"), 42); + BOOST_CHECK_EQUAL(ConfigurableParam::getValueAs("TestParam.dValue"), 3.14); + BOOST_CHECK_EQUAL(ConfigurableParam::getValueAs("TestParam.bValue"), true); + BOOST_CHECK_EQUAL(ConfigurableParam::getValueAs("TestParam.sValue"), "default"); +} + +BOOST_AUTO_TEST_CASE(ConfigurableParam_SG_Fundamental) +{ + // tests runtime setting and getting for fundamental types + ConfigurableParam::setValue("TestParam.iValue", "100"); + ConfigurableParam::setValue("TestParam.dValue", "2.718"); + ConfigurableParam::setValue("TestParam.bValue", "0"); + ConfigurableParam::setValue("TestParam.sValue", "modified"); + ConfigurableParam::setValue("TestParam.eValue", "0"); + + auto& param = TestParam::Instance(); + param.printKeyValues(); + BOOST_CHECK_EQUAL(param.iValue, 100); + BOOST_CHECK_EQUAL(param.dValue, 2.718); + BOOST_CHECK_EQUAL(param.bValue, false); + BOOST_CHECK_EQUAL(param.sValue, "modified"); + BOOST_CHECK_EQUAL(static_cast(param.eValue), 0); +} + +BOOST_AUTO_TEST_CASE(ConfigurableParam_SG_CArray) +{ + // tests setting and getting for a c-style array type + auto& param = TestParam::Instance(); + BOOST_CHECK_EQUAL(ConfigurableParam::getValueAs("TestParam.caValue[0]"), 0); + BOOST_CHECK_EQUAL(ConfigurableParam::getValueAs("TestParam.caValue[1]"), 1); + BOOST_CHECK_EQUAL(ConfigurableParam::getValueAs("TestParam.caValue[2]"), 2); + + ConfigurableParam::setValue("TestParam.caValue[1]", "99"); + BOOST_CHECK_EQUAL(ConfigurableParam::getValueAs("TestParam.caValue[1]"), 99); +} + +BOOST_AUTO_TEST_CASE(ConfigurableParam_Provenance) +{ + // tests correct setting of provenance + BOOST_CHECK_EQUAL(ConfigurableParam::getProvenance("TestParam.iValueProvenanceTest"), ConfigurableParam::EParamProvenance::kCODE); + ConfigurableParam::setValue("TestParam.iValueProvenanceTest", "123"); + BOOST_CHECK_EQUAL(ConfigurableParam::getProvenance("TestParam.iValueProvenanceTest"), ConfigurableParam::EParamProvenance::kRT); +} + +BOOST_AUTO_TEST_CASE(ConfigurableParam_FileIO_Ini) +{ + // test for ini file serialization + const std::string testFileName = "test_config.ini"; + auto iValueBefore = TestParam::Instance().iValue; + auto sValueBefore = TestParam::Instance().sValue; + ConfigurableParam::writeINI(testFileName); + ConfigurableParam::setValue("TestParam.iValue", "999"); + ConfigurableParam::setValue("TestParam.sValue", testFileName); + ConfigurableParam::updateFromFile(testFileName); + BOOST_CHECK_EQUAL(TestParam::Instance().iValue, iValueBefore); + BOOST_CHECK_EQUAL(TestParam::Instance().sValue, sValueBefore); + std::remove(testFileName.c_str()); +} + +BOOST_AUTO_TEST_CASE(ConfigurableParam_FileIO_Json) +{ + // test for json file serialization + const std::string testFileName = "test_config.json"; + auto iValueBefore = TestParam::Instance().iValue; + auto sValueBefore = TestParam::Instance().sValue; + ConfigurableParam::writeJSON(testFileName); + ConfigurableParam::setValue("TestParam.iValue", "999"); + ConfigurableParam::setValue("TestParam.sValue", testFileName); + ConfigurableParam::updateFromFile(testFileName); + BOOST_CHECK_EQUAL(TestParam::Instance().iValue, iValueBefore); + BOOST_CHECK_EQUAL(TestParam::Instance().sValue, sValueBefore); + std::remove(testFileName.c_str()); +} + +BOOST_AUTO_TEST_CASE(ConfigurableParam_FileIO_ROOT) +{ + // test for root file serialization + const std::string testFileName = "test_config.root"; + auto iValueBefore = TestParam::Instance().iValue; + auto sValueBefore = TestParam::Instance().sValue; + TFile* testFile = TFile::Open(testFileName.c_str(), "RECREATE"); + TestParam::Instance().serializeTo(testFile); + testFile->Close(); + ConfigurableParam::setValue("TestParam.iValue", "999"); + ConfigurableParam::setValue("TestParam.sValue", testFileName); + ConfigurableParam::fromCCDB(testFileName); + BOOST_CHECK_EQUAL(TestParam::Instance().iValue, iValueBefore); + BOOST_CHECK_EQUAL(TestParam::Instance().sValue, sValueBefore); + std::remove(testFileName.c_str()); +} + +BOOST_AUTO_TEST_CASE(ConfigurableParam_Cli) +{ + // test setting values from as a cli arg string + ConfigurableParam::updateFromString("TestParam.iValue=55;TestParam.sValue=cli"); + BOOST_CHECK_EQUAL(TestParam::Instance().iValue, 55); + BOOST_CHECK_EQUAL(TestParam::Instance().sValue, "cli"); +} + +BOOST_AUTO_TEST_CASE(ConfigurableParam_LiteralSuffix) +{ + // test setting values with the correct literal suffix + ConfigurableParam::updateFromString("TestParam.fValue=42.f"); + BOOST_CHECK_EQUAL(TestParam::Instance().fValue, 42.f); + + ConfigurableParam::setValue("TestParam.ullValue", "999ull"); + BOOST_CHECK_EQUAL(TestParam::Instance().ullValue, 999ULL); + // check using wrong literal suffix fails, prints error to std + ConfigurableParam::setValue("TestParam.ullValue", "888u"); + BOOST_CHECK_NE(TestParam::Instance().ullValue, 888); +} diff --git a/Common/Utils/test/testEnumFlags.cxx b/Common/Utils/test/testEnumFlags.cxx new file mode 100644 index 0000000000000..9101ffb97fdfe --- /dev/null +++ b/Common/Utils/test/testEnumFlags.cxx @@ -0,0 +1,653 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#define BOOST_TEST_MODULE Test Flags +#define BOOST_TEST_MAIN +#define BOOST_TEST_DYN_LINK + +#include +#include +#include + +#include +#include + +#include "CommonUtils/EnumFlags.h" + +// Example enum to use with EnumFlags +enum class TestEnum : uint8_t { + Bit1 = 0, + Bit2, + Bit3, + Bit4, + Bit5VeryLongName, +}; + +// Very long enum +// to test that it works beyond 32 bits upto 64 bits +#define ENUM_BIT_NAME(n) Bit##n +#define ENUM_BIT_NAME_EXPAND(n) ENUM_BIT_NAME(n) +#define ENUM_BIT(z, n, _) ENUM_BIT_NAME_EXPAND(BOOST_PP_INC(n)) = (n), +enum class TestEnumLong : uint64_t { + BOOST_PP_REPEAT(64, ENUM_BIT, _) +}; +#undef ENUM_BIT +#undef ENUM_BIT_NAME +#undef ENUM_BIT_NAME_EXPAND + +BOOST_AUTO_TEST_CASE(Flags_test) +{ + using EFlags = o2::utils::EnumFlags; + + // Test default initialization + EFlags flags; + BOOST_TEST(flags.None == 0); + BOOST_TEST(flags.All == 31); + BOOST_TEST(flags.value() == 0); + BOOST_TEST(!flags.any()); + + // Test initialization with a single flag + EFlags flag1(TestEnum::Bit1); + BOOST_TEST(flag1.test(TestEnum::Bit1)); + BOOST_TEST(!flag1.test(TestEnum::Bit2)); + BOOST_TEST(flag1.value() == (1 << static_cast(TestEnum::Bit1))); + + // Test initialization with initializer list + EFlags multipleFlags({TestEnum::Bit1, TestEnum::Bit3}); + BOOST_TEST(multipleFlags.test(TestEnum::Bit1)); + BOOST_TEST(multipleFlags.test(TestEnum::Bit3)); + BOOST_TEST(!multipleFlags.test(TestEnum::Bit2)); + BOOST_TEST(multipleFlags.any()); + + // Test reset + multipleFlags.reset(TestEnum::Bit1); + BOOST_TEST(!multipleFlags.test(TestEnum::Bit1)); + BOOST_TEST(multipleFlags.test(TestEnum::Bit3)); + multipleFlags.reset(); + BOOST_TEST(!multipleFlags.any()); + + // Test multiset + multipleFlags.reset(); + multipleFlags.set(TestEnum::Bit2, TestEnum::Bit4); + BOOST_TEST(!multipleFlags.test(TestEnum::Bit1)); + BOOST_TEST(multipleFlags.test(TestEnum::Bit2)); + BOOST_TEST(!multipleFlags.test(TestEnum::Bit3)); + BOOST_TEST(multipleFlags.test(TestEnum::Bit4)); + BOOST_TEST(!multipleFlags.test(TestEnum::Bit5VeryLongName)); + + // Test operator| + EFlags combinedFlags = flag1 | EFlags(TestEnum::Bit2); + BOOST_TEST(combinedFlags.test(TestEnum::Bit1)); + BOOST_TEST(combinedFlags.test(TestEnum::Bit2)); + BOOST_TEST(!combinedFlags.test(TestEnum::Bit3)); + combinedFlags |= TestEnum::Bit5VeryLongName; + BOOST_TEST(combinedFlags.test(TestEnum::Bit5VeryLongName)); + + // Test operator[] + BOOST_TEST(combinedFlags[TestEnum::Bit1]); + BOOST_TEST(combinedFlags[TestEnum::Bit2]); + BOOST_TEST(!combinedFlags[TestEnum::Bit3]); + + // Test operator|= + combinedFlags |= TestEnum::Bit3; + BOOST_TEST(combinedFlags.test(TestEnum::Bit3)); + + // Test operator& + EFlags intersection = combinedFlags & TestEnum::Bit1; + BOOST_TEST(intersection.test(TestEnum::Bit1)); + BOOST_TEST(!intersection.test(TestEnum::Bit2)); + BOOST_TEST(intersection.value() == (1 << static_cast(TestEnum::Bit1))); + + // Test operator&= + combinedFlags &= TestEnum::Bit1; + BOOST_TEST(combinedFlags.test(TestEnum::Bit1)); + BOOST_TEST(!combinedFlags.test(TestEnum::Bit2)); + BOOST_TEST(!combinedFlags.test(TestEnum::Bit3)); + + // Test operator~ (complement) + EFlags complement = ~EFlags(TestEnum::Bit1); + BOOST_TEST(!complement.test(TestEnum::Bit1)); + BOOST_TEST(complement.test(TestEnum::Bit2)); + BOOST_TEST(complement.test(TestEnum::Bit3)); + + // Test string() method + { + std::string flagString = flag1.string(); + BOOST_TEST(flagString.back() == '1'); // Ensure the least significant bit is set for flag1 + } + + // Test set with binary string + { + std::string binaryStr = "101"; + flags.set(binaryStr, 2); + BOOST_TEST(flags.test(TestEnum::Bit1)); + BOOST_TEST(!flags.test(TestEnum::Bit2)); + BOOST_TEST(flags.test(TestEnum::Bit3)); + } + + // Test invalid binary string in set + BOOST_CHECK_THROW(flags.set(std::string("invalid"), 2), std::invalid_argument); + + // Test range validation in set + BOOST_CHECK_THROW(flags.set(std::string("100000000"), 2), std::out_of_range); + + { // Test that return lists are sensible + const auto n = flags.getNames(); + const auto v = flags.getValues(); + BOOST_CHECK(n.size() == v.size()); + } + + { // print test + std::cout << flags; + } + + // Test flag tokenization and parsing + { + { // only one scoped flag + std::string str = "TestEnum::Bit2"; + flags.set(str); + BOOST_TEST(flags.test(TestEnum::Bit2)); + BOOST_TEST(flags.none_of(TestEnum::Bit1, TestEnum::Bit3, TestEnum::Bit4)); + } + + { // test with ws-triming and scope mixing + std::string str = "Bit4|TestEnum::Bit2 | Bit1 "; + flags.set(str); + BOOST_TEST(flags.test(TestEnum::Bit1)); + BOOST_TEST(flags.test(TestEnum::Bit2)); + BOOST_TEST(!flags.test(TestEnum::Bit3)); + BOOST_TEST(flags.test(TestEnum::Bit4)); + } + + { // test with , delimiter + std::string str = "Bit4,TestEnum::Bit2 , Bit1 "; + flags.set(str); + BOOST_TEST(flags.test(TestEnum::Bit1)); + BOOST_TEST(flags.test(TestEnum::Bit2)); + BOOST_TEST(!flags.test(TestEnum::Bit3)); + BOOST_TEST(flags.test(TestEnum::Bit4)); + } + + { // test with ; delimiter + std::string str = "Bit4;TestEnum::Bit2 ; Bit1 "; + flags.set(str); + BOOST_TEST(flags.test(TestEnum::Bit1)); + BOOST_TEST(flags.test(TestEnum::Bit2)); + BOOST_TEST(!flags.test(TestEnum::Bit3)); + BOOST_TEST(flags.test(TestEnum::Bit4)); + } + + { // throw test with mixed delimiter + std::string str = "Bit4|TestEnum::Bit2 , Bit1 "; + BOOST_CHECK_THROW(flags.set(str), std::invalid_argument); + } + + { // test throw + std::string str = "Invalid"; + BOOST_CHECK_THROW(flags.set(str), std::invalid_argument); + } + } + + // Test all_of and none_of + { + EFlags allFlags({TestEnum::Bit1, TestEnum::Bit2, TestEnum::Bit3}); + BOOST_TEST(allFlags.all_of(TestEnum::Bit1, TestEnum::Bit2)); + BOOST_TEST(!allFlags.all_of(TestEnum::Bit4)); + BOOST_TEST(allFlags.none_of(TestEnum::Bit4)); + } + + // Test toggle + { + EFlags toggleFlags; + toggleFlags.toggle(TestEnum::Bit4); + BOOST_TEST(toggleFlags.test(TestEnum::Bit4)); + toggleFlags.toggle(TestEnum::Bit4); + BOOST_TEST(!toggleFlags.test(TestEnum::Bit4)); + } + + // Create a flag set and serialize it + { + EFlags serializedFlags{TestEnum::Bit1, TestEnum::Bit3}; + std::string serialized = serializedFlags.serialize(); + BOOST_CHECK_EQUAL(serialized, "5"); // 5 in binary is 0101, meaning Bit1 and Bit3 are set. + + // Deserialize back into a flag set + EFlags deserializedFlags; + deserializedFlags.deserialize(serialized); + BOOST_CHECK(deserializedFlags == serializedFlags); // Ensure the deserialized flags match the original + } + + // Test with an empty flag set + { + EFlags emptyFlags; + std::string serialized = emptyFlags.serialize(); + BOOST_CHECK_EQUAL(serialized, "0"); + + EFlags deserialized; + deserialized.deserialize(serialized); + BOOST_CHECK(deserialized == emptyFlags); + + // Test with all flags set + EFlags allFlags(EFlags::All); + serialized = allFlags.serialize(); + BOOST_CHECK_EQUAL(serialized, std::to_string(EFlags::All)); + + deserialized.deserialize(serialized); + BOOST_CHECK(deserialized == allFlags); + } + + // check throw deserializng out of range + { + EFlags flag; + std::string str = "999999"; + BOOST_CHECK_THROW(flag.deserialize(str), std::out_of_range); + } + + // Create two flag sets + { + EFlags flags1{TestEnum::Bit1, TestEnum::Bit2}; + EFlags flags2{TestEnum::Bit3, TestEnum::Bit4}; + + // Perform a union operation + EFlags unionFlags = flags1.union_with(flags2); + BOOST_CHECK(unionFlags.test(TestEnum::Bit1)); + BOOST_CHECK(unionFlags.test(TestEnum::Bit2)); + BOOST_CHECK(unionFlags.test(TestEnum::Bit3)); + BOOST_CHECK(unionFlags.test(TestEnum::Bit4)); + BOOST_CHECK_EQUAL(unionFlags.value(), 15); // 1111 in binary + } + + // Create two overlapping flag sets + { + EFlags flags3{TestEnum::Bit1, TestEnum::Bit2, TestEnum::Bit3}; + EFlags flags4{TestEnum::Bit2, TestEnum::Bit3, TestEnum::Bit4}; + + // test xor + auto flagsXOR = flags3 ^ flags4; + BOOST_CHECK(flagsXOR.test(TestEnum::Bit1, TestEnum::Bit4)); + + // test and + auto flagsAND = flags3 & flags4; + BOOST_CHECK(flagsAND.test(TestEnum::Bit2, TestEnum::Bit3)); + + // Perform an intersection operation + EFlags intersectionFlags = flags3.intersection_with(flags4); + BOOST_CHECK(intersectionFlags.test(TestEnum::Bit2)); + BOOST_CHECK(intersectionFlags.test(TestEnum::Bit3)); + BOOST_CHECK(!intersectionFlags.test(TestEnum::Bit1)); + BOOST_CHECK(!intersectionFlags.test(TestEnum::Bit4)); + BOOST_CHECK_EQUAL(intersectionFlags.value(), 6); // 0110 in binary + } + + { + // Check special flag names. + EFlags flag("all"); + BOOST_CHECK(flag.all()); + flag.set("none"); + BOOST_CHECK(!flag.any()); + } + + { + // Create two flag sets + EFlags flags1{TestEnum::Bit1, TestEnum::Bit2, TestEnum::Bit3}; + EFlags flags2{TestEnum::Bit2, TestEnum::Bit3}; + + // Check containment + BOOST_CHECK(flags1.contains(flags2)); // flags1 contains all flags in flags2 + BOOST_CHECK(!flags2.contains(flags1)); // flags2 does not contain all flags in flags1 + + // Test with disjoint sets + EFlags flags3{TestEnum::Bit4}; + BOOST_CHECK(!flags1.contains(flags3)); // flags1 does not contain flags3 + } + + { + // Test compilation using an enum with more than 32 bits + // Also tests space delimiter and construction from string. + o2::utils::EnumFlags test("Bit32 Bit34"); + BOOST_CHECK(test.test(TestEnumLong::Bit32, TestEnumLong::Bit34)); + BOOST_CHECK(!test.test(TestEnumLong::Bit1, TestEnumLong::Bit23)); + } +} + +BOOST_AUTO_TEST_CASE(Flags_case_insensitive_test) +{ + using EFlags = o2::utils::EnumFlags; + + // Test case-insensitive flag names + { + EFlags flags("bit1"); // lowercase + BOOST_CHECK(flags.test(TestEnum::Bit1)); + BOOST_CHECK(!flags.test(TestEnum::Bit2)); + } + + { + EFlags flags("BIT2"); // uppercase + BOOST_CHECK(flags.test(TestEnum::Bit2)); + BOOST_CHECK(!flags.test(TestEnum::Bit1)); + } + + { + EFlags flags("BiT3"); // mixed case + BOOST_CHECK(flags.test(TestEnum::Bit3)); + } + + { + EFlags flags("bit1|BIT2|BiT3"); // mixed case with delimiter + BOOST_CHECK(flags.test(TestEnum::Bit1)); + BOOST_CHECK(flags.test(TestEnum::Bit2)); + BOOST_CHECK(flags.test(TestEnum::Bit3)); + } + + // Test special keywords case-insensitive + { + EFlags flags("ALL"); + BOOST_CHECK(flags.all()); + } + + { + EFlags flags("None"); + BOOST_CHECK(!flags.any()); + } +} + +BOOST_AUTO_TEST_CASE(Flags_error_recovery_test) +{ + using EFlags = o2::utils::EnumFlags; + + // Test that previous state is restored on exception + { + EFlags flags({TestEnum::Bit1, TestEnum::Bit2}); + auto previousValue = flags.value(); + + // Try to set with invalid string + BOOST_CHECK_THROW(flags.set("InvalidFlag"), std::invalid_argument); + + // Verify state was restored + BOOST_CHECK_EQUAL(flags.value(), previousValue); + BOOST_CHECK(flags.test(TestEnum::Bit1)); + BOOST_CHECK(flags.test(TestEnum::Bit2)); + } + + { + EFlags flags({TestEnum::Bit3, TestEnum::Bit4}); + auto previousValue = flags.value(); + + // Try to set with out-of-range value + BOOST_CHECK_THROW(flags.set("999999", 10), std::out_of_range); + + // Verify state was restored + BOOST_CHECK_EQUAL(flags.value(), previousValue); + BOOST_CHECK(flags.test(TestEnum::Bit3)); + BOOST_CHECK(flags.test(TestEnum::Bit4)); + } + + { + EFlags flags(TestEnum::Bit5VeryLongName); + auto previousValue = flags.value(); + + // Try to set with invalid binary string + BOOST_CHECK_THROW(flags.set("10102", 2), std::invalid_argument); + + // Verify state was restored + BOOST_CHECK_EQUAL(flags.value(), previousValue); + BOOST_CHECK(flags.test(TestEnum::Bit5VeryLongName)); + } +} + +BOOST_AUTO_TEST_CASE(Flags_whitespace_handling_test) +{ + using EFlags = o2::utils::EnumFlags; + + // Test leading/trailing whitespace + { + EFlags flags(" Bit1 "); + BOOST_CHECK(flags.test(TestEnum::Bit1)); + } + + { + EFlags flags(" Bit1 | Bit2 "); + BOOST_CHECK(flags.test(TestEnum::Bit1)); + BOOST_CHECK(flags.test(TestEnum::Bit2)); + } + + // Test excessive whitespace between flags + { + EFlags flags("Bit1 | Bit3"); + BOOST_CHECK(flags.test(TestEnum::Bit1)); + BOOST_CHECK(flags.test(TestEnum::Bit3)); + BOOST_CHECK(!flags.test(TestEnum::Bit2)); + } + + // Test tabs and other whitespace (should work with space delimiter) + { + EFlags flags("Bit1 Bit2 Bit3"); + BOOST_CHECK(flags.test(TestEnum::Bit1)); + BOOST_CHECK(flags.test(TestEnum::Bit2)); + BOOST_CHECK(flags.test(TestEnum::Bit3)); + } +} + +BOOST_AUTO_TEST_CASE(Flags_count_bits_test) +{ + using EFlags = o2::utils::EnumFlags; + + // Test counting set bits + { + EFlags flags; + BOOST_CHECK_EQUAL(flags.count(), 0); + } + + { + EFlags flags(TestEnum::Bit1); + BOOST_CHECK_EQUAL(flags.count(), 1); + } + + { + EFlags flags({TestEnum::Bit1, TestEnum::Bit2}); + BOOST_CHECK_EQUAL(flags.count(), 2); + } + + { + EFlags flags({TestEnum::Bit1, TestEnum::Bit2, TestEnum::Bit3, TestEnum::Bit4}); + BOOST_CHECK_EQUAL(flags.count(), 4); + } + + { + EFlags flags(EFlags::All); + BOOST_CHECK_EQUAL(flags.count(), 5); // TestEnum has 5 members + } + + // Test count after operations + { + EFlags flags({TestEnum::Bit1, TestEnum::Bit2, TestEnum::Bit3}); + BOOST_CHECK_EQUAL(flags.count(), 3); + + flags.reset(TestEnum::Bit2); + BOOST_CHECK_EQUAL(flags.count(), 2); + + flags.set(TestEnum::Bit4); + BOOST_CHECK_EQUAL(flags.count(), 3); + + flags.toggle(TestEnum::Bit1); + BOOST_CHECK_EQUAL(flags.count(), 2); + } +} + +BOOST_AUTO_TEST_CASE(Flags_mixed_delimiter_validation_test) +{ + using EFlags = o2::utils::EnumFlags; + + // Test that mixed delimiters throw an error + { + BOOST_CHECK_THROW(EFlags("Bit1|Bit2,Bit3"), std::invalid_argument); + } + + { + BOOST_CHECK_THROW(EFlags("Bit1;Bit2|Bit3"), std::invalid_argument); + } + + { + BOOST_CHECK_THROW(EFlags("Bit1,Bit2;Bit3"), std::invalid_argument); + } + + { + BOOST_CHECK_THROW(EFlags("Bit1|Bit2,Bit3;Bit4"), std::invalid_argument); + } + + // Test that single delimiter types work + { + EFlags flags1("Bit1|Bit2|Bit3"); + BOOST_CHECK_EQUAL(flags1.count(), 3); + } + + { + EFlags flags2("Bit1,Bit2,Bit3"); + BOOST_CHECK_EQUAL(flags2.count(), 3); + } + + { + EFlags flags3("Bit1;Bit2;Bit3"); + BOOST_CHECK_EQUAL(flags3.count(), 3); + } +} + +BOOST_AUTO_TEST_CASE(Flags_empty_and_edge_cases_test) +{ + using EFlags = o2::utils::EnumFlags; + + // Test empty string + { + EFlags flags({TestEnum::Bit1, TestEnum::Bit2}); + flags.set(""); // Should be no-op + BOOST_CHECK(flags.test(TestEnum::Bit1)); + BOOST_CHECK(flags.test(TestEnum::Bit2)); + } + + // Test with only whitespace + { + EFlags flags({TestEnum::Bit1}); + flags.set(" "); // Should result in empty after tokenization + // Depending on implementation, this might clear or throw + // Adjust expectation based on actual behavior + } + + // Test duplicate flags (should work, setting same bit twice is idempotent) + { + EFlags flags("Bit1|Bit1|Bit1"); + BOOST_CHECK(flags.test(TestEnum::Bit1)); + BOOST_CHECK_EQUAL(flags.count(), 1); + } + + // Test scoped and unscoped mixed + { + EFlags flags("Bit1|TestEnum::Bit2"); + BOOST_CHECK(flags.test(TestEnum::Bit1)); + BOOST_CHECK(flags.test(TestEnum::Bit2)); + } +} + +BOOST_AUTO_TEST_CASE(Flags_binary_decimal_parsing_test) +{ + using EFlags = o2::utils::EnumFlags; + + // Test binary parsing + { + EFlags flags("101", 2); + BOOST_CHECK(flags.test(TestEnum::Bit1)); // bit 0 + BOOST_CHECK(!flags.test(TestEnum::Bit2)); // bit 1 + BOOST_CHECK(flags.test(TestEnum::Bit3)); // bit 2 + } + + // Test decimal parsing + { + EFlags flags("7", 10); // 7 = 0b111 + BOOST_CHECK(flags.test(TestEnum::Bit1)); + BOOST_CHECK(flags.test(TestEnum::Bit2)); + BOOST_CHECK(flags.test(TestEnum::Bit3)); + BOOST_CHECK(!flags.test(TestEnum::Bit4)); + } + + // Test hexadecimal parsing + { + EFlags flags("F", 16); // 15 = 0b1111 + BOOST_CHECK(flags.test(TestEnum::Bit1)); + BOOST_CHECK(flags.test(TestEnum::Bit2)); + BOOST_CHECK(flags.test(TestEnum::Bit3)); + BOOST_CHECK(flags.test(TestEnum::Bit4)); + BOOST_CHECK(!flags.test(TestEnum::Bit5VeryLongName)); + } + + // Test hexadecimal with 0x prefix + { + EFlags flags("0xA", 16); // 10 = 0b1010 + BOOST_CHECK(!flags.test(TestEnum::Bit1)); + BOOST_CHECK(flags.test(TestEnum::Bit2)); + BOOST_CHECK(!flags.test(TestEnum::Bit3)); + BOOST_CHECK(flags.test(TestEnum::Bit4)); + } + + // Test hexadecimal with 0X prefix (uppercase) + { + EFlags flags("0X1F", 16); // 31 = all 5 bits + BOOST_CHECK(flags.all()); + } + + // Test lowercase hex digits + { + EFlags flags("0xa", 16); + BOOST_CHECK_EQUAL(flags.value(), 10); + } + + // Test thros + { + BOOST_CHECK_THROW(EFlags("0xAbCd", 16), std::out_of_range); + } + + // Test invalid binary string (contains 2) + { + BOOST_CHECK_THROW(EFlags("1012", 2), std::invalid_argument); + } + + // Test out of range for base + { + BOOST_CHECK_THROW(EFlags("100000", 2), std::out_of_range); + } +} + +BOOST_AUTO_TEST_CASE(Flags_operator_bool_test) +{ + using EFlags = o2::utils::EnumFlags; + + // Test explicit bool conversion + { + EFlags empty; + BOOST_CHECK(!static_cast(empty)); + } + + { + EFlags withFlag(TestEnum::Bit1); + BOOST_CHECK(static_cast(withFlag)); + } + + // Test in conditional + { + EFlags flags; + if (flags) { + BOOST_FAIL("Empty flags should be false"); + } + + flags.set(TestEnum::Bit1); + if (!flags) { + BOOST_FAIL("Non-empty flags should be true"); + } + } +} diff --git a/Common/Utils/test/testMemFileHelper.cxx b/Common/Utils/test/testMemFileHelper.cxx index 2a55d67b9884a..79fa7e86f96e9 100644 --- a/Common/Utils/test/testMemFileHelper.cxx +++ b/Common/Utils/test/testMemFileHelper.cxx @@ -40,3 +40,35 @@ BOOST_AUTO_TEST_CASE(test_memfile_helper) BOOST_CHECK(rvec); BOOST_CHECK(*rvec == vec); } + +#include "CommonUtils/FileSystemUtils.h" +#include + +BOOST_AUTO_TEST_CASE(test_expandenv) +{ + { + std::string noenv("simple_file.root"); + auto expandedFileName = o2::utils::expandShellVarsInFileName(noenv); + BOOST_CHECK(expandedFileName.size() > 0); + BOOST_CHECK_EQUAL(expandedFileName, noenv); + } + + { + std::string withenv("${PWD}/simple_file.root"); + auto expandedFileName = o2::utils::expandShellVarsInFileName(withenv); + BOOST_CHECK(expandedFileName.size() > 0); + } + + { + setenv("FOO_123", "BAR", 0); + std::string withenv("/tmp/${FOO_123}/simple_file.root"); + auto expandedFileName = o2::utils::expandShellVarsInFileName(withenv); + BOOST_CHECK_EQUAL(expandedFileName, "/tmp/BAR/simple_file.root"); + } + + { // what if the variable doesn't exist --> should return unmodified string + std::string withenv("/tmp/${FOO_DOESNOTEXIST}/simple_file.root"); + auto expandedFileName = o2::utils::expandShellVarsInFileName(withenv); + BOOST_CHECK_EQUAL(expandedFileName, withenv); + } +} \ No newline at end of file diff --git a/Common/Utils/test/testTreeStream.cxx b/Common/Utils/test/testTreeStream.cxx index 7424a84ef2c1e..2491fea7f6efd 100644 --- a/Common/Utils/test/testTreeStream.cxx +++ b/Common/Utils/test/testTreeStream.cxx @@ -22,7 +22,6 @@ #include "CommonUtils/TreeStreamRedirector.h" #include "CommonUtils/RootChain.h" #include "ReconstructionDataFormats/Track.h" -#include #include using namespace o2::utils; @@ -34,7 +33,6 @@ BOOST_AUTO_TEST_CASE(TreeStream_test) // Example test function to show functionality of TreeStreamRedirector // create the redirector associated with file (testredirector.root) - FairLogger* logger = FairLogger::GetLogger(); LOG(info) << "Testing TreeStream creation"; std::string outFName("testTreeStream.root"); @@ -55,12 +53,28 @@ BOOST_AUTO_TEST_CASE(TreeStream_test) tstStream << "TrackTreeR" << "id=" << i << "x=" << x << "track=" << trc << "\n"; } + + // test for c-arrays + int iArray[6] = {1, 2, 3, 4, 5, 6}; + float fArray[6] = {1.1f, 2.2f, 3.3f, 4.4f, 5.5f, 6.6f}; + for (int i{0}; i < nit; ++i) { + for (int j{0}; j < 6; ++j) { + iArray[j] += i; + fArray[j] += (float)i; + } + tstStream << "ArrayTree" + << "id=" << i + << "iArray[6]=" << iArray + << "fArray[6]=" << fArray + << "\n"; + } + // on destruction of tstTreem the trees will be stored, but we can also force it by tstStream.Close(); } // - LOG(info) << "Testing reading back tree maid by the TreeStream "; - // read back tracks + LOG(info) << "Testing reading back tree made by the TreeStream "; + // read back tracks and arrays { TFile inpf(outFName.data()); BOOST_CHECK(!inpf.IsZombie()); @@ -82,6 +96,27 @@ BOOST_AUTO_TEST_CASE(TreeStream_test) trc->printParam(); BOOST_CHECK(std::abs(x - trc->getX()) < 1e-4); } + + // check arrays + tree = (TTree*)inpf.GetObjectChecked("ArrayTree", "TTree"); + BOOST_CHECK(tree); + nent = tree->GetEntries(); + BOOST_CHECK(nent == nit); + int iArray[6]; + float fArray[6]; + BOOST_CHECK(!tree->SetBranchAddress("id", &id)); + BOOST_CHECK(!tree->SetBranchAddress("iArray", iArray)); + BOOST_CHECK(!tree->SetBranchAddress("fArray", fArray)); + for (int i = 0; i < nit; i++) { + BOOST_CHECK(tree->GetEntry(i) > 0); + BOOST_CHECK(id == i); + for (int j = 0; j < 6; j++) { + BOOST_CHECK(iArray[j] == (1 + j + i * (i + 1) / 2)); + } + for (int j = 0; j < 6; j++) { + BOOST_CHECK_CLOSE(fArray[j], (1.f + j + i * (i + 1) / 2.f + 0.1 * (j + 1)), 1e-5); + } + } } LOG(info) << "Testing loading tree via RootChain"; @@ -106,7 +141,6 @@ BOOST_AUTO_TEST_CASE(TreeStream_test) nit = 1000; BOOST_CHECK(UnitTestSparse(0.5, nit)); BOOST_CHECK(UnitTestSparse(0.1, nit)); - // } //_________________________________________________ diff --git a/DEBUGGING.md b/DEBUGGING.md new file mode 100644 index 0000000000000..c782b99987ef9 --- /dev/null +++ b/DEBUGGING.md @@ -0,0 +1,13 @@ + + +# How do I run in debug mode? + +By default, O2 builds with optimizations (`-O2`) turned on, while leaving debug symbols available. +This allows doing some simple set of debugging operations, e.g. getting a reasonable stacktrace, however it does not work when using a debugger. This results in the typical effect of single stepping in gdb / lldb jumping around the sourcecode. +In order to fix this you need to turn off the optimization and there are several ways you could do this, depending on how permanent you want the change to be. + +* Add `-DCMAKE_BUILD_TYPE=Debug` in `alidist/o2.sh` and then rebuild using aliBuild. This will be a permanent change, however it requires rebuilding everything. +* Change `sw/BUILD//O2-latest/O2/CMakeCache.txt` to have `CMAKE_BUILD_TYPE=Debug` and the type `ninja install` in the same folder. This will be undone by the next time you run aliBuild, however it has the advantage that the ninja command can be targeted to a specific subsystem, e.g. `ninja Framework/install`. Notice also that by default `-Og` is used, so you might have to change `CMAKE_CXX_FLAGS_DEBUG` in the same file to use `-O0`. +* Change `sw/BUILD//O2-latest/O2/build.ninja` to use `-Og` or `-Os`, for the specific targets you are interested in. This in the most fine grained option. diff --git a/DataFormats/Calibration/include/DataFormatsCalibration/MeanVertexObject.h b/DataFormats/Calibration/include/DataFormatsCalibration/MeanVertexObject.h index bbda9be7dccc2..46c885ae1a18d 100644 --- a/DataFormats/Calibration/include/DataFormatsCalibration/MeanVertexObject.h +++ b/DataFormats/Calibration/include/DataFormatsCalibration/MeanVertexObject.h @@ -13,8 +13,6 @@ #define MEAN_VERTEX_OBJECT_H_ #include -#include "Rtypes.h" - #include "Framework/Logger.h" #include "ReconstructionDataFormats/Vertex.h" @@ -28,24 +26,16 @@ class MeanVertexObject : public VertexBase public: MeanVertexObject(float x, float y, float z, float sigmax, float sigmay, float sigmaz, float slopeX, float slopeY) { - gpu::gpustd::array cov; - cov[CovElems::kCovXX] = sigmax; - cov[CovElems::kCovYY] = sigmay; - cov[CovElems::kCovZZ] = sigmaz; setXYZ(x, y, z); - setCov(cov); + setSigma({sigmax, sigmay, sigmaz}); mSlopeX = slopeX; mSlopeY = slopeY; } MeanVertexObject(std::array pos, std::array sigma, float slopeX, float slopeY) { math_utils::Point3D p(pos[0], pos[1], pos[2]); - gpu::gpustd::array cov; - cov[CovElems::kCovXX] = sigma[0]; - cov[CovElems::kCovYY] = sigma[1]; - cov[CovElems::kCovZZ] = sigma[2]; setPos(p); - setCov(cov); + setSigma(sigma); mSlopeX = slopeX; mSlopeY = slopeY; } @@ -53,46 +43,51 @@ class MeanVertexObject : public VertexBase ~MeanVertexObject() = default; MeanVertexObject(const MeanVertexObject& other) = default; MeanVertexObject(MeanVertexObject&& other) = default; - MeanVertexObject& operator=(MeanVertexObject& other) = default; + MeanVertexObject& operator=(const MeanVertexObject& other) = default; MeanVertexObject& operator=(MeanVertexObject&& other) = default; void set(int icoord, float val); void setSigma(int icoord, float val); - void setSigmaX(float val) { setSigmaX2(val); } - void setSigmaY(float val) { setSigmaY2(val); } - void setSigmaZ(float val) { setSigmaZ2(val); } void setSigma(std::array val) { - setSigmaX2(val[0]); - setSigmaY2(val[1]); - setSigmaZ2(val[2]); + setSigmaX(val[0]); + setSigmaY(val[1]); + setSigmaZ(val[2]); } void setSlopeX(float val) { mSlopeX = val; } void setSlopeY(float val) { mSlopeY = val; } math_utils::Point3D& getPos() { return getXYZ(); } math_utils::Point3D getPos() const { return getXYZ(); } - float getSigmaX() const { return getSigmaX2(); } - float getSigmaY() const { return getSigmaY2(); } - float getSigmaZ() const { return getSigmaZ2(); } - const gpu::gpustd::array& getSigma() const { return getCov(); } float getSlopeX() const { return mSlopeX; } float getSlopeY() const { return mSlopeY; } - float getXAtZ(float z) { return getX() + mSlopeX * (z - getZ()); } - float getYAtZ(float z) { return getY() + mSlopeY * (z - getZ()); } + float getXAtZ(float z) const { return getX() + mSlopeX * (z - getZ()); } + float getYAtZ(float z) const { return getY() + mSlopeY * (z - getZ()); } void print() const; std::string asString() const; - VertexBase getMeanVertex(float z) + /// sample a vertex from the MeanVertex parameters + math_utils::Point3D sample() const; + + VertexBase getMeanVertex(float z) const { + // set z-dependent x,z, assuming that the cov.matrix is already set VertexBase v = *this; v.setXYZ(getXAtZ(z), getYAtZ(z), z); return v; } + void setMeanXYVertexAtZ(VertexBase& v, float z) const + { + float dz = z - getZ(); + v.setX(getX() + mSlopeX * dz); + v.setY(getY() + mSlopeY * dz); + v.setZ(z); + } + const VertexBase& getMeanVertex() const { return (const VertexBase&)(*this); @@ -102,7 +97,7 @@ class MeanVertexObject : public VertexBase float mSlopeX{0.f}; // slope of x = f(z) float mSlopeY{0.f}; // slope of y = f(z) - ClassDefNV(MeanVertexObject, 1); + ClassDefNV(MeanVertexObject, 2); }; std::ostream& operator<<(std::ostream& os, const o2::dataformats::MeanVertexObject& o); diff --git a/DataFormats/Calibration/src/MeanVertexObject.cxx b/DataFormats/Calibration/src/MeanVertexObject.cxx index e9f1be7a15e74..167be39b5e1bd 100644 --- a/DataFormats/Calibration/src/MeanVertexObject.cxx +++ b/DataFormats/Calibration/src/MeanVertexObject.cxx @@ -10,6 +10,7 @@ // or submit itself to any jurisdiction. #include "DataFormatsCalibration/MeanVertexObject.h" +#include "TRandom.h" namespace o2 { @@ -44,7 +45,7 @@ void MeanVertexObject::setSigma(int icoord, float val) std::string MeanVertexObject::asString() const { - return fmt::format("Slopes {{{:+.4e},{:+.4e}}}", mSlopeX, mSlopeY); + return VertexBase::asString() + fmt::format(" Slopes {{{:+.4e},{:+.4e}}}", mSlopeX, mSlopeY); } std::ostream& operator<<(std::ostream& os, const o2::dataformats::MeanVertexObject& o) @@ -56,9 +57,18 @@ std::ostream& operator<<(std::ostream& os, const o2::dataformats::MeanVertexObje void MeanVertexObject::print() const { - VertexBase::print(); std::cout << *this << std::endl; } +math_utils::Point3D MeanVertexObject::sample() const +{ + // this assumes gaussian sampling + // first determine z; then x and y + const auto z = gRandom->Gaus(getZ(), getSigmaZ()); + const auto x = gRandom->Gaus(getXAtZ(z), getSigmaX()); + const auto y = gRandom->Gaus(getYAtZ(z), getSigmaY()); + return math_utils::Point3D(x, y, z); +} + } // namespace dataformats } // namespace o2 diff --git a/DataFormats/Detectors/CMakeLists.txt b/DataFormats/Detectors/CMakeLists.txt index a93a1c8c11da0..3d322776a3890 100644 --- a/DataFormats/Detectors/CMakeLists.txt +++ b/DataFormats/Detectors/CMakeLists.txt @@ -17,6 +17,7 @@ add_subdirectory(ITSMFT) add_subdirectory(MUON) add_subdirectory(TOF) add_subdirectory(FIT) +add_subdirectory(FOCAL) add_subdirectory(HMPID) add_subdirectory(EMCAL) add_subdirectory(ZDC) @@ -26,6 +27,7 @@ add_subdirectory(CPV) add_subdirectory(CTP) add_subdirectory(GlobalTracking) add_subdirectory(DCS) + if(ENABLE_UPGRADES) add_subdirectory(Upgrades) else() diff --git a/DataFormats/Detectors/CPV/include/DataFormatsCPV/RawFormats.h b/DataFormats/Detectors/CPV/include/DataFormatsCPV/RawFormats.h index 4dc033e42544d..2f8a88072a43d 100644 --- a/DataFormats/Detectors/CPV/include/DataFormatsCPV/RawFormats.h +++ b/DataFormats/Detectors/CPV/include/DataFormatsCPV/RawFormats.h @@ -21,7 +21,7 @@ namespace o2 namespace cpv { -//Pack information into 24 bit words +// Pack information into 24 bit words union PadWord { uint32_t mDataWord; struct { @@ -39,18 +39,17 @@ class CpvWord public: CpvWord() = default; CpvWord(std::vector::const_iterator b, std::vector::const_iterator e) - { //Reading - //resposibility of coller to esure that - //array will not end while reading - for (int i = 0; i < 16 && b != e; i++, b++) { + { // Reading + // resposibility of coller to esure that + // array will not end while reading + for (int i = 0; i < 10 && b != e; i++, b++) { mBytes[i] = *b; } } ~CpvWord() = default; bool isOK() const { - return (mBytes[9] < static_cast(24)) && - (mBytes[15] == 0) && (mBytes[14] == 0) && (mBytes[13] == 0) && (mBytes[12] == 0) && (mBytes[11] == 0) && (mBytes[10] == 0); + return (mBytes[9] < static_cast(24)); } short ccId() const { return short(mBytes[9]); } uint32_t cpvPadWord(int i) const @@ -63,7 +62,7 @@ class CpvWord } public: - unsigned char mBytes[16] = {0}; + unsigned char mBytes[10] = {0}; }; class CpvHeader @@ -71,41 +70,38 @@ class CpvHeader public: CpvHeader() = default; CpvHeader(std::vector::const_iterator b, std::vector::const_iterator e) - { //reading header from file - for (int i = 0; i < 16 && b != e; i++, b++) { //read up to 16 mBytes + { // reading header from file + for (int i = 0; i < 10 && b != e; i++, b++) { // read up to 10 mBytes mBytes[i] = *b; } } CpvHeader(InteractionRecord orbitBC, bool isNoDataExpected, bool isDataContinued) - { //writing header - //header is 128-bit word. - //|127-120|119-112|111-104|103-96|95-88 |87-80 |79-72|71-64|63-56|55-48|47-40|39-32|31-24|23-16|15-8 |7-0 | + { // writing header + // header is 128-bit word. + // |127-120|119-112|111-104|103-96|95-88 |87-80 |79-72|71-64|63-56|55-48|47-40|39-32|31-24|23-16|15-8 |7-0 | // byte15 byte14 byte13 byte12 byte11 byte10 byte9 byte8 byte7 byte6 byte5 byte4 byte3 byte2 byte1 byte0 - //byte = |76543210| - mBytes[0] = (0x010 & 0x0ff); //bits 11 - 0 trigger id (0x010 = physics trigger) - mBytes[1] = ((0x010 & 0xf00) >> 8) //bits 11 - 0 trigger id (0x010 = physics trigger) - + 0b00100000 * isNoDataExpected + 0b0100000 * isDataContinued; //bit 13 (no data for this trigger) + bit 14 (payload continues from previous page) - mBytes[2] = (orbitBC.bc & 0x00ff); //bits 27 - 16 bunch crossing - mBytes[3] = (orbitBC.bc & 0x0f00) >> 8; //bits 27 - 16 bunch crossing - mBytes[4] = (orbitBC.orbit & 0x000000ff); //bits 63 - 32 orbit - mBytes[5] = (orbitBC.orbit & 0x0000ff00) >> 8; //bits 63 - 32 orbit - mBytes[6] = (orbitBC.orbit & 0x00ff0000) >> 16; //bits 63 - 32 orbit - mBytes[7] = (orbitBC.orbit & 0xff000000) >> 24; //bits 63 - 32 orbit - mBytes[8] = 0x00; //bits 64-71 reserved - mBytes[9] = 0xe0; //word ID of cpv header (bits 79 - 72) - for (int i = 10; i < 16; i++) { - mBytes[i] = 0; //bits 127-80 must be zeros - } + // byte = |76543210| + mBytes[0] = (0x010 & 0x0ff); // bits 11 - 0 trigger id (0x010 = physics trigger) + mBytes[1] = ((0x010 & 0xf00) >> 8) // bits 11 - 0 trigger id (0x010 = physics trigger) + + 0b00100000 * isNoDataExpected + 0b0100000 * isDataContinued; // bit 13 (no data for this trigger) + bit 14 (payload continues from previous page) + mBytes[2] = (orbitBC.bc & 0x00ff); // bits 27 - 16 bunch crossing + mBytes[3] = (orbitBC.bc & 0x0f00) >> 8; // bits 27 - 16 bunch crossing + mBytes[4] = (orbitBC.orbit & 0x000000ff); // bits 63 - 32 orbit + mBytes[5] = (orbitBC.orbit & 0x0000ff00) >> 8; // bits 63 - 32 orbit + mBytes[6] = (orbitBC.orbit & 0x00ff0000) >> 16; // bits 63 - 32 orbit + mBytes[7] = (orbitBC.orbit & 0xff000000) >> 24; // bits 63 - 32 orbit + mBytes[8] = 0x00; // bits 64-71 reserved + mBytes[9] = 0xe0; // word ID of cpv header (bits 79 - 72) } ~CpvHeader() = default; - bool isOK() const { return (mBytes[9] == 0xe0) && (mBytes[10] == 0) && (mBytes[11] == 0) && (mBytes[12] == 0) && (mBytes[13] == 0) && (mBytes[14] == 0) && (mBytes[15] == 0); } + bool isOK() const { return (mBytes[9] == 0xe0); } bool isNoDataExpected() const { return mBytes[1] & 0b00100000; } bool isDataContinued() const { return mBytes[1] & 0b0100000; } uint16_t bc() const { return static_cast(mBytes[2]) + static_cast((mBytes[3] & 0x0f) << 8); } uint32_t orbit() const { return mBytes[4] + (mBytes[5] << 8) + (mBytes[6] << 16) + (mBytes[7] << 24); } public: - unsigned char mBytes[16] = {0}; //0 - 127 bits (16 bytes) + unsigned char mBytes[10] = {0}; // 0 - 79 bits (10 bytes) }; class CpvTrailer @@ -113,34 +109,32 @@ class CpvTrailer public: CpvTrailer() = default; CpvTrailer(std::vector::const_iterator b, std::vector::const_iterator e) - { //reading - for (int i = 0; i < 16 && b != e; i++, b++) { //read up to 16 mBytes + { // reading + for (int i = 0; i < 10 && b != e; i++, b++) { // read up to 10 mBytes mBytes[i] = *b; } } CpvTrailer(unsigned short wordCounter, uint16_t bunchCrossing, bool isAllDataSent) - { //writing - mBytes[0] = bunchCrossing & 0x00ff; //bits 11 - 0 bunch crossing - mBytes[1] = ((bunchCrossing & 0x0f00) >> 8) //bits 11 - 0 bunch crossing - + ((wordCounter & 0x0f) << 4); //bits 20 - 12 wordCounter - mBytes[2] = (wordCounter & 0b111110000) >> 4; //bits 20 - 12 wordCounter + { // writing + mBytes[0] = bunchCrossing & 0x00ff; // bits 11 - 0 bunch crossing + mBytes[1] = ((bunchCrossing & 0x0f00) >> 8) // bits 11 - 0 bunch crossing + + ((wordCounter & 0x0f) << 4); // bits 20 - 12 wordCounter + mBytes[2] = (wordCounter & 0b1111110000) >> 4; // bits 21 - 12 wordCounter + for (int i = 3; i < 8; i++) { - mBytes[i] = 0; //bits 70 - 21 reserved - } - mBytes[8] = isAllDataSent * 0b10000000; //bit 71 all data is sent for current trigger - mBytes[9] = char(0xf0); //word ID of cpv trailer - for (int i = 10; i < 16; i++) { - mBytes[i] = 0; + mBytes[i] = 0; // bits 70 - 22 reserved } + mBytes[8] = isAllDataSent * 0b10000000; // bit 71 all data is sent for current trigger + mBytes[9] = char(0xf0); // word ID of cpv trailer } ~CpvTrailer() = default; - bool isOK() const { return (mBytes[9] == 0xf0) && (mBytes[10] == 0) && (mBytes[11] == 0) && (mBytes[12] == 0) && (mBytes[13] == 0) && (mBytes[14] == 0) && (mBytes[15] == 0); } - uint16_t wordCounter() const { return (mBytes[1] >> 4) + ((mBytes[2] & 0b00011111) << 4); } + bool isOK() const { return (mBytes[9] == 0xf0); } + uint16_t wordCounter() const { return (mBytes[1] >> 4) + ((mBytes[2] & 0b00111111) << 4); } bool isAllDataSent() const { return (mBytes[8] & 0b10000000); } uint16_t bc() const { return mBytes[0] + ((mBytes[1] & 0x0f) << 8); } public: - unsigned char mBytes[16] = {0}; + unsigned char mBytes[10] = {0}; }; } // namespace cpv diff --git a/DataFormats/Detectors/CPV/src/BadChannelMap.cxx b/DataFormats/Detectors/CPV/src/BadChannelMap.cxx index 3be0faa577cb6..7e16b603f3c64 100644 --- a/DataFormats/Detectors/CPV/src/BadChannelMap.cxx +++ b/DataFormats/Detectors/CPV/src/BadChannelMap.cxx @@ -12,7 +12,7 @@ #include "CPVBase/Geometry.h" #include "DataFormatsCPV/BadChannelMap.h" -#include "FairLogger.h" +#include #include diff --git a/DataFormats/Detectors/CPV/src/CalibParams.cxx b/DataFormats/Detectors/CPV/src/CalibParams.cxx index 31cd4a7230367..e6877f34b12d2 100644 --- a/DataFormats/Detectors/CPV/src/CalibParams.cxx +++ b/DataFormats/Detectors/CPV/src/CalibParams.cxx @@ -12,7 +12,7 @@ #include "DataFormatsCPV/CalibParams.h" #include "CPVBase/Geometry.h" -#include "FairLogger.h" +#include #include diff --git a/DataFormats/Detectors/CPV/src/Digit.cxx b/DataFormats/Detectors/CPV/src/Digit.cxx index 51197ae0b1fe4..7ab64ced0a1f6 100644 --- a/DataFormats/Detectors/CPV/src/Digit.cxx +++ b/DataFormats/Detectors/CPV/src/Digit.cxx @@ -8,7 +8,7 @@ // In applying this license CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. -#include "FairLogger.h" +#include #include "DataFormatsCPV/Digit.h" #include "DataFormatsCPV/Hit.h" diff --git a/DataFormats/Detectors/CPV/src/Pedestals.cxx b/DataFormats/Detectors/CPV/src/Pedestals.cxx index 2a95513d65260..13ef127e73fcc 100644 --- a/DataFormats/Detectors/CPV/src/Pedestals.cxx +++ b/DataFormats/Detectors/CPV/src/Pedestals.cxx @@ -10,7 +10,7 @@ // or submit itself to any jurisdiction. #include "DataFormatsCPV/Pedestals.h" -#include "FairLogger.h" +#include #include #include diff --git a/DataFormats/Detectors/CTP/CMakeLists.txt b/DataFormats/Detectors/CTP/CMakeLists.txt index 2e1c84ffed9c3..37c5af7b07b6e 100644 --- a/DataFormats/Detectors/CTP/CMakeLists.txt +++ b/DataFormats/Detectors/CTP/CMakeLists.txt @@ -8,22 +8,26 @@ # In applying this license CERN does not waive the privileges and immunities # granted to it by virtue of its status as an Intergovernmental Organization # or submit itself to any jurisdiction. - o2_add_library(DataFormatsCTP SOURCES src/Digits.cxx src/Configuration.cxx src/Scalers.cxx src/CTF.cxx src/TriggerOffsetsParam.cxx + src/LumiInfo.cxx + src/CTPRateFetcher.cxx PUBLIC_LINK_LIBRARIES O2::CommonDataFormat O2::Headers - O2::SimulationDataFormat O2::CommonUtils + O2::DetectorsCommonDataFormats + O2::DataFormatsParameters O2::CommonConstants) o2_target_root_dictionary(DataFormatsCTP HEADERS include/DataFormatsCTP/Digits.h + include/DataFormatsCTP/CTF.h include/DataFormatsCTP/Configuration.h include/DataFormatsCTP/Scalers.h - include/DataFormatsCTP/CTF.h + include/DataFormatsCTP/LumiInfo.h + include/DataFormatsCTP/CTPRateFetcher.h include/DataFormatsCTP/TriggerOffsetsParam.h) diff --git a/DataFormats/Detectors/CTP/include/DataFormatsCTP/CTF.h b/DataFormats/Detectors/CTP/include/DataFormatsCTP/CTF.h index 3e42d9fe315ad..3635b8ede44db 100644 --- a/DataFormats/Detectors/CTP/include/DataFormatsCTP/CTF.h +++ b/DataFormats/Detectors/CTP/include/DataFormatsCTP/CTF.h @@ -27,11 +27,18 @@ namespace ctp /// Header for a single CTF struct CTFHeader : public o2::ctf::CTFDictHeader { - uint32_t nTriggers = 0; /// number of triggers - uint32_t firstOrbit = 0; /// orbit of 1st trigger - uint16_t firstBC = 0; /// bc of 1st trigger - - ClassDefNV(CTFHeader, 1); + uint64_t lumiCounts = 0; /// FT0 Luminosity counts moving average over lumiNHBFs orbits + uint64_t lumiCountsFV0 = 0; /// FV0 Luminosity counts moving average over lumiNHBFs orbits + uint32_t lumiNHBFs = 0; /// Number of HBFs over which lumi is integrated + uint32_t lumiNHBFsFV0 = 0; /// Number of FV0 HBFs over which lumi is integrated + uint32_t lumiOrbit = 0; /// 1st orbit of TF where lumi was updated, can be compared with firstOrbit + uint32_t nTriggers = 0; /// number of triggers + uint32_t firstOrbit = 0; /// orbit of 1st trigger + uint16_t firstBC = 0; /// bc of 1st trigger + uint16_t inp1 = 0; /// lumiCounts input ID + uint16_t inp2 = 0; /// lumiCountsFV0 input ID + + ClassDefNV(CTFHeader, 5); }; /// wrapper for the Entropy-encoded trigger inputs and classes of the TF @@ -43,7 +50,7 @@ struct CTF : public o2::ctf::EncodedBlocks { BLC_bytesInput, // bytes of the CTPInputMask bitset (6 bytes from lowest to highest) BLC_bytesClass // bytes of the CTPClassMask bitset (8 bytes from lowest to highest) }; - ClassDefNV(CTF, 1); + ClassDefNV(CTF, 2); }; } // namespace ctp diff --git a/DataFormats/Detectors/CTP/include/DataFormatsCTP/CTPRateFetcher.h b/DataFormats/Detectors/CTP/include/DataFormatsCTP/CTPRateFetcher.h new file mode 100644 index 0000000000000..78c4245b16b20 --- /dev/null +++ b/DataFormats/Detectors/CTP/include/DataFormatsCTP/CTPRateFetcher.h @@ -0,0 +1,62 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef COMMON_CCDB_CTPRATEFETCHER_H_ +#define COMMON_CCDB_CTPRATEFETCHER_H_ + +#include + +#include "DataFormatsParameters/GRPLHCIFData.h" +#include "DataFormatsCTP/Configuration.h" +#include "DataFormatsCTP/Scalers.h" + +namespace o2::ccdb +{ +class BasicCCDBManager; +} + +namespace o2::ctp +{ + +class CTPRateFetcher +{ + public: + CTPRateFetcher() = default; + double fetch(o2::ccdb::BasicCCDBManager* ccdb, uint64_t timeStamp, int runNumber, const std::string sourceName); + double fetchNoPuCorr(o2::ccdb::BasicCCDBManager* ccdb, uint64_t timeStamp, int runNumber, const std::string sourceName); + void setupRun(int runNumber, o2::ccdb::BasicCCDBManager* ccdb, uint64_t timeStamp, bool initScalers); + void updateScalers(ctp::CTPRunScalers& scalers); + int getRates(std::array& rates, o2::ccdb::BasicCCDBManager* ccdb, int runNumber, const std::string sourceName); // rates at start,stop and middle of the run + double getLumi(o2::ccdb::BasicCCDBManager* ccdb, int runNumber, const std::string sourceName, int puCorr = 0); // total lumi for a run + double getLumiNoPuCorr(const std::string& classname, int type = 1); + double getLumiWPuCorr(const std::string& classname, int type = 1); + void setOrbit(bool orb) { mOrbit = orb; } // use orbit instead of time + void setOutsideLimits(bool qc) { mOutsideLimits = qc; } // return first/last rate of time outside of run + + private: + double fetchCTPratesInputs(uint64_t timeStamp, int input); + double fetchCTPratesClasses(uint64_t timeStamp, const std::string& className, int inputType = 1); + double fetchCTPratesInputsNoPuCorr(uint64_t timeStamp, int input); + double fetchCTPratesClassesNoPuCorr(uint64_t timeStamp, const std::string& className, int inputType = 1); + double getLumi(const std::string& classname, int type = 1, int puCorr = 0); + double pileUpCorrection(double rate); + int mRunNumber = -1; + bool mOutsideLimits = 0; + bool mOrbit = 0; + o2::ctp::CTPConfiguration mConfig{}; + o2::ctp::CTPRunScalers mScalers{}; + o2::parameters::GRPLHCIFData mLHCIFdata{}; + ClassDefNV(CTPRateFetcher, 1); +}; +} // namespace o2::ctp + + +#endif // COMMON_CCDB_CTPRATEFETCHER_H_ diff --git a/DataFormats/Detectors/CTP/include/DataFormatsCTP/Configuration.h b/DataFormats/Detectors/CTP/include/DataFormatsCTP/Configuration.h index a273754d411d5..ff1462084d53d 100644 --- a/DataFormats/Detectors/CTP/include/DataFormatsCTP/Configuration.h +++ b/DataFormats/Detectors/CTP/include/DataFormatsCTP/Configuration.h @@ -23,7 +23,7 @@ #include #include #include -#include +#include namespace o2 { namespace ctp @@ -32,14 +32,18 @@ namespace ctp const std::string CCDBPathCTPConfig = "CTP/Config/Config"; /// /// CTP Config items +/// +// Bunch Crossing (BC) mask struct BCMask { BCMask() = default; std::string name = ""; std::string mask = ""; std::bitset BCmask; + int setBCmask(std::vector& tokens); void printStream(std::ostream& stream) const; ClassDefNV(BCMask, 1); }; +/// CTP internal generator: 4 for L0 and 4 for LM levels struct CTPGenerator { static const std::set Generators; std::string name = ""; @@ -47,6 +51,8 @@ struct CTPGenerator { void printStream(std::ostream& stream) const; ClassDefNV(CTPGenerator, 1); }; +/// CTP inputs +/// Default input config is in CTPConfiguration struct CTPInput { const static std::map run2DetToRun3Det; CTPInput() = default; @@ -55,14 +61,15 @@ struct CTPInput { std::string name = ""; std::string level = ""; std::uint64_t inputMask = 0; - o2::detectors::DetID::ID detID = 13; + o2::detectors::DetID::ID detID = 16; // CTP bool neg = 1; - uint32_t getIndex() const { return ((inputMask > 1) ? 1 + log2(inputMask >> 1) : 0) + 1; } + uint32_t getIndex() const { return ((inputMask > 0) ? 1 + log2(inputMask) : 0xff); } std::string getInputDetName() const { return o2::detectors::DetID::getName(detID); } void setRun3DetName(std::string& run2Name); void printStream(std::ostream& strem) const; ClassDefNV(CTPInput, 3); }; +/// Descriptor = Generator or List of [negated] inputs struct CTPDescriptor { CTPDescriptor() = default; std::string name = ""; @@ -84,6 +91,7 @@ struct CTPDetector { void printStream(std::ostream& stream) const; ClassDefNV(CTPDetector, 1) }; +/// List of detectors struct CTPCluster { CTPCluster() = default; std::string name = ""; @@ -93,27 +101,32 @@ struct CTPCluster { void printStream(std::ostream& strem) const; ClassDefNV(CTPCluster, 3) }; +/// Class = Mask+Descriptor+Cluster struct CTPClass { CTPClass() = default; std::string name = ""; std::uint64_t classMask = 0; CTPDescriptor const* descriptor = nullptr; CTPCluster const* cluster = nullptr; - int clusterIndex = 0; + int clusterIndex = 0xff; int descriptorIndex = 0xff; uint32_t downScale = 1; std::vector BCClassMask; - uint64_t getClassMaskForInput(int inputindex) const; + int getIndex() const { return ((classMask > 0) ? log2(classMask) : 0xff); } void printStream(std::ostream& strem) const; ClassDefNV(CTPClass, 4); }; struct CTPInputsConfiguration { CTPInputsConfiguration() = default; std::vector CTPInputs; + static const std::vector CTPInputsDefault; int createInputsConfigFromFile(std::string& filename); void printStream(std::ostream& strem) const; static CTPInputsConfiguration defaultInputConfig; static void initDefaultInputConfig(); + static std::string getInputNameFromIndex100(uint32_t index); + static std::string getInputNameFromIndex(uint32_t index); + static int getInputIndexFromName(std::string& name); ClassDefNV(CTPInputsConfiguration, 0); }; class CTPConfiguration @@ -125,11 +138,14 @@ class CTPConfiguration bool isDetector(const o2::detectors::DetID& det); static void capitaliseString(std::string& str); static bool isNumber(const std::string& s); + int addInput(std::string& inp, int clsindex, std::map>& descInputsIndex); enum ConfigPart { START, + VERSION, RUN, INPUTS, MASKS, GENS, + DESCRIPTORS, LTG, LTGitems, CLUSTER, @@ -137,33 +153,44 @@ class CTPConfiguration UNKNOWN }; int loadConfigurationRun3(const std::string& ctpconfiguartion); void printStream(std::ostream& stream) const; + void setRunNumber(uint32_t runnumber) { mRunNumber = runnumber; } std::vector& getCTPInputs() { return mInputs; } std::vector& getCTPClasses() { return mCTPClasses; } + const std::vector& getCTPInputs() const { return mInputs; } // Read-only interface + const std::vector& getCTPClasses() const { return mCTPClasses; } // Read-only interface uint64_t getInputMask(const std::string& name) const; int getInputIndex(const std::string& name) const; + std::string getClassNameFromIndex(int index); + std::string getClassNameFromHWIndex(int index); + const CTPClass* getCTPClassFromHWIndex(const int index) const; bool isMaskInInputs(const uint64_t& mask) const; bool isBCMaskInConfig(const std::string maskname) const; + const BCMask* isBCMaskInConfigP(const std::string bcmask) const; const CTPInput* isInputInConfig(const std::string inpname) const; - const CTPInput* isInputInConfig(const int index) const; + const CTPInput* isInputInConfig(const uint32_t index) const; + const CTPDescriptor* isDescriptorInConfig(const std::string descname, int& index) const; void createInputsInDecriptorsFromNames(); uint64_t getDecrtiptorInputsMask(const std::string& name) const; std::map> getDet2InputMap(); uint64_t getTriggerClassMask() const; + uint64_t getTriggerClassMaskWInputs() const; + uint64_t getTriggerClassMaskWInputsNoTrgDets() const; std::vector getTriggerClassList() const; uint32_t getRunNumber() { return mRunNumber; }; std::vector getDetectorList() const; o2::detectors::DetID::mask_t getDetectorMask() const; - void createDefaultInputsConfig(); - uint64_t getClassMaskForInput(int inputindex) const; - uint64_t getClassMaskForInput(const std::string& name) const; - void printConfigString() { std::cout << mConfigString << std::endl; }; + uint64_t getClassMaskForInputMask(uint64_t inputMask) const; + void printConfigString(); std::string getConfigString() { return mConfigString; }; + CTPDescriptor* getDescriptor(int index) { return &mDescriptors[index]; }; + int assignDescriptors(); + int checkConfigConsistency() const; private: std::string mConfigString = ""; uint32_t mRunNumber = 0; std::string mName = ""; - std::string mVersion = "1"; + std::string mVersion = "0"; std::vector mBCMasks; std::vector mGenerators; std::vector mInputs; @@ -171,54 +198,25 @@ class CTPConfiguration std::vector mDetectors; std::vector mClusters; std::vector mCTPClasses; - int processConfigurationLineRun3(std::string& line, int& level); + int processConfigurationLineRun3(std::string& line, int& level, std::map>& descInputsIndex); + int processConfigurationLineRun3v2(std::string& line, int& level, std::map>& descInputsIndex); ClassDefNV(CTPConfiguration, 6); }; -// Run Manager -struct CTPActiveRun { - CTPActiveRun() = default; - long timeStart; - long timeStop; - CTPConfiguration cfg; - CTPRunScalers scalers; -}; -class CTPRunManager -{ - public: - CTPRunManager() = default; - void init(); - int loadRun(const std::string& cfg); - int startRun(const std::string& cfg); - int stopRun(uint32_t irun); - int addScalers(uint32_t irun, std::time_t time); - int processMessage(std::string& topic, const std::string& message); - void printActiveRuns() const; - int saveRunScalersToCCDB(int i); - int saveRunConfigToCCDB(CTPConfiguration* cfg, long timeStart); - static CTPConfiguration getConfigFromCCDB(long timestamp, std::string run); - CTPRunScalers getScalersFromCCDB(long timestamp, std::string); - int loadScalerNames(); - // void setCCDBPathConfig(std::string path) { mCCDBPathCTPConfig = path;}; - void setCCDBPathScalers(std::string path) { mCCDBPathCTPScalers = path; }; - static void setCCDBHost(std::string host) { mCCDBHost = host; }; - void setCTPQC(int qc) { mQC = qc; }; - void printCounters(); - private: - /// Database constants - // std::string mCCDBHost = "http://ccdb-test.cern.ch:8080"; - static std::string mCCDBHost; - std::string mCCDBPathCTPScalers = "CTP/Calib/Scalers"; - std::array mActiveRuns; - std::array mActiveRunNumbers; - std::array mCounters; - std::map mScalerName2Position; - std::map mRunsLoaded; - int mEOX = 0; // redundancy check - int mQC = 0; // 1 - no CCDB: used for QC - ClassDefNV(CTPRunManager, 5); -}; +std::ostream& operator<<(std::ostream& in, const CTPConfiguration& conf); +struct CtpCfg { + CtpCfg() = default; + std::string filename = "ctp.cfg"; + int readAndSave(std::string& path); + uint32_t TFOrbits = 0; + int ccdb = -1; // -1 means def constructor was called + uint32_t orbitShift = 0; + uint32_t irInputs_1_24 = 0; + uint32_t irInputs_25_48 = 0; + std::vector listOfUsedInputs(); + ClassDefNV(CtpCfg, 2) +}; } // namespace ctp } // namespace o2 #endif //_CTP_CONFIGURATION_H_ diff --git a/DataFormats/Detectors/CTP/include/DataFormatsCTP/Digits.h b/DataFormats/Detectors/CTP/include/DataFormatsCTP/Digits.h index f7e85a07556fc..f52e63c24f36e 100644 --- a/DataFormats/Detectors/CTP/include/DataFormatsCTP/Digits.h +++ b/DataFormats/Detectors/CTP/include/DataFormatsCTP/Digits.h @@ -25,6 +25,7 @@ namespace o2 namespace ctp { /// CTP related constants +static constexpr int CRUPageAlignment = 16; static constexpr uint32_t GBTLinkIDIntRec = 0; static constexpr uint32_t NIntRecPayload = 48 + 12; static constexpr uint32_t GBTLinkIDClassRec = 1; @@ -33,6 +34,7 @@ static constexpr uint32_t NGBT = 80; static constexpr std::uint32_t NumOfHBInTF = 128; static constexpr uint32_t NRUNS = 16; typedef std::bitset gbtword80_t; +typedef std::bitset<128> gbtword128_t; // static constexpr std::uint32_t CTP_NINPUTS = 48; /// Max number of CTP inputs for all levels static constexpr std::uint32_t CTP_NCLASSES = 64; /// Number of classes in hardware @@ -49,10 +51,12 @@ struct CTPDigit { o2::InteractionRecord intRecord; std::bitset CTPInputMask; std::bitset CTPClassMask; - CTPDigit() = default; void printStream(std::ostream& stream) const; void setInputMask(gbtword80_t mask); void setClassMask(gbtword80_t mask); + bool isInputEmpty() const { return CTPInputMask.count() == 0; } + bool isClassEmpty() const { return CTPClassMask.count() == 0; } + bool isEmpty() const { return isInputEmpty() && isClassEmpty(); } bool operator==(const CTPDigit& d) const { return intRecord == d.intRecord && CTPInputMask == d.CTPInputMask && CTPClassMask == d.CTPClassMask; @@ -67,7 +71,6 @@ struct CTPInputDigit { o2::InteractionRecord intRecord; std::bitset inputsMask; o2::detectors::DetID::ID detector; - CTPInputDigit() = default; ClassDefNV(CTPInputDigit, 1) }; } // namespace ctp diff --git a/DataFormats/Detectors/CTP/include/DataFormatsCTP/LumiInfo.h b/DataFormats/Detectors/CTP/include/DataFormatsCTP/LumiInfo.h new file mode 100644 index 0000000000000..c75fcc32ddaf4 --- /dev/null +++ b/DataFormats/Detectors/CTP/include/DataFormatsCTP/LumiInfo.h @@ -0,0 +1,45 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef _ALICEO2_CTP_LUMIINFO_H_ +#define _ALICEO2_CTP_LUMIINFO_H_ +#include "CommonConstants/LHCConstants.h" +#include +#include + +/// \brief Luminosity information as a moving average over certain number of TFs + +namespace o2 +{ +namespace ctp +{ +struct LumiInfo { + uint32_t orbit = 0; // orbit of TF when was updated + uint32_t nHBFCounted = 0; // length of interval in HB + uint32_t nHBFCountedFV0 = 0; + uint64_t counts = 0; // counts in the interval for the nominal lumi detector (FT0) + uint64_t countsFV0 = 0; // connts for FV0 (less reliable) + int inp1 = 3; // TVX + int inp2 = 6; // VBA + float getLumi() const { return nHBFCounted > 0 ? float(counts / (nHBFCounted * o2::constants::lhc::LHCOrbitMUS * 1e-6)) : 0.f; } + float getLumiFV0() const { return nHBFCountedFV0 > 0 ? float(countsFV0 / (nHBFCountedFV0 * o2::constants::lhc::LHCOrbitMUS * 1e-6)) : 0.f; } + float getLumiAlt() const { return getLumiFV0(); } + float getLumiError() const { return nHBFCounted > 0 ? float(std::sqrt(counts) / (nHBFCounted * o2::constants::lhc::LHCOrbitMUS * 1e-6)) : 0.f; } + float getLumiFV0Error() const { return nHBFCountedFV0 > 0 ? float(std::sqrt(countsFV0) / (nHBFCountedFV0 * o2::constants::lhc::LHCOrbitMUS * 1e-6)) : 0.f; } + float getLumiAltError() const { return getLumiFV0Error(); } + void printInputs() const; + ClassDefNV(LumiInfo, 3); +}; +} // namespace ctp + +} // namespace o2 + +#endif // _ALICEO2_CTP_LUMIINFO_H_ diff --git a/DataFormats/Detectors/CTP/include/DataFormatsCTP/Scalers.h b/DataFormats/Detectors/CTP/include/DataFormatsCTP/Scalers.h index 05f66ea7b8747..45d54b034f8d9 100644 --- a/DataFormats/Detectors/CTP/include/DataFormatsCTP/Scalers.h +++ b/DataFormats/Detectors/CTP/include/DataFormatsCTP/Scalers.h @@ -28,6 +28,14 @@ namespace ctp /// raw scalers produced by CTP and send to O2 either via /// - ZeroMQ published at CTP control machine /// - CTPreadout to FLP +struct errorCounters { + errorCounters() = default; + void printStream(std::ostream& stream) const; + uint32_t lmB = 0, l0B = 0, l1B = 0, lmA = 0, l0A = 0, l1A = 0; // decreasing counters + uint32_t lmBlmA = 0, lmAl0B = 0, l0Bl0A = 0, l0Al1B = 0, l1Bl1A = 0; // between levels countres + uint32_t lmBlmAd1 = 0, lmAl0Bd1 = 0, l0Bl0Ad1 = 0, l0Al1Bd1 = 0, l1Bl1Ad1 = 0; // between levels countres - diff =1 - just warning + uint32_t MAXPRINT = 3; +}; struct CTPScalerRaw { CTPScalerRaw() = default; uint32_t classIndex; @@ -52,6 +60,7 @@ struct CTPScalerO2 { uint64_t l1Before; uint64_t l1After; void printStream(std::ostream& stream) const; + void printFromZero(std::ostream& stream, CTPScalerO2& scaler0) const; ClassDefNV(CTPScalerO2, 1); }; struct CTPScalerRecordRaw { @@ -59,41 +68,105 @@ struct CTPScalerRecordRaw { o2::InteractionRecord intRecord; double_t epochTime; std::vector scalers; - std::vector scalersDets; + // std::vector scalersDets; + std::vector scalersInps; void printStream(std::ostream& stream) const; - ClassDefNV(CTPScalerRecordRaw, 3); + ClassDefNV(CTPScalerRecordRaw, 4); }; struct CTPScalerRecordO2 { CTPScalerRecordO2() = default; o2::InteractionRecord intRecord; double_t epochTime; std::vector scalers; - std::vector scalersDets; + // std::vector scalersDets; + std::vector scalersInps; void printStream(std::ostream& stream) const; - ClassDefNV(CTPScalerRecordO2, 3); + void printFromZero(std::ostream& stream, CTPScalerRecordO2& record0) const; + ClassDefNV(CTPScalerRecordO2, 4); }; class CTPRunScalers { public: + // + // static constexpr uint32_t NCOUNTERS = 1052; + // v1 + // static constexpr uint32_t NCOUNTERS = 1070; + // v2 - orbitid added at the end + static constexpr uint32_t NCOUNTERSv2 = 1071; + static constexpr uint32_t NCOUNTERS = 1085; + static std::vector scalerNames; CTPRunScalers() = default; void printStream(std::ostream& stream) const; + void printO2(std::ostream& stream) const; + void printFromZero(std::ostream& stream) const; void printClasses(std::ostream& stream) const; std::vector getClassIndexes() const; + uint32_t getRunNumber() { return mRunNumber; }; + int getScalerIndexForClass(uint32_t cls) const; + std::vector& getScalerRecordO2() { return mScalerRecordO2; }; + std::vector& getScalerRecordRaw() { return mScalerRecordRaw; }; + void setEpochTime(std::time_t tt, int index) { mScalerRecordRaw[index].epochTime = tt; }; int readScalers(const std::string& rawscalers); int convertRawToO2(); - int checkConsistency(const CTPScalerO2& scal0, const CTPScalerO2& scal1) const; - int checkConsistency(const CTPScalerRecordO2& rec0, const CTPScalerRecordO2& rec1) const; + int checkConsistency(const CTPScalerO2& scal0, const CTPScalerO2& scal1, errorCounters& eCnts) const; + int checkConsistency(const CTPScalerRecordO2& rec0, const CTPScalerRecordO2& rec1, errorCounters& eCnts) const; void setClassMask(std::bitset classMask) { mClassMask = classMask; }; void setDetectorMask(o2::detectors::DetID::mask_t mask) { mDetectorMask = mask; }; void setRunNumber(uint32_t rnumber) { mRunNumber = rnumber; }; void addScalerRacordRaw(CTPScalerRecordRaw& scalerrecordraw) { mScalerRecordRaw.push_back(scalerrecordraw); }; - uint32_t getRunNUmber() { return mRunNumber; }; int printRates(); int printIntegrals(); + int printInputRateAndIntegral(int inp); + int printClassBRateAndIntegralII(int icls); + int printClassBRateAndIntegral(int iclsinscalers); // - // static constexpr uint32_t NCOUNTERS = 1052; - static constexpr uint32_t NCOUNTERS = 1070; - static std::vector scalerNames; + int addOrbitOffset(uint32_t offset); + // + void printLMBRateVsT() const; // prints LMB interaction rate vs time for debugging + // returns the pair of global (levelled) interaction rate, as well as interpolated + // rate in Hz at a certain orbit number within the run + std::pair getRate(uint32_t orbit, int classindex, int type, bool qc = 0) const; + + /// same with absolute timestamp (not orbit) as argument + std::pair getRateGivenT(double timestamp, int classindex, int type, bool qc = 0) const; + + /// retrieves integral for class + std::array getIntegralForClass(int i) const + { + return { + mScalerRecordO2[0].scalers[i].classIndex, + mScalerRecordO2[mScalerRecordO2.size() - 1].scalers[i].lmBefore - mScalerRecordO2[0].scalers[i].lmBefore, + mScalerRecordO2[mScalerRecordO2.size() - 1].scalers[i].lmAfter - mScalerRecordO2[0].scalers[i].lmAfter, + mScalerRecordO2[mScalerRecordO2.size() - 1].scalers[i].l0Before - mScalerRecordO2[0].scalers[i].l0Before, + mScalerRecordO2[mScalerRecordO2.size() - 1].scalers[i].l0After - mScalerRecordO2[0].scalers[i].l0After, + mScalerRecordO2[mScalerRecordO2.size() - 1].scalers[i].l1Before - mScalerRecordO2[0].scalers[i].l1Before, + mScalerRecordO2[mScalerRecordO2.size() - 1].scalers[i].l1After - mScalerRecordO2[0].scalers[i].l1After, + }; + } + /// retrieves integral - same interface as getRate, no pileup correction + uint64_t getLumiNoPuCorr(int classindex, int type) const; + /// retrieves vector of counters - same interface as getRate, needed for + std::vector> getRatesForIndex(int classindex, int type) const; + /// retrieves time boundaries of this scaler object from O2 scalers + std::pair getTimeLimit() const + { + return std::make_pair((unsigned long)mScalerRecordO2[0].epochTime * 1000, (unsigned long)mScalerRecordO2[mScalerRecordO2.size() - 1].epochTime * 1000); + } + /// retrieves time boundaries of this scaler object from Raw: should be same as from O2 and can be used without convertRawToO2 call + std::pair getTimeLimitFromRaw() const + { + return std::make_pair((unsigned long)mScalerRecordRaw[0].epochTime * 1000, (unsigned long)mScalerRecordRaw[mScalerRecordRaw.size() - 1].epochTime * 1000); + } + /// retrieves orbit boundaries of this scaler object from O2 + std::pair getOrbitLimit() const + { + return std::make_pair((unsigned long)mScalerRecordO2[0].intRecord.orbit, (unsigned long)mScalerRecordO2[mScalerRecordO2.size() - 1].intRecord.orbit); + } + /// retrieves orbit boundaries of this scaler object from Raw: should be same as from O2 and can be used without convertRawToO2 call + std::pair getOrbitLimitFromRaw() const + { + return std::make_pair((unsigned long)mScalerRecordRaw[0].intRecord.orbit, (unsigned long)mScalerRecordRaw[mScalerRecordRaw.size() - 1].intRecord.orbit); + } private: // map from class index to overflow @@ -106,10 +179,11 @@ class CTPRunScalers o2::detectors::DetID::mask_t mDetectorMask; std::vector mScalerRecordRaw; std::vector mScalerRecordO2; - int processScalerLine(const std::string& line, int& level, int& nclasses); - int copyRawToO2ScalerRecord(const CTPScalerRecordRaw& rawrec, CTPScalerRecordO2& o2rec, overflows_t& classesoverflows); + int processScalerLine(const std::string& line, int& level, uint32_t& nclasses); + int copyRawToO2ScalerRecord(const CTPScalerRecordRaw& rawrec, CTPScalerRecordO2& o2rec, overflows_t& classesoverflows, std::array& overflows); int updateOverflows(const CTPScalerRecordRaw& rec0, const CTPScalerRecordRaw& rec1, overflows_t& classesoverflows) const; int updateOverflows(const CTPScalerRaw& scal0, const CTPScalerRaw& scal1, std::array& overflow) const; + int updateOverflowsInps(const CTPScalerRecordRaw& rec0, const CTPScalerRecordRaw& rec1, std::array& overflow) const; ClassDefNV(CTPRunScalers, 2); }; } // namespace ctp diff --git a/DataFormats/Detectors/CTP/include/DataFormatsCTP/TriggerOffsetsParam.h b/DataFormats/Detectors/CTP/include/DataFormatsCTP/TriggerOffsetsParam.h index 25b8ddb62327b..063336e5461ce 100644 --- a/DataFormats/Detectors/CTP/include/DataFormatsCTP/TriggerOffsetsParam.h +++ b/DataFormats/Detectors/CTP/include/DataFormatsCTP/TriggerOffsetsParam.h @@ -22,9 +22,12 @@ namespace o2 namespace ctp { struct TriggerOffsetsParam : public o2::conf::ConfigurableParamHelper { + static constexpr int MaxNDet = 32; // take with margin to account for possible changes / upgrades int64_t LM_L0 = 15; - int64_t L0_L1 = 280; - + int64_t L0_L1 = 281; // trigger input latency + int64_t globalInputsShift = 0; // Global shift of inps; customOffset[CTP] is global shift of classes + int64_t customOffset[MaxNDet] = {}; + int64_t L0_L1_classes = 280; // trigger input latency O2ParamDef(TriggerOffsetsParam, "TriggerOffsetsParam"); // boilerplate stuff + make principal key }; } // namespace ctp diff --git a/DataFormats/Detectors/CTP/src/CTPRateFetcher.cxx b/DataFormats/Detectors/CTP/src/CTPRateFetcher.cxx new file mode 100644 index 0000000000000..5f31fe5741240 --- /dev/null +++ b/DataFormats/Detectors/CTP/src/CTPRateFetcher.cxx @@ -0,0 +1,274 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include "DataFormatsCTP/CTPRateFetcher.h" +#include "CCDB/BasicCCDBManager.h" + +#include +#include +#include "CommonConstants/LHCConstants.h" + +using namespace o2::ctp; +double CTPRateFetcher::fetch(o2::ccdb::BasicCCDBManager* ccdb, uint64_t timeStamp, int runNumber, std::string sourceName) +{ + auto triggerRate = fetchNoPuCorr(ccdb, timeStamp, runNumber, sourceName); + if (triggerRate >= 0) { + return pileUpCorrection(triggerRate); + } + return -1; +} +double CTPRateFetcher::fetchNoPuCorr(o2::ccdb::BasicCCDBManager* ccdb, uint64_t timeStamp, int runNumber, std::string sourceName) +{ + setupRun(runNumber, ccdb, timeStamp, 1); + if (sourceName.find("ZNC") != std::string::npos) { + if (runNumber < 544448) { + return fetchCTPratesInputsNoPuCorr(timeStamp, 25) / (sourceName.find("hadronic") != std::string::npos ? 28. : 1.); + } else { + return fetchCTPratesClassesNoPuCorr(timeStamp, "C1ZNC-B-NOPF-CRU", 6) / (sourceName.find("hadronic") != std::string::npos ? 28. : 1.); + } + } else if (sourceName == "T0CE") { + return fetchCTPratesClassesNoPuCorr(timeStamp, "CMTVXTCE-B-NOPF"); + } else if (sourceName == "T0SC") { + return fetchCTPratesClassesNoPuCorr(timeStamp, "CMTVXTSC-B-NOPF"); + } else if (sourceName == "T0VTX") { + if (runNumber < 534202) { + return fetchCTPratesClassesNoPuCorr(timeStamp, "minbias_TVX_L0", 3); // 2022 + } else { + double ret = fetchCTPratesClassesNoPuCorr(timeStamp, "CMTVX-B-NOPF"); + if (ret == -2.) { + LOG(info) << "Trying different class"; + ret = fetchCTPratesClassesNoPuCorr(timeStamp, "CMTVX-NONE"); + if (ret < 0) { + LOG(error) << "None of the classes used for lumi found"; + return -1.; + } + } + return ret; + } + } + LOG(error) << "CTP rate for " << sourceName << " not available"; + return -1.; +} +void CTPRateFetcher::updateScalers(ctp::CTPRunScalers& scalers) +{ + mScalers = scalers; + mScalers.convertRawToO2(); +} +// +int CTPRateFetcher::getRates(std::array& rates, o2::ccdb::BasicCCDBManager* ccdb, int runNumber, const std::string sourceName) // rates at start,stop and middle of the run +{ + setupRun(runNumber, ccdb, 0, 1); + mOrbit = 1; + mOutsideLimits = 1; + auto orbitlimits = mScalers.getOrbitLimit(); + // std::cout << "1st orbit:" << orbitlimits.first << " last:" << orbitlimits.second << " Middle:" << (orbitlimits.first + orbitlimits.second)/2 << std::endl; + double rate0 = fetch(ccdb, orbitlimits.first, mRunNumber, sourceName); + double rateLast = fetch(ccdb, orbitlimits.second, mRunNumber, sourceName); + double rateM = fetch(ccdb, (orbitlimits.first + orbitlimits.second) / 2, mRunNumber, sourceName); + // std::cout << rate0 << " " << rateLast << " " << rateM << std::endl; + rates[0] = rate0; + rates[1] = rateLast; + rates[2] = rateM; + return 0; +} +double CTPRateFetcher::getLumiNoPuCorr(const std::string& classname, int type) +{ + if (classname == "zncinp") { + return mScalers.getLumiNoPuCorr(26, 7); + } + std::vector& ctpcls = mConfig.getCTPClasses(); + std::vector clslist = mConfig.getTriggerClassList(); + int classIndex = -1; + for (size_t i = 0; i < clslist.size(); i++) { + if (ctpcls[i].name.find(classname) != std::string::npos) { + classIndex = i; + break; + } + } + if (classIndex == -1) { + LOG(warn) << "Trigger class " << classname << " not found in CTPConfiguration"; + return -1; + } + return mScalers.getLumiNoPuCorr(classIndex, type); +} +double CTPRateFetcher::getLumiWPuCorr(const std::string& classname, int type) +{ + std::vector> scals; + if (classname == "zncinp") { + scals = mScalers.getRatesForIndex(26, 7); + } else { + std::vector& ctpcls = mConfig.getCTPClasses(); + std::vector clslist = mConfig.getTriggerClassList(); + int classIndex = -1; + for (size_t i = 0; i < clslist.size(); i++) { + if (ctpcls[i].name.find(classname) != std::string::npos) { + classIndex = i; + break; + } + } + if (classIndex == -1) { + LOG(warn) << "Trigger class " << classname << " not found in CTPConfiguration"; + return -1; + } + scals = mScalers.getRatesForIndex(classIndex, type); + } + double lumi = 0; + for (auto const& ss : scals) { + // std::cout << ss.first << " " << ss.second << " " << pileUpCorrection(ss.first/ss.second) << std::endl; + lumi += pileUpCorrection(ss.first / ss.second) * ss.second; + } + return lumi; +} +double CTPRateFetcher::getLumi(const std::string& classname, int type, int puCorr) +{ + if (puCorr) { + return getLumiWPuCorr(classname, type); + } else { + return getLumiNoPuCorr(classname, type); + } +} + +double CTPRateFetcher::getLumi(o2::ccdb::BasicCCDBManager* ccdb, int runNumber, const std::string sourceName, int puCorr) +{ + // setupRun(runNumber, ccdb, timeStamp, 1); + if (sourceName.find("ZNC") != std::string::npos) { + if (runNumber < 544448) { + return getLumi("zncinp", 1, puCorr) / (sourceName.find("hadronic") != std::string::npos ? 28. : 1.); + } else { + return getLumi("C1ZNC-B-NOPF-CRU", 6, puCorr) / (sourceName.find("hadronic") != std::string::npos ? 28. : 1.); + } + } else if (sourceName == "T0CE") { + return getLumi("CMTVXTCE-B-NOPF", 1, puCorr); + } else if (sourceName == "T0SC") { + return getLumi("CMTVXTSC-B-NOPF", 1, puCorr); + } else if (sourceName == "T0VTX") { + if (runNumber < 534202) { + return getLumi("minbias_TVX_L0", 3, puCorr); // 2022 + } else { + double ret = getLumi("CMTVX-B-NOPF", 1, puCorr); + if (ret == -1.) { + LOG(info) << "Trying different class"; + ret = getLumi("CMTVX-NONE", 1, puCorr); + if (ret < 0) { + LOG(fatal) << "None of the classes used for lumi found"; + } + } + return ret; + } + } + LOG(error) << "CTP Lumi for " << sourceName << " not available"; + return 0; +} +// +double CTPRateFetcher::fetchCTPratesClasses(uint64_t timeStamp, const std::string& className, int inputType) +{ + auto triggerRate = fetchCTPratesClassesNoPuCorr(timeStamp, className, inputType); + if (triggerRate >= 0) { + return pileUpCorrection(triggerRate); + } + return -1; +} +double CTPRateFetcher::fetchCTPratesClassesNoPuCorr(uint64_t timeStamp, const std::string& className, int inputType) +{ + std::vector& ctpcls = mConfig.getCTPClasses(); + std::vector clslist = mConfig.getTriggerClassList(); + int classIndex = -1; + for (size_t i = 0; i < clslist.size(); i++) { + if (ctpcls[i].name.find(className) != std::string::npos) { + classIndex = i; + break; + } + } + if (classIndex == -1) { + LOG(warn) << "Trigger class " << className << " not found in CTPConfiguration"; + return -2.; + } + if (mOrbit) { + auto rate{mScalers.getRate((uint32_t)timeStamp, classIndex, inputType, mOutsideLimits)}; + return rate.second; + } else { + auto rate{mScalers.getRateGivenT(timeStamp * 1.e-3, classIndex, inputType, mOutsideLimits)}; + return rate.second; + } +} +double CTPRateFetcher::fetchCTPratesInputs(uint64_t timeStamp, int input) +{ + std::vector& recs = mScalers.getScalerRecordO2(); + if (recs[0].scalersInps.size() == 48) { + if (mOrbit) { + return pileUpCorrection(mScalers.getRate((uint32_t)timeStamp, input, 7, mOutsideLimits).second); + } else { + return pileUpCorrection(mScalers.getRateGivenT(timeStamp * 1.e-3, input, 7, mOutsideLimits).second); + } + } else { + LOG(error) << "Inputs not available"; + return -1.; + } +} +double CTPRateFetcher::fetchCTPratesInputsNoPuCorr(uint64_t timeStamp, int input) +{ + std::vector& recs = mScalers.getScalerRecordO2(); + if (recs[0].scalersInps.size() == 48) { + if (mOrbit) { + return mScalers.getRate((uint32_t)timeStamp, input, 7, mOutsideLimits).second; + } else { + return mScalers.getRateGivenT(timeStamp * 1.e-3, input, 7, mOutsideLimits).second; // qc flag implemented only for time + } + } else { + LOG(error) << "Inputs not available"; + return -1.; + } +} + +double CTPRateFetcher::pileUpCorrection(double triggerRate) +{ + if (mLHCIFdata.getFillNumber() == 0) { + LOG(fatal) << "No filling" << std::endl; + } + auto bfilling = mLHCIFdata.getBunchFilling(); + std::vector bcs = bfilling.getFilledBCs(); + double nbc = bcs.size(); + double nTriggersPerFilledBC = triggerRate / nbc / constants::lhc::LHCRevFreq; + double mu = -std::log(1 - nTriggersPerFilledBC); + return mu * nbc * constants::lhc::LHCRevFreq; +} + +void CTPRateFetcher::setupRun(int runNumber, o2::ccdb::BasicCCDBManager* ccdb, uint64_t timeStamp, bool initScalers) +{ + if (runNumber == mRunNumber) { + return; + } + mRunNumber = runNumber; + LOG(info) << "Setting up CTP scalers for run " << mRunNumber << " and timestamp : " << timeStamp; + auto ptrLHCIFdata = ccdb->getSpecific("GLO/Config/GRPLHCIF", timeStamp); + if (ptrLHCIFdata == nullptr) { + LOG(error) << "GRPLHCIFData not in database, timestamp:" << timeStamp; + return; + } + mLHCIFdata = *ptrLHCIFdata; + std::map metadata; + metadata["runNumber"] = std::to_string(mRunNumber); + auto ptrConfig = ccdb->getSpecific("CTP/Config/Config", timeStamp, metadata); + if (ptrConfig == nullptr) { + LOG(error) << "CTPRunConfig not in database, timestamp:" << timeStamp; + return; + } + mConfig = *ptrConfig; + if (initScalers) { + auto ptrScalers = ccdb->getSpecific("CTP/Calib/Scalers", timeStamp, metadata); + if (ptrScalers) { + mScalers = *ptrScalers; + mScalers.convertRawToO2(); + } else { + LOG(error) << "CTPRunScalers not in database, timestamp:" << timeStamp; + } + } +} diff --git a/DataFormats/Detectors/CTP/src/Configuration.cxx b/DataFormats/Detectors/CTP/src/Configuration.cxx index d128139d78bee..98458ef06d1d3 100644 --- a/DataFormats/Detectors/CTP/src/Configuration.cxx +++ b/DataFormats/Detectors/CTP/src/Configuration.cxx @@ -18,11 +18,11 @@ #include #include #include "CommonUtils/StringUtils.h" -#include "FairLogger.h" +#include +#include using namespace o2::ctp; // -std::string CTPRunManager::mCCDBHost = "http://o2-ccdb.internal"; const std::map CTPInput::run2DetToRun3Det = {{"T", "FT0"}, {"V", "FV0"}, {"U", "FDD"}, {"E", "EMC"}, {"D", "EMC"}, {"H", "TRD"}, {"O", "TOF"}, {"P", "PHS"}, {"Z", "ZDC"}}; const std::map CTPConfiguration::detName2LTG = {{"FV0", "1"}, {"FT0", "2"}, {"FDD", "3"}, {"ITS", "4"}, {"TOF", "5"}, {"MFT", "6"}, {"TPC", "7"}, {"MCH", "8"}, {"MID", "9"}, {"TST", "10"}, {"TRD", "13"}, {"HMP", "14"}, {"ZDC", "15"}, {"PHS", "16"}, {"EMC", "17"}, {"CPV", "18"}}; // @@ -48,10 +48,71 @@ bool CTPConfiguration::isNumber(const std::string& s) s.end(), [](unsigned char c) { return !std::isdigit(c); }) == s.end(); } // +int BCMask::setBCmask(std::vector& tokens) +{ + BCmask.reset(); + name = tokens[1]; + bool coded = tokens[2].find("L") != std::string::npos; + coded |= tokens[2].find("H") != std::string::npos; + std::cout << "coded:" << coded << std::endl; + if (coded) { + // jusko notation + std::string bcmaskstr = tokens[2]; + size_t pos = 0; + size_t posnext = 0; + int bccur = 0; + while (bccur < 3564) { + // std::cout << "pos:" << pos << std::endl; + size_t posH = bcmaskstr.find('H', pos); + size_t posL = bcmaskstr.find('L', pos); + // std::cout << "H:" << posH << " L:" << posL << std::endl; + bool b = 1; + posnext = posH; + if (posL < posH) { + posnext = posL; + b = 0; + } + std::string bcsub = bcmaskstr.substr(pos, posnext - pos); + // std::cout << "bcsub:" << bcsub << " b:" << b << std::endl; + int bcrange = 0; + try { + bcrange = std::stoull(bcsub); + } catch (...) { + LOG(warning) << "problem in bcmask decoding H:" << posH << " posL:" << posL << " bcsub:" << bcsub; + return 1; + } + if (b) { + for (int bc = bccur; bc < bccur + bcrange; bc++) { + try { + BCmask.set(bc, 1); + } catch (...) { + LOG(warning) << "BC mask decoding to big bc:" << bc; + } + } + } + bccur += bcrange; + pos = posnext + 1; + // std::cout << "bccur:" << bccur << std::endl; + } + } else { + // list of integers + for (uint32_t i = 2; i < tokens.size(); i++) { + uint32_t bc; + try { + bc = std::stoull(tokens[i]); + } catch (...) { + LOG(info) << "mask syntax:" << i << ":" << tokens[i]; + continue; + } + BCmask.set(bc, 1); + } + } + return 0; +} void BCMask::printStream(std::ostream& stream) const { - stream << "CTP BC mask:" << name << ":" << mask << std::endl; - /// << ":" << BCmask << std::endl; + stream << "CTP BC mask:" << name << ":" << mask; /// << std::endl; + stream << " # of active BC:" << BCmask.count() << std::endl; } // const std::set CTPGenerator::Generators = {"bcd1m", "bcd2m", "bcd10", "bcd20", "rnd1m", "rnd2m", "rnd10", "rnd20"}; @@ -113,15 +174,6 @@ void CTPCluster::printStream(std::ostream& stream) const stream << " clust index:" << hwMask; stream << std::endl; } -// -uint64_t CTPClass::getClassMaskForInput(int inputindex) const -{ - uint64_t clsmask = 0; - if (descriptor != nullptr) { - clsmask = getClassMaskForInput(inputindex); - } - return clsmask; -} void CTPClass::printStream(std::ostream& stream) const { stream << "CTP Class:" << name << " Hardware mask:" << classMask << " Cluster index:" << clusterIndex << " Desc index:" << descriptorIndex; @@ -140,39 +192,112 @@ void CTPClass::printStream(std::ostream& stream) const } /// CTP configuration /// Assuming Run2 format + LTG +int CTPConfiguration::addInput(std::string& inp, int clsindex, std::map>& descInputsIndex) +{ + LOG(info) << "adding input:" << inp; + CTPInput ctpinp; + std::string sinp = inp; + ; + if (inp[0] == '~') { + sinp = inp.substr(1, inp.size() - 1); + ctpinp.neg = 0; + } + if (inp[0] == 'b') { // BC downscale + ctpinp.level = "b"; + ctpinp.name = inp; + } else if (inp[0] == 'r') { // randpm gen + ctpinp.level = "r"; + ctpinp.name = inp; + } else if (isNumber(sinp)) { // inputs as number + int index = std::stoi(sinp); + ctpinp.name = CTPInputsConfiguration::getInputNameFromIndex100(index); + ctpinp.inputMask = 1ull << (index - 1); + ctpinp.level = ctpinp.name[0]; + if (ctpinp.neg == 0) { + ctpinp.name = "~" + ctpinp.name; + } + } else { // input as string or error + ctpinp.name = inp; + int index = CTPInputsConfiguration::getInputIndexFromName(inp); + ctpinp.level = sinp[0]; + ctpinp.inputMask = 1ull << (index - 1); + } + // add to desc + // check if already there + for (uint32_t i = 0; i < mInputs.size(); i++) { + if (mInputs[i].name == ctpinp.name) { + LOG(info) << "input found at:" << i; + descInputsIndex[clsindex].push_back(i); + return 0; + } + } + mInputs.push_back(ctpinp); + descInputsIndex[clsindex].push_back(mInputs.size() - 1); + LOG(info) << "input inderted at:" << mInputs.size() - 1; + return 0; +} int CTPConfiguration::loadConfigurationRun3(const std::string& ctpconfiguration) { LOG(info) << "Loading CTP configuration."; + std::map> clsDescIndex; + CTPInputsConfiguration::initDefaultInputConfig(); mConfigString = ctpconfiguration; std::istringstream iss(ctpconfiguration); int ret = 0; int level = START; std::string line; + int ver = 0; + int iline = 0; while (std::getline(iss, line)) { o2::utils::Str::trim(line); - if ((ret = processConfigurationLineRun3(line, level)) != 0) { + if (line.size() == 0) { + continue; + } + if (line.at(0) == '#') { + continue; + } + if (iline == 0) { + if (line.find("ver") != std::string::npos) { + ver = 1; + LOG(info) << "CTP Config vesrion:" << line; + } else { + LOG(info) << "CTP Config version: 0"; + } + } + iline++; + if (ver == 0) { + ret = processConfigurationLineRun3(line, level, clsDescIndex); + } else { + ret = processConfigurationLineRun3v2(line, level, clsDescIndex); + } + if (ret) { return ret; } } - for (auto& cls : mCTPClasses) { - cls.cluster = &mClusters[cls.clusterIndex]; - if (cls.descriptorIndex != 0xff) { - cls.descriptor = &mDescriptors[cls.descriptorIndex]; + if (ver == 0) { + for (auto& cls : mCTPClasses) { + cls.cluster = &mClusters[cls.clusterIndex]; + if (cls.descriptorIndex != 0xff) { + cls.descriptor = &mDescriptors[cls.descriptorIndex]; + if (cls.getIndex() != 0xff) { + for (auto const& inp : clsDescIndex[cls.getIndex()]) { + mDescriptors.at(cls.descriptorIndex).inputs.push_back(&mInputs.at(inp)); + } + } + } + } + createInputsInDecriptorsFromNames(); + } else { + for (auto& cls : mCTPClasses) { + cls.cluster = &mClusters[cls.clusterIndex]; } } - createInputsInDecriptorsFromNames(); return ret; } -int CTPConfiguration::processConfigurationLineRun3(std::string& line, int& level) +int CTPConfiguration::processConfigurationLineRun3(std::string& line, int& level, std::map>& descInputsIndex) { LOG(info) << "Processing line"; LOG(info) << "line:" << line << " lev:" << level; - if (line.size() == 0) { - return 0; - } - if (line.at(0) == '#') { - return 0; - } // std::vector tokens = o2::utils::Str::tokenize(line, ' '); size_t ntokens = tokens.size(); @@ -185,9 +310,9 @@ int CTPConfiguration::processConfigurationLineRun3(std::string& line, int& level level = RUN; } else if ((line.find("inp") != std::string::npos) && ((level == RUN) || (level == INPUTS))) { level = INPUTS; - } else if (((first = line.find("bcm")) != std::string::npos) && ((level == INPUTS) || (level == MASKS))) { + } else if (((first = line.find("bcm")) != std::string::npos) && ((level == RUN) || (level == INPUTS) || (level == MASKS))) { level = MASKS; - } else if (CTPGenerator::Generators.count(tokens[0]) && ((level == INPUTS) || (level == MASKS) || (level == GENS))) { + } else if (CTPGenerator::Generators.count(tokens[0]) && ((level == RUN) || (level == INPUTS) || (level == MASKS) || (level == GENS))) { level = GENS; } else if ((first = line.find("LTG")) != std::string::npos) { level = LTG; @@ -202,7 +327,11 @@ int CTPConfiguration::processConfigurationLineRun3(std::string& line, int& level LOG(info) << "Level:" << level; switch (level) { case RUN: { - mRunNumber = std::stoul(tokens[1]); + try { + mRunNumber = std::stoul(tokens[1]); + } catch (...) { + LOG(error) << "RUN:" << tokens[1] << std::endl; + } level = RUN; break; } @@ -230,9 +359,6 @@ int CTPConfiguration::processConfigurationLineRun3(std::string& line, int& level break; } bcmask.name = tokens[1]; - for (int i = 2; i < tokens.size(); i++) { - bcmask.mask = tokens[i] = " "; - } bool coded = tokens[2].find("L") != std::string::npos; coded |= tokens[2].find("H") != std::string::npos; // std::cout << "coded:" << coded << std::endl; @@ -240,12 +366,12 @@ int CTPConfiguration::processConfigurationLineRun3(std::string& line, int& level // jusko notation } else { // list of integers - for (int i = 2; i < ntokens; i++) { + for (uint32_t i = 2; i < ntokens; i++) { uint32_t bc; try { bc = std::stoull(tokens[i]); } catch (...) { - LOG(info) << "mask syntax:" << tokens[i]; + LOG(info) << "mask syntax:" << i << ":" << tokens[i]; continue; } bcmask.BCmask.set(bc, 1); @@ -295,7 +421,7 @@ int CTPConfiguration::processConfigurationLineRun3(std::string& line, int& level LOG(info) << "Cluster:" << line; cluster.name = tokens[2]; o2::detectors::DetID::mask_t mask; - for (int item = 3; item < ntokens; item++) { + for (uint32_t item = 3; item < ntokens; item++) { std::string detname = tokens[item]; capitaliseString(detname); // LOG(info) << "Detector:" << detname; @@ -327,8 +453,10 @@ int CTPConfiguration::processConfigurationLineRun3(std::string& line, int& level cls.classMask = 1ull << index; cls.name = tokens[1]; cls.clusterIndex = mClusters.size() - 1; + CTPDescriptor desc; + desc.name = "d" + cls.name; // LOG(info) << "point:" << cls.cluster << " " << &mClusters.front(); - for (int i = 2; i < tokens.size(); i++) { + for (uint32_t i = 2; i < tokens.size(); i++) { std::string token = tokens[i]; bool isGenerator = 0; for (auto const& gen : CTPGenerator::Generators) { @@ -338,42 +466,21 @@ int CTPConfiguration::processConfigurationLineRun3(std::string& line, int& level } } if (isGenerator) { - CTPDescriptor desc; - desc.name = token; - mDescriptors.push_back(desc); - cls.descriptorIndex = mDescriptors.size() - 1; + addInput(token, index, descInputsIndex); LOG(info) << "Class generator found:" << desc.name; } else if (token.find("~") != std::string::npos) { // inverted input // std::cout << "Inverted input" << std::endl; - std::string sinp = token.substr(1, token.size() - 1); - // uint32_t inp = std::strtoul(sinp); - if (cls.descriptorIndex == 0xff) { - CTPDescriptor desc; - desc.name = token; - mDescriptors.push_back(desc); - cls.descriptorIndex = mDescriptors.size() - 1; - } else { - CTPDescriptor desc = mDescriptors.at(cls.descriptorIndex); - desc.name += " " + token; - } - } else if (isNumber(token)) { // normal input + addInput(token, index, descInputsIndex); + } else if (isNumber(token)) { // normal input as number // std::cout << "Normal input" << std::endl; - if (cls.descriptorIndex == 0xff) { - CTPDescriptor desc; - desc.name = token; - mDescriptors.push_back(desc); - cls.descriptorIndex = mDescriptors.size() - 1; - } else { - CTPDescriptor desc = mDescriptors.at(cls.descriptorIndex); - desc.name += " " + token; - } - LOG(info) << "Class input descriptor:" << mDescriptors[mDescriptors.size() - 1].name; + addInput(token, index, descInputsIndex); + // LOG(info) << "Class input descriptor:" << mDescriptors[mDescriptors.size() - 1].name; } else if (token.find("0x") != std::string::npos) { // downscale // std::cout << "Downscale" << std::endl; cls.downScale = std::stoul(token, nullptr, 16); - } else { // mask + } else if (token.find("bcm") != std::string::npos) { // bcmask // std::cout << "Mask" << std::endl; - int i = 0; + uint32_t i = 0; for (auto const& bcm : mBCMasks) { if (bcm.name == token) { cls.BCClassMask.push_back(&bcm); @@ -383,10 +490,14 @@ int CTPConfiguration::processConfigurationLineRun3(std::string& line, int& level i++; } if (i == mBCMasks.size()) { - LOG(error) << "Class BCMask NOT found:" << token; + LOG(error) << "Class BCMask NOT found:" << token << " assuming input"; } + } else { // input as string or descriptor + addInput(token, index, descInputsIndex); } } + mDescriptors.push_back(desc); + cls.descriptorIndex = mDescriptors.size() - 1; // mCTPClasses.push_back(cls); break; @@ -397,19 +508,243 @@ int CTPConfiguration::processConfigurationLineRun3(std::string& line, int& level } return 0; } +int CTPConfiguration::processConfigurationLineRun3v2(std::string& line, int& level, std::map>& descInputsIndex) +{ + LOG(debug) << "Processing line"; + LOG(debug) << "line:" << line << " lev:" << level; + // + std::vector tokens = o2::utils::Str::tokenize(line, ' '); + size_t ntokens = tokens.size(); + if (ntokens == 0) { + LOG(warning) << "# of tokens zero in line:" << line; + return 0; + } + size_t first; + if (((first = line.find("ver")) != std::string::npos) && (level == START)) { + mVersion = line; + // std::cout << "debug:" << mVersion << std::endl; + level = VERSION; + return 0; + } else if (((first = line.find("run")) != std::string::npos) && (level == VERSION)) { + level = RUN; + } else if ((line.find("INPUTS") != std::string::npos) && (level == RUN)) { + level = INPUTS; + return 0; + } else if ((line.find("inp") != std::string::npos) && (level == INPUTS)) { + level = INPUTS; + } else if ((line.find("BCMASKS") != std::string::npos) && ((level == INPUTS) || (level == RUN))) { + level = MASKS; + return 0; + } else if (((first = line.find("bcm")) != std::string::npos) && (level == MASKS)) { + level = MASKS; + } else if (line.find("GENS") != std::string::npos) { + level = GENS; + return 0; + } else if (CTPGenerator::Generators.count(tokens[0]) && (level == GENS)) { + level = GENS; + } else if (line.find("DESCRIPTORS") != std::string::npos) { + level = DESCRIPTORS; + return 0; + } else if ((tokens[0][0] == 'D') && (level == DESCRIPTORS)) { + level = DESCRIPTORS; + } else if ((first = line.find("LTG")) != std::string::npos) { + level = LTG; + } else if ((first = line.find("cluster")) != std::string::npos) { + level = CLUSTER; + } else { + bool knownlevels = ((level == LTGitems) || (level == CLASS)); + if (knownlevels == false) { + level = UNKNOWN; + } + } + LOG(debug) << "Level:" << level; + switch (level) { + case VERSION: { + break; + } + case RUN: { + try { + mRunNumber = std::stoul(tokens[1]); + } catch (...) { + LOG(error) << "RUN line:" << line; + } + level = RUN; + break; + } + case INPUTS: { + level = INPUTS; + if (tokens.size() != 3) { + LOG(error) << "Wrong input line:" << line; + return 1; + } + CTPInput ctpinp; + ctpinp.name = tokens[1]; + ctpinp.level = tokens[1][0]; + std::string run2Name{tokens[1][1]}; + ctpinp.setRun3DetName(run2Name); + uint32_t index = std::stoul(tokens[2]); + ctpinp.inputMask = (1ull << (index - 1)); + mInputs.push_back(ctpinp); + LOG(debug) << "Input:" << ctpinp.name << " index:" << index; + break; + } + case MASKS: { + BCMask bcmask; + if (tokens.size() < 3) { + LOG(error) << "Wrong bc mask:" << line; + break; + } + bcmask.setBCmask(tokens); + mBCMasks.push_back(bcmask); + LOG(debug) << "BC mask added:" << bcmask.name; + break; + } + case GENS: { + CTPGenerator gen; + gen.name = tokens[0]; + gen.frequency = tokens[1]; + mGenerators.push_back(gen); + LOG(debug) << "Gen added:" << line; + break; + } + case DESCRIPTORS: { + if ((tokens.size() < 2)) { + if (line.find("TRUE") != std::string::npos) { + CTPDescriptor desc; + desc.name = tokens[0]; + mDescriptors.push_back(desc); + break; + } else { + LOG(warning) << "Unexpected Descriptor:" << line; + break; + } + } + CTPDescriptor desc; + desc.name = tokens[0]; + for (uint32_t i = 1; i < tokens.size(); i++) { + const CTPInput* inp = isInputInConfig(tokens[i]); + if (inp != nullptr) { + desc.inputs.push_back(inp); + } + } + mDescriptors.push_back(desc); + break; + } + case LTG: { + CTPDetector ctpdet; + std::string detname = tokens[1]; + capitaliseString(detname); + o2::detectors::DetID det(detname.c_str()); + if (isDetector(det)) { + ctpdet.detID = det.getID(); + LOG(debug) << "Detector found:" << det.getID() << " " << detname; + } else { + LOG(error) << "Unknown detectors:" << line; + } + mDetectors.push_back(ctpdet); + level = LTGitems; + break; + } + case LTGitems: { + if (ntokens == 1) { + mDetectors.back().mode = tokens[0]; + } + LOG(debug) << "LTGitem:" << line; + break; + } + case CLUSTER: { + CTPCluster cluster; + try { + cluster.hwMask = std::stoull(tokens[0]); + } catch (...) { + LOG(error) << "Cluster syntax error:" << line; + return level; + } + LOG(debug) << "Cluster:" << line; + cluster.name = tokens[2]; + o2::detectors::DetID::mask_t mask; + for (uint32_t item = 3; item < ntokens; item++) { + std::string detname = tokens[item]; + capitaliseString(detname); + // LOG(info) << "Detector:" << detname; + o2::detectors::DetID det(detname.c_str()); + isDetector(det); + mask |= det.getMask(); + } + cluster.maskCluster = mask; + mClusters.push_back(cluster); + level = CLASS; + // LOG(info) << "Cluster done:" << cluster.name << std::endl; + break; + } + case CLASS: { + // add to the last cluster + if (tokens.size() < 6) { + LOG(error) << "CTPClass items < 6" << line; + break; + } + uint64_t index; + try { + index = std::stoull(tokens[1]); + } catch (...) { + LOG(error) << "Class syntax error:" << line; + return level; + } + LOG(debug) << "Class:" << line; + CTPClass cls; + cls.classMask = 1ull << index; + cls.name = tokens[0]; + // Descriptor + std::string descname = tokens[2]; + int dindex; + if (descname.find("DTRUE") != std::string::npos) { + descname = "DTRUE"; + } + const CTPDescriptor* desc = isDescriptorInConfig(descname, dindex); + if (desc != nullptr) { + cls.descriptor = desc; + cls.descriptorIndex = dindex; + } + cls.clusterIndex = mClusters.size() - 1; + // PF not member of class + std::string bcmask = tokens[5]; + bcmask = bcmask.substr(1, bcmask.size() - 2); + if (bcmask.size()) { + const BCMask* bcm = isBCMaskInConfigP(bcmask); + if (bcm != nullptr) { + cls.BCClassMask.push_back(bcm); + } + } + // Down scaling + if (tokens.size() > 6) { + cls.downScale = std::stoul(tokens[6], nullptr, 16); + } + mCTPClasses.push_back(cls); + break; + } + default: { + LOG(warning) << "unknown line:" << line; + } + } + return 0; +} void CTPConfiguration::printStream(std::ostream& stream) const { - stream << "Configuration:" << mName << "\n Version:" << mVersion << std::endl; - stream << "Run:" << mRunNumber << " cfg name:" << mName; + stream << "Configuration:" << mName << " Version:" << mVersion << std::endl; + stream << "Run:" << mRunNumber << " cfg name:" << mName << std::endl; stream << "CTP BC masks:" << std::endl; for (const auto& i : mBCMasks) { i.printStream(stream); } - stream << "CTP inputs:" << std::endl; + stream << "CTP inputs:" << mInputs.size() << std::endl; for (const auto& i : mInputs) { i.printStream(stream); } - stream << "CTP descriptors:" << std::endl; + stream << "CTP generators:" << std::endl; + for (const auto& i : mGenerators) { + i.printStream(stream); + } + stream << "CTP descriptors:" << mDescriptors.size() << std::endl; for (const auto& i : mDescriptors) { i.printStream(stream); } @@ -437,12 +772,43 @@ uint64_t CTPConfiguration::getInputMask(const std::string& name) const } int CTPConfiguration::getInputIndex(const std::string& name) const { + int index = 0xff; const CTPInput* inp = isInputInConfig(name); - if (inp == nullptr) { - return 0xff; + if (inp != nullptr) { + index = inp->getIndex(); + } + LOG(info) << "input:" << name << " index:" << index; + return index; +} +std::string CTPConfiguration::getClassNameFromIndex(int index) +{ + if (index < (int)mCTPClasses.size()) { + return mCTPClasses[index].name; } else { - return inp->getIndex(); + std::string name = "Cls" + std::to_string(index); + return name; } +}; +std::string CTPConfiguration::getClassNameFromHWIndex(int index) +{ + for (auto& cls : mCTPClasses) { + if (cls.classMask == (1ull << index)) { + return cls.name; + } + } + std::string ret = "not found"; + return ret; +} +const CTPClass* CTPConfiguration::getCTPClassFromHWIndex(int index) const +{ + const CTPClass* clsfound = nullptr; + for (auto const& cls : mCTPClasses) { + if (index == cls.getIndex()) { + clsfound = &cls; + break; + } + } + return clsfound; } bool CTPConfiguration::isMaskInInputs(const uint64_t& mask) const { @@ -462,16 +828,29 @@ bool CTPConfiguration::isBCMaskInConfig(const std::string maskname) const } return false; } +const BCMask* CTPConfiguration::isBCMaskInConfigP(const std::string maskname) const +{ + for (const auto& bcm : mBCMasks) { + if (bcm.name == maskname) { + LOG(info) << "isBCMaskInConfigP found:" << maskname; + return &bcm; + } + } + LOG(info) << "isBCMaskInConfigP NOT found:" << maskname; + return nullptr; +} const CTPInput* CTPConfiguration::isInputInConfig(const std::string inpname) const { for (const auto& inp : mInputs) { if (inp.name == inpname) { + LOG(info) << "isInputInConfig found:" << inpname; return &inp; } } + LOG(info) << "isInputInConfig NOT found:" << inpname; return nullptr; } -const CTPInput* CTPConfiguration::isInputInConfig(const int index) const +const CTPInput* CTPConfiguration::isInputInConfig(const uint32_t index) const { for (const auto& inp : mInputs) { // std::cout << "isInputINConfig:" << inp.name << " " << inp.getIndex() << " " << index << std::endl; @@ -482,9 +861,23 @@ const CTPInput* CTPConfiguration::isInputInConfig(const int index) const } return nullptr; } +const CTPDescriptor* CTPConfiguration::isDescriptorInConfig(const std::string descname, int& index) const +{ + index = 0; + for (const auto& desc : mDescriptors) { + if (desc.name == descname) { + LOG(info) << "isDescriptorInConfig found:" << descname; + return &desc; + } + index++; + } + LOG(info) << "isDescriptorInConfig NOT found:" << descname; + return nullptr; +} void CTPConfiguration::createInputsInDecriptorsFromNames() // using run3 conventions for inputs { + LOG(info) << "Creating Inputs"; for (auto& des : mDescriptors) { if (CTPConfiguration::isNumber(des.name)) { // parse here if more inputs @@ -493,6 +886,7 @@ void CTPConfiguration::createInputsInDecriptorsFromNames() index = index - 100; } // CTPInput* inp = const_cast(isInputInConfig(index)); + LOG(info) << "Desc index:" << index; const CTPInput* inp = isInputInConfig(index); if (inp) { des.inputs.push_back(inp); @@ -504,15 +898,6 @@ void CTPConfiguration::createInputsInDecriptorsFromNames() } } } -uint64_t CTPConfiguration::getDecrtiptorInputsMask(const std::string& name) const -{ - for (auto const& desc : mDescriptors) { - if (desc.name == name) { - return desc.getInputsMask(); - } - } - return 0xffffffff; -} std::map> CTPConfiguration::getDet2InputMap() { std::map> det2inp; @@ -529,6 +914,32 @@ uint64_t CTPConfiguration::getTriggerClassMask() const } return clsmask; } +uint64_t CTPConfiguration::getTriggerClassMaskWInputs() const +{ + uint64_t clsmask = 0; + for (auto const& cls : mCTPClasses) { + if (cls.name.find("TRUE") != std::string::npos) { // ignoring internal ctp generators + continue; + } + clsmask |= cls.classMask; + } + return clsmask; +} +uint64_t CTPConfiguration::getTriggerClassMaskWInputsNoTrgDets() const +{ + uint64_t clsmask = 0; + for (auto const& cls : mCTPClasses) { + bool exclude = cls.name.find("TRUE") != std::string::npos; // ignoring internal ctp generators + exclude += cls.name.find("EMC") != std::string::npos; + exclude += cls.name.find("TRD") != std::string::npos; + exclude += cls.name.find("HMP") != std::string::npos; + if (!exclude) { + clsmask |= cls.classMask; + } + } + return clsmask; +} +// Hardware positions of classes std::vector CTPConfiguration::getTriggerClassList() const { uint64_t clsmask = getTriggerClassMask(); @@ -557,383 +968,103 @@ o2::detectors::DetID::mask_t CTPConfiguration::getDetectorMask() const } return mask; } -uint64_t CTPConfiguration::getClassMaskForInput(int inputindex) const +// This is special case of general approach: +// classmask = fin(inputmask)g +uint64_t CTPConfiguration::getClassMaskForInputMask(uint64_t inputMask) const { uint64_t clsmask = 0; for (auto const& cls : mCTPClasses) { - clsmask += cls.getClassMaskForInput(inputindex); + if (cls.descriptor) { + // std::cout << cls.name << std::hex << " " << cls.descriptor->getInputsMask() << " " << inputMask << std::endl; + if (cls.descriptor->getInputsMask() & inputMask) { + clsmask += cls.classMask; + // std::cout << " clsmask:" << clsmask << std::endl; + } + } } return clsmask; } -uint64_t CTPConfiguration::getClassMaskForInput(const std::string& name) const -{ - uint64_t clsmask = 0; - int index = getInputIndex(name); - return getClassMaskForInput(index); -} -//=============================================== -// -void CTPRunManager::init() -{ - for (auto r : mActiveRuns) { - r = nullptr; - } - loadScalerNames(); - LOG(info) << "CCDB host:" << mCCDBHost; - LOG(info) << "CTP QC:" << mQC; - LOG(info) << "CTPRunManager initialised."; -} -int CTPRunManager::loadRun(const std::string& cfg) +int CTPConfiguration::assignDescriptors() { - LOG(info) << "Loading run: " << cfg; - const auto now = std::chrono::system_clock::now(); - const long timeStamp = std::chrono::duration_cast(now.time_since_epoch()).count(); - CTPActiveRun* activerun = new CTPActiveRun; - activerun->timeStart = timeStamp; - activerun->cfg.loadConfigurationRun3(cfg); - activerun->cfg.printStream(std::cout); - // - uint32_t runnumber = activerun->cfg.getRunNumber(); - activerun->scalers.setRunNumber(runnumber); - activerun->scalers.setClassMask(activerun->cfg.getTriggerClassMask()); - o2::detectors::DetID::mask_t detmask = activerun->cfg.getDetectorMask(); - activerun->scalers.setDetectorMask(detmask); - // - mRunsLoaded[runnumber] = activerun; - saveRunConfigToCCDB(&activerun->cfg, timeStamp); - return 0; -} -int CTPRunManager::startRun(const std::string& cfg) -{ - return 0; -} -int CTPRunManager::stopRun(uint32_t irun) -{ - LOG(info) << "Stopping run index: " << irun; - if (mActiveRuns[irun] == nullptr) { - LOG(error) << "No config for run index:" << irun; - return 1; + for (auto& cls : mCTPClasses) { + cls.descriptor = &mDescriptors[cls.descriptorIndex]; } - const auto now = std::chrono::system_clock::now(); - const long timeStamp = std::chrono::duration_cast(now.time_since_epoch()).count(); - mActiveRuns[irun]->timeStop = timeStamp; - saveRunScalersToCCDB(irun); - delete mActiveRuns[irun]; - mActiveRuns[irun] = nullptr; return 0; } -int CTPRunManager::addScalers(uint32_t irun, std::time_t time) +int CTPConfiguration::checkConfigConsistency() const { - if (mActiveRuns[irun] == nullptr) { - LOG(error) << "No config for run index:" << irun; - return 1; - } - std::string orb = "extorb"; - CTPScalerRecordRaw scalrec; - scalrec.epochTime = time; - std::vector clslist = mActiveRuns[irun]->cfg.getTriggerClassList(); - for (auto const& cls : clslist) { - std::string cmb = "clamb" + std::to_string(cls + 1); - std::string cma = "clama" + std::to_string(cls + 1); - std::string c0b = "cla0b" + std::to_string(cls + 1); - std::string c0a = "cla0a" + std::to_string(cls + 1); - std::string c1b = "cla1b" + std::to_string(cls + 1); - std::string c1a = "cla1a" + std::to_string(cls + 1); - CTPScalerRaw scalraw; - scalraw.classIndex = (uint32_t)cls; - // std::cout << "cls:" << cls << " " << scalraw.classIndex << std::endl; - scalraw.lmBefore = mCounters[mScalerName2Position[cmb]]; - scalraw.lmAfter = mCounters[mScalerName2Position[cma]]; - scalraw.l0Before = mCounters[mScalerName2Position[c0b]]; - scalraw.l0After = mCounters[mScalerName2Position[c0a]]; - scalraw.l1Before = mCounters[mScalerName2Position[c1b]]; - scalraw.l1After = mCounters[mScalerName2Position[c1a]]; - // std::cout << "positions:" << cmb << " " << mScalerName2Position[cmb] << std::endl; - // std::cout << "positions:" << cma << " " << mScalerName2Position[cma] << std::endl; - scalrec.scalers.push_back(scalraw); + LOG(info) << "Checking consistency run:" << mRunNumber; + int ret = 0; + // All inputs used ? + // std::map inputs; + std::map inputs; + for (auto const& inp : mInputs) { + inputs[inp.name] = 0; } - // detectors - // std::vector detlist = mActiveRuns[irun]->cfg.getDetectorList(); - o2::detectors::DetID::mask_t detmask = mActiveRuns[irun]->cfg.getDetectorMask(); - for (uint32_t i = 0; i < 32; i++) { - o2::detectors::DetID::mask_t deti = 1ul << i; - bool detin = (detmask & deti).count(); - if (detin) { - std::string detname(o2::detectors::DetID::getName(i)); - std::string countername = "ltg" + CTPConfiguration::detName2LTG.at(detname) + "_PH"; - uint32_t detcount = mCounters[mScalerName2Position[countername]]; - scalrec.scalersDets.push_back(detcount); - // LOG(info) << "Scaler for detector:" << countername << ":" << detcount; + // Are all descriptors used + // std::map descs; + std::map descs; + for (auto const& desc : mDescriptors) { + descs[desc.name] = 0; + // std::cout << "1 " << &desc << std::endl; + for (auto const inp : desc.inputs) { + inputs[inp->name] += 1; } } + std::cout << "desc1:" << descs.size() << std::endl; // - scalrec.intRecord.orbit = mCounters[mScalerName2Position[orb]]; - scalrec.intRecord.bc = 0; - mActiveRuns[irun]->scalers.addScalerRacordRaw(scalrec); - LOG(info) << "Adding scalers for orbit:" << scalrec.intRecord.orbit; - // scalrec.printStream(std::cout); - // printCounters(); - return 0; -} -int CTPRunManager::processMessage(std::string& topic, const std::string& message) -{ - if (mQC == 1) { - LOG(info) << "processMessage: skipping, QC=1"; - return 1; - } - LOG(info) << "Processing message with topic:" << topic; - std::string firstcounters; - if (topic.find("ctpconfig") != std::string::npos) { - LOG(info) << "ctpcfg received"; - loadRun(message); - return 0; - } - if (topic.find("sox") != std::string::npos) { - // get config - size_t irun = message.find("run"); - if (irun == std::string::npos) { - LOG(error) << "run keyword not found in SOX:\n" - << message; - return 1; + for (const auto& cls : mCTPClasses) { + if (cls.classMask == 0) { + std::cout << "ERROR class:" << cls.name << " NO CLASS MASK" << std::endl; + ret++; } - LOG(info) << "SOX received, Run keyword position:" << irun; - std::string cfg = message.substr(irun, message.size() - irun); - startRun(cfg); - firstcounters = message.substr(0, irun); - } - if (topic.find("eox") != std::string::npos) { - LOG(info) << "EOX received"; - mEOX = 1; - } - // - std::vector tokens; - if (firstcounters.size() > 0) { - tokens = o2::utils::Str::tokenize(firstcounters, ' '); - } else { - tokens = o2::utils::Str::tokenize(message, ' '); - } - if (tokens.size() != (CTPRunScalers::NCOUNTERS + 1)) { - LOG(error) << "Scalers size wrong:" << tokens.size() << " expected:" << CTPRunScalers::NCOUNTERS + 1; - return 1; - } - double timeStamp = std::stold(tokens.at(0)); - std::time_t tt = timeStamp; - LOG(info) << "Processing scalers, all good, time:" << tokens.at(0) << " " << std::asctime(std::localtime(&tt)); - for (int i = 1; i < tokens.size(); i++) { - mCounters[i - 1] = std::stoull(tokens.at(i)); - if (i < (NRUNS + 1)) { - std::cout << mCounters[i - 1] << " "; + if (cls.cluster == nullptr) { + std::cout << "ERROR class:" << cls.name << " NO CLUSTER" << std::endl; + ret++; } - } - std::cout << std::endl; - LOG(info) << "Counter size:" << tokens.size(); - // - for (uint32_t i = 0; i < NRUNS; i++) { - if ((mCounters[i] == 0) && (mActiveRunNumbers[i] == 0)) { - // not active - } else if ((mCounters[i] != 0) && (mActiveRunNumbers[i] == mCounters[i])) { - // active , do scalers - LOG(info) << "Run continue:" << mCounters[i]; - addScalers(i, tt); - } else if ((mCounters[i] != 0) && (mActiveRunNumbers[i] == 0)) { - LOG(info) << "Run started:" << mCounters[i]; - auto run = mRunsLoaded.find(mCounters[i]); - if (run != mRunsLoaded.end()) { - mActiveRunNumbers[i] = mCounters[i]; - mActiveRuns[i] = run->second; - mRunsLoaded.erase(run); - addScalers(i, tt); - } else { - LOG(error) << "Trying to start run which is not loaded:" << mCounters[i]; - } - } else if ((mCounters[i] == 0) && (mActiveRunNumbers[i] != 0)) { - if (mEOX != 1) { - LOG(error) << "Internal error in processMessage: mEOX != 1 expected 0: mEOX:" << mEOX; - } - LOG(info) << "Run stopped:" << mActiveRunNumbers[i]; - addScalers(i, tt); - mActiveRunNumbers[i] = 0; - mEOX = 0; - stopRun(i); + if (cls.clusterIndex == 0xff) { + std::cout << "ERROR class:" << cls.name << " NO CLUSTER INDEX" << std::endl; + ret++; } - } - mEOX = 0; - printActiveRuns(); - return 0; -} -void CTPRunManager::printActiveRuns() const -{ - std::cout << "Active runs:"; - for (auto const& arun : mActiveRunNumbers) { - std::cout << arun << " "; - } - std::cout << " #loaded runs:" << mRunsLoaded.size(); - for (auto const& lrun : mRunsLoaded) { - std::cout << " " << lrun.second->cfg.getRunNumber(); - } - std::cout << std::endl; -} -int CTPRunManager::saveRunScalersToCCDB(int i) -{ - // data base - CTPActiveRun* run = mActiveRuns[i]; - using namespace std::chrono_literals; - std::chrono::seconds days3 = 259200s; - std::chrono::seconds min10 = 600s; - long time3days = std::chrono::duration_cast(days3).count(); - long time10min = std::chrono::duration_cast(min10).count(); - long tmin = run->timeStart - time10min; - long tmax = run->timeStop + time3days; - o2::ccdb::CcdbApi api; - map metadata; // can be empty - metadata["runNumber"] = std::to_string(run->cfg.getRunNumber()); - api.init(mCCDBHost.c_str()); // or http://localhost:8080 for a local installation - // store abitrary user object in strongly typed manner - api.storeAsTFileAny(&(run->scalers), mCCDBPathCTPScalers, metadata, tmin, tmax); - LOG(info) << "CTP scalers saved in ccdb:" << mCCDBHost << " run:" << run->cfg.getRunNumber() << " tmin:" << tmin << " tmax:" << tmax; - return 0; -} -int CTPRunManager::saveRunConfigToCCDB(CTPConfiguration* cfg, long timeStart) -{ - // data base - using namespace std::chrono_literals; - std::chrono::seconds days3 = 259200s; - std::chrono::seconds min10 = 600s; - long time3days = std::chrono::duration_cast(days3).count(); - long time10min = std::chrono::duration_cast(min10).count(); - long tmin = timeStart - time10min; - long tmax = timeStart + time3days; - o2::ccdb::CcdbApi api; - map metadata; // can be empty - metadata["runNumber"] = std::to_string(cfg->getRunNumber()); - api.init(mCCDBHost.c_str()); // or http://localhost:8080 for a local installation - // store abitrary user object in strongly typed manner - api.storeAsTFileAny(cfg, CCDBPathCTPConfig, metadata, tmin, tmax); - LOG(info) << "CTP config saved in ccdb:" << mCCDBHost << " run:" << cfg->getRunNumber() << " tmin:" << tmin << " tmax:" << tmax; - return 0; -} -CTPConfiguration CTPRunManager::getConfigFromCCDB(long timestamp, std::string run) -{ - auto& mgr = o2::ccdb::BasicCCDBManager::instance(); - mgr.setURL(mCCDBHost); - map metadata; // can be empty - metadata["runNumber"] = run; - auto ctpconfigdb = mgr.getSpecific(CCDBPathCTPConfig, timestamp, metadata); - if (ctpconfigdb == nullptr) { - LOG(info) << "CTP config not in database, timestamp:" << timestamp; - } else { - ctpconfigdb->printStream(std::cout); - } - return *ctpconfigdb; -} -CTPRunScalers CTPRunManager::getScalersFromCCDB(long timestamp, std::string run) -{ - auto& mgr = o2::ccdb::BasicCCDBManager::instance(); - mgr.setURL(mCCDBHost); - map metadata; // can be empty - metadata["runNumber"] = run; - auto ctpscalers = mgr.getSpecific(mCCDBPathCTPScalers, timestamp, metadata); - if (ctpscalers == nullptr) { - LOG(info) << "CTPRunScalers not in database, timestamp:" << timestamp; - } else { - // ctpscalers->printStream(std::cout); - } - return *ctpscalers; -} -int CTPRunManager::loadScalerNames() -{ - if (CTPRunScalers::NCOUNTERS != CTPRunScalers::scalerNames.size()) { - LOG(fatal) << "NCOUNTERS:" << CTPRunScalers::NCOUNTERS << " different from names vector:" << CTPRunScalers::scalerNames.size(); - return 1; - } - // try to open files of no success use default - for (uint32_t i = 0; i < CTPRunScalers::scalerNames.size(); i++) { - mScalerName2Position[CTPRunScalers::scalerNames[i]] = i; - } - return 0; -} -void CTPRunManager::printCounters() -{ - int NDET = 18; - int NINPS = 48; - int NCLKFP = 7; - int NLTG_start = NRUNS; - int NCLKFP_start = NLTG_start + NDET * 32; - int NINPS_start = NCLKFP_start + 7; - int NCLS_start = NINPS_start + NINPS; - std::cout << "====> CTP counters:" << std::endl; - std::cout << "RUNS:" << std::endl; - int ipos = 0; - for (int i = 0; i < NRUNS; i++) { - std::cout << ipos << ":" << mCounters[i] << " "; - ipos++; - } - std::cout << std::endl; - for (int i = 0; i < NDET; i++) { - std::cout << "LTG" << i + 1 << std::endl; - for (int j = NLTG_start + i * 32; j < NLTG_start + (i + 1) * 32; j++) { - std::cout << ipos << ":" << mCounters[j] << " "; - ipos++; + if (cls.descriptor == nullptr) { + std::cout << "ERROR class:" << cls.name << " NO DESCRIPTOR" << std::endl; + ret++; + } else { + descs[cls.descriptor->name] += 1; + // std::cout << "2 " << cls.descriptor << std::endl; + } + if (cls.descriptorIndex == 0xff) { + std::cout << "ERROR class:" << cls.name << " NO DESCRIPTOR INDEX" << std::endl; + ret++; + } else { + // std::cout << "3 " << &mDescriptors[cls.descriptorIndex] << std::endl; } - std::cout << std::endl; - } - std::cout << "BC40,BC240,Orbit,pulser, fastlm, busy,spare" << std::endl; - for (int i = NCLKFP_start; i < NCLKFP_start + 7; i++) { - std::cout << ipos << ":" << mCounters[i] << " "; - ipos++; - } - std::cout << std::endl; - std::cout << "INPUTS:" << std::endl; - for (int i = NINPS_start; i < NINPS_start + NINPS; i++) { - std::cout << ipos << ":" << mCounters[i] << " "; - ipos++; - } - std::cout << std::endl; - std::cout << "CLASS M Before" << std::endl; - for (int i = NCLS_start; i < NCLS_start + 64; i++) { - std::cout << ipos << ":" << mCounters[i] << " "; - ipos++; - } - std::cout << std::endl; - std::cout << "CLASS M After" << std::endl; - for (int i = NCLS_start + 64; i < NCLS_start + 2 * 64; i++) { - std::cout << ipos << ":" << mCounters[i] << " "; - ipos++; - } - std::cout << std::endl; - std::cout << "CLASS 0 Before" << std::endl; - for (int i = NCLS_start + 2 * 64; i < NCLS_start + 3 * 64; i++) { - std::cout << ipos << ":" << mCounters[i] << " "; - ipos++; - } - std::cout << std::endl; - std::cout << "CLASS 0 After" << std::endl; - for (int i = NCLS_start + 3 * 64; i < NCLS_start + 4 * 64; i++) { - std::cout << ipos << ":" << mCounters[i] << " "; - ipos++; - } - std::cout << std::endl; - std::cout << "CLASS 1 Before" << std::endl; - for (int i = NCLS_start + 4 * 64; i < NCLS_start + 5 * 64; i++) { - std::cout << ipos << ":" << mCounters[i] << " "; - ipos++; } - std::cout << std::endl; - std::cout << "CLASS 1 After" << std::endl; - for (int i = NCLS_start + 5 * 64; i < NCLS_start + 6 * 64; i++) { - std::cout << ipos << ":" << mCounters[i] << " "; - ipos++; + int iw = 0; + for (auto const& inp : inputs) { + if (inp.second == 0) { + iw++; + std::cout << "WARNING inputs:"; + } + std::cout << inp.first << " " << inp.second << std::endl; } - std::cout << std::endl; - std::cout << " REST:" << std::endl; - for (int i = NCLS_start + 6 * 64; i < mCounters.size(); i++) { - if ((ipos % 10) == 0) { - std::cout << std::endl; - std::cout << ipos << ":"; + std::cout << "Descriptors check:" << descs.size() << std::endl; + for (auto const& desc : descs) { + if (desc.second == 0) { + iw++; + std::cout << "WARNING descriptors:"; } - std::cout << mCounters[i] << " "; + // std::cout << (desc.first)->name << " " << desc.second << std::endl; + std::cout << (desc.first) << " " << desc.second << std::endl; } + std::cout << "CTP Config consistency checked. WARNINGS:" << iw << " ERRORS:" << ret << std::endl; + return ret; } +void CTPConfiguration::printConfigString() +{ + std::cout << mConfigString << std::endl; +}; +// int CTPInputsConfiguration::createInputsConfigFromFile(std::string& filename) { int ret = 0; @@ -974,8 +1105,11 @@ int CTPInputsConfiguration::createInputsConfigFromFile(std::string& filename) ret++; } return ret; - ; } +/// +/// CTP inputs config +/// Only default used. +/// CTPInputsConfiguration CTPInputsConfiguration::defaultInputConfig; void CTPInputsConfiguration::printStream(std::ostream& stream) const { @@ -983,43 +1117,134 @@ void CTPInputsConfiguration::printStream(std::ostream& stream) const input.printStream(stream); } } +// +// EMBA - software generated input for EMC - Min Bias Accepted +// +const std::vector CTPInputsConfiguration::CTPInputsDefault = + { + CTPInput("MT0A", "FT0", 1), CTPInput("MT0C", "FT0", 2), CTPInput("MTVX", "FT0", 3), CTPInput("MTSC", "FT0", 4), CTPInput("MTCE", "FT0", 5), + CTPInput("MVBA", "FV0", 6), CTPInput("MVOR", "FV0", 7), CTPInput("MVIR", "FV0", 8), CTPInput("MVNC", "FV0", 9), CTPInput("MVCH", "FV0", 10), + CTPInput("0UCE", "FDD", 13), CTPInput("0USC", "FDD", 15), CTPInput("0UVX", "FDD", 16), CTPInput("0U0C", "FDD", 17), CTPInput("0U0A", "FDD", 18), + CTPInput("0DMC", "EMC", 14), CTPInput("0DJ1", "EMC", 41), CTPInput("0DG1", "EMC", 42), CTPInput("0DJ2", "EMC", 43), CTPInput("0DG2", "EMC", 44), + CTPInput("0EMC", "EMC", 21), CTPInput("0EJ1", "EMC", 37), CTPInput("0EG1", "EMC", 38), CTPInput("0EJ2", "EMC", 39), CTPInput("0EG2", "EMC", 40), + CTPInput("0PH0", "PHS", 22), CTPInput("1PHL", "PHS", 27), CTPInput("1PHH", "PHS", 28), CTPInput("1PHL", "PHM", 29), + CTPInput("1ZED", "ZDC", 25), CTPInput("1ZNC", "ZDC", 26), CTPInput("EMBA", "EMC", 48)}; void CTPInputsConfiguration::initDefaultInputConfig() { - defaultInputConfig.CTPInputs.push_back(CTPInput("MT0A", "FT0", 1)); - defaultInputConfig.CTPInputs.push_back(CTPInput("MT0C", "FT0", 2)); - defaultInputConfig.CTPInputs.push_back(CTPInput("MTVX", "FT0", 3)); - defaultInputConfig.CTPInputs.push_back(CTPInput("MTSC", "FT0", 4)); - defaultInputConfig.CTPInputs.push_back(CTPInput("MTCE", "FT0", 5)); - // - defaultInputConfig.CTPInputs.push_back(CTPInput("MVBA", "FV0", 6)); - defaultInputConfig.CTPInputs.push_back(CTPInput("MVOR", "FV0", 7)); - defaultInputConfig.CTPInputs.push_back(CTPInput("MVIR", "FV0", 8)); - defaultInputConfig.CTPInputs.push_back(CTPInput("MVNC", "FV0", 9)); - defaultInputConfig.CTPInputs.push_back(CTPInput("MVCH", "FV0", 10)); - // - defaultInputConfig.CTPInputs.push_back(CTPInput("0UCE", "FT0", 13)); - defaultInputConfig.CTPInputs.push_back(CTPInput("0USC", "FT0", 15)); - defaultInputConfig.CTPInputs.push_back(CTPInput("0UVX", "FT0", 16)); - defaultInputConfig.CTPInputs.push_back(CTPInput("0U0C", "FT0", 17)); - defaultInputConfig.CTPInputs.push_back(CTPInput("0U0A", "FT0", 18)); - // - defaultInputConfig.CTPInputs.push_back(CTPInput("0DMC", "EMC", 14)); - defaultInputConfig.CTPInputs.push_back(CTPInput("0DJ1", "EMC", 41)); - defaultInputConfig.CTPInputs.push_back(CTPInput("0DG1", "EMC", 42)); - defaultInputConfig.CTPInputs.push_back(CTPInput("0DJ2", "EMC", 43)); - defaultInputConfig.CTPInputs.push_back(CTPInput("0DG2", "EMC", 44)); - // - defaultInputConfig.CTPInputs.push_back(CTPInput("0EMC", "EMC", 21)); - defaultInputConfig.CTPInputs.push_back(CTPInput("0EJ1", "EMC", 37)); - defaultInputConfig.CTPInputs.push_back(CTPInput("0EG1", "EMC", 38)); - defaultInputConfig.CTPInputs.push_back(CTPInput("0EJ2", "EMC", 39)); - defaultInputConfig.CTPInputs.push_back(CTPInput("0EG2", "EMC", 40)); - // - defaultInputConfig.CTPInputs.push_back(CTPInput("0PH0", "PHS", 22)); - defaultInputConfig.CTPInputs.push_back(CTPInput("1PHL", "PHS", 27)); - defaultInputConfig.CTPInputs.push_back(CTPInput("1PHH", "PHS", 28)); - defaultInputConfig.CTPInputs.push_back(CTPInput("1PHL", "PHM", 29)); - // - defaultInputConfig.CTPInputs.push_back(CTPInput("1ZED", "ZDC", 25)); - defaultInputConfig.CTPInputs.push_back(CTPInput("1ZNC", "ZDC", 26)); + defaultInputConfig.CTPInputs = CTPInputsConfiguration::CTPInputsDefault; } +/// Return input name from default inputs configuration. +/// Take into account convention that L0 inputs has (index+100) in the first version of CTP config file (*.rcfg) +std::string CTPInputsConfiguration::getInputNameFromIndex100(uint32_t index) +{ + uint32_t indexcor = index; + if (index > 100) { + indexcor = index - 100; + } + for (auto& inp : defaultInputConfig.CTPInputs) { + if (inp.getIndex() == indexcor) { + std::string name = inp.name; + if (index > 100) { + name[0] = '0'; + } + return name; + } + } + LOG(info) << "Input with index:" << index << " not in deafult input config"; + return ""; +} +/// Return input name from default inputs configuration. +/// Index has to be in range [1::48] +std::string CTPInputsConfiguration::getInputNameFromIndex(uint32_t index) +{ + if (index > o2::ctp::CTP_NINPUTS) { + LOG(warn) << "getInputNameFRomIndex: index too big:" << index; + return "none"; + } + for (auto& inp : o2::ctp::CTPInputsConfiguration::CTPInputsDefault) { + if (inp.getIndex() == index) { + std::string name = inp.name; + return name; + } + } + LOG(info) << "Input with index:" << index << " not in deafult input config"; + return "none"; +} +int CTPInputsConfiguration::getInputIndexFromName(std::string& name) +{ + std::string namecorr = name; + if ((name[0] == '0') || (name[0] == 'M') || (name[0] == '1')) { + namecorr = namecorr.substr(1, namecorr.size() - 1); + } else { + LOG(warn) << "Input name without level:" << name; + } + for (auto& inp : o2::ctp::CTPInputsConfiguration::CTPInputsDefault) { + if (inp.name.find(namecorr) != std::string::npos) { + return inp.getIndex(); + } + } + LOG(warn) << "Input with name:" << name << " not in default input config"; + return 0xff; +} + +int CtpCfg::readAndSave(std::string& path) +{ + std::string file = path + filename; + std::ifstream ctpcfg(file); + if (ctpcfg.is_open()) { + std::string line; + while (std::getline(ctpcfg, line)) { + o2::utils::Str::trim(line); + if (line.size() == 0) { + continue; + } + if (line[0] == '#') { + continue; + } + std::vector tokens = o2::utils::Str::tokenize(line, ' '); + size_t ntokens = tokens.size(); + if (ntokens < 2) { + LOG(warn) << "Not enough tokens"; + continue; + } + if (tokens[0].find("TForbits") != std::string::npos) { + TFOrbits = std::atol(tokens[1].c_str()); + } else if (tokens[0].find("ccdb") != std::string::npos) { + ccdb = std::atoi(tokens[1].c_str()); + } else if (tokens[0].find("orbitshift") != std::string::npos) { + orbitShift = std::atol(tokens[1].c_str()); + } else if (tokens[0].find("ir_inputs") != std::string::npos) { + irInputs_1_24 = std::stoul(tokens[2].c_str(), nullptr, 16); + irInputs_25_48 = std::stoul(tokens[1].c_str(), nullptr, 16); + } else { + LOG(warn) << " Token not found:" << tokens[0]; + } + } + LOG(warn) << "Open file success:" << file; + } else { + LOG(warn) << "Can not open file:" << file; + return 1; + } + return 0; +} +std::vector CtpCfg::listOfUsedInputs() +{ + std::cout << std::hex << "0x" << irInputs_1_24 << " " << irInputs_25_48 << std::dec << std::endl; + std::vector inputList; + for (int i = 0; i < 24; i++) { + if ((1ul << i) & irInputs_1_24) { + inputList.push_back(i); + } + } + for (int i = 0; i < 24; i++) { + if ((1ul << i) & irInputs_25_48) { + inputList.push_back(i + 24); + } + } + return inputList; +} +std::ostream& o2::ctp::operator<<(std::ostream& in, const o2::ctp::CTPConfiguration& conf) +{ + conf.printStream(in); + return in; +} \ No newline at end of file diff --git a/DataFormats/Detectors/CTP/src/DataFormatsCTPLinkDef.h b/DataFormats/Detectors/CTP/src/DataFormatsCTPLinkDef.h index 43648feba3ea2..ac2a83d31edda 100644 --- a/DataFormats/Detectors/CTP/src/DataFormatsCTPLinkDef.h +++ b/DataFormats/Detectors/CTP/src/DataFormatsCTPLinkDef.h @@ -14,6 +14,7 @@ #pragma link off all globals; #pragma link off all classes; #pragma link off all functions; +#pragma link C++ class std::bitset < 48> + ; #pragma link C++ class o2::ctp::CTPDigit + ; #pragma link C++ class vector < o2::ctp::CTPDigit> + ; #pragma link C++ class o2::ctp::CTPInputDigit + ; @@ -43,7 +44,9 @@ #pragma link C++ class o2::ctp::CTPScalerRecordO2 + ; #pragma link C++ class vector < o2::ctp::CTPScalerRecordO2> + ; #pragma link C++ class o2::ctp::CTPRunScalers + ; -#pragma link C++ class o2::ctp::CTPRunManager + ; +#pragma link C++ class o2::ctp::LumiInfo + ; +#pragma link C++ class vector < o2::ctp::LumiInfo> + ; +#pragma link C++ class o2::ctp::CTPRateFetcher + ; #pragma link C++ struct o2::ctp::CTFHeader + ; #pragma link C++ struct o2::ctp::CTF + ; @@ -52,4 +55,6 @@ #pragma link C++ class o2::ctp::TriggerOffsetsParam + ; #pragma link C++ class o2::conf::ConfigurableParamHelper < o2::ctp::TriggerOffsetsParam> + ; +#pragma link C++ class o2::ctp::CtpCfg + ; + #endif diff --git a/DataFormats/Detectors/CTP/src/Digits.cxx b/DataFormats/Detectors/CTP/src/Digits.cxx index aa44216270721..205225a35500a 100644 --- a/DataFormats/Detectors/CTP/src/Digits.cxx +++ b/DataFormats/Detectors/CTP/src/Digits.cxx @@ -30,13 +30,13 @@ void CTPDigit::printStream(std::ostream& stream) const void CTPDigit::setInputMask(gbtword80_t mask) { - for (int i = 0; i < CTP_NINPUTS; i++) { + for (uint32_t i = 0; i < CTP_NINPUTS; i++) { CTPInputMask[i] = mask[i]; } } void CTPDigit::setClassMask(gbtword80_t mask) { - for (int i = 0; i < CTP_NCLASSES; i++) { + for (uint32_t i = 0; i < CTP_NCLASSES; i++) { CTPClassMask[i] = mask[i]; } } diff --git a/DataFormats/Detectors/CTP/src/LumiInfo.cxx b/DataFormats/Detectors/CTP/src/LumiInfo.cxx new file mode 100644 index 0000000000000..08f8e5e7b0279 --- /dev/null +++ b/DataFormats/Detectors/CTP/src/LumiInfo.cxx @@ -0,0 +1,23 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file LumiInfo.cxx + +#include "DataFormatsCTP/LumiInfo.h" +#include "DataFormatsCTP/Configuration.h" +#include + +using namespace o2::ctp; + +void LumiInfo::printInputs() const +{ + LOG(info) << "Lumi inp1:" << inp1 << ":" << o2::ctp::CTPInputsConfiguration::getInputNameFromIndex(inp1) << " inp2:" << inp2 << ":" << o2::ctp::CTPInputsConfiguration::getInputNameFromIndex(inp2); +} diff --git a/DataFormats/Detectors/CTP/src/Scalers.cxx b/DataFormats/Detectors/CTP/src/Scalers.cxx index cb088b27c7367..256722fc1e5ae 100644 --- a/DataFormats/Detectors/CTP/src/Scalers.cxx +++ b/DataFormats/Detectors/CTP/src/Scalers.cxx @@ -16,15 +16,24 @@ #include #include #include "CommonUtils/StringUtils.h" -#include "FairLogger.h" +#include using namespace o2::ctp; - +void errorCounters::printStream(std::ostream& stream) const +{ + // stream << "Counter warnings diff 1 lmBlmA:" << lmBlmAd1 << " lmAl0B:" << lmAl0Bd1 << " l0Bl0A:" << l0Bl0Ad1 << " l0Al1B: " << l0Al1Bd1 << " l1Bl1A:" << l1Bl1Ad1; + // stream << std::endl; + if (lmB + l0B + l1B + lmA + l0A + l1A + lmBlmA + lmAl0B + l0Bl0A + l0Al1B + l1Bl1A) { + stream << "Counter errorrs: lmB:" << lmB << " l0B:" << l0B << " l1B:" << l1B << " lmA:" << lmA << " l0A:" << l0A << " l1A:" << l1A; + stream << " lmBlmA:" << lmBlmA << " lmAl0B:" << lmAl0B << " l0Bl0A:" << l0Bl0A << " l0Al1B: " << l0Al1B << " l1Bl1A:" << l1Bl1A; + stream << std::endl; + } +} void CTPScalerRaw::printStream(std::ostream& stream) const { stream << "Class:" << std::setw(2) << classIndex << " RAW"; stream << " LMB:" << std::setw(10) << lmBefore << " LMA:" << std::setw(10) << lmAfter; - stream << " LOB:" << std::setw(10) << l0Before << " L0A:" << std::setw(10) << l0After; + stream << " L0B:" << std::setw(10) << l0Before << " L0A:" << std::setw(10) << l0After; stream << " L1B:" << std::setw(10) << l1Before << " L1A:" << std::setw(10) << l1After << std::endl; } // @@ -43,9 +52,16 @@ void CTPScalerO2::printStream(std::ostream& stream) const { stream << "Class:" << std::setw(2) << classIndex << " O2"; stream << " LMB:" << std::setw(10) << lmBefore << " LMA:" << std::setw(10) << lmAfter; - stream << " LOB:" << std::setw(10) << l0Before << " L0A:" << std::setw(10) << l0After; + stream << " L0B:" << std::setw(10) << l0Before << " L0A:" << std::setw(10) << l0After; stream << " L1B:" << std::setw(10) << l1Before << " L1A:" << std::setw(10) << l1After << std::endl; } +void CTPScalerO2::printFromZero(std::ostream& stream, CTPScalerO2& scaler0) const +{ + stream << "Class:" << std::setw(2) << classIndex << " O2"; + stream << " LMB:" << std::setw(10) << lmBefore - scaler0.lmBefore << " LMA:" << std::setw(10) << lmAfter - scaler0.lmAfter; + stream << " LOB:" << std::setw(10) << l0Before - scaler0.l0Before << " L0A:" << std::setw(10) << l0After - scaler0.l0After; + stream << " L1B:" << std::setw(10) << l1Before - scaler0.l1Before << " L1A:" << std::setw(10) << l1After - scaler0.l1After << std::endl; +} void CTPScalerRecordRaw::printStream(std::ostream& stream) const { stream << "Orbit:" << intRecord.orbit << " BC:" << intRecord.bc; @@ -53,7 +69,8 @@ void CTPScalerRecordRaw::printStream(std::ostream& stream) const for (auto const& cnts : scalers) { cnts.printStream(stream); } - for (auto const& dets : scalersDets) { + std::cout << "Inputs:" << scalersInps.size() << std::endl; + for (auto const& dets : scalersInps) { stream << dets << " "; } stream << std::endl; @@ -65,11 +82,22 @@ void CTPScalerRecordO2::printStream(std::ostream& stream) const for (auto const& cnts : scalers) { cnts.printStream(stream); } - for (auto const& dets : scalersDets) { + std::cout << "Inputs:" << scalersInps.size() << std::endl; + for (auto const& dets : scalersInps) { stream << dets << " "; } stream << std::endl; } +void CTPScalerRecordO2::printFromZero(std::ostream& stream, CTPScalerRecordO2& record0) const +{ + stream << "printFromZero Orbit:" << intRecord.orbit - record0.intRecord.orbit << " BC:" << intRecord.bc; + stream << " miliSeconds:" << std::setprecision(20) << epochTime - record0.epochTime << std::endl; + // this-record0 + for (uint32_t i = 0; i < scalers.size(); i++) { + scalers[i].printFromZero(stream, record0.scalers[i]); + } + stream << std::endl; +} // // CTPRunScalers // @@ -86,10 +114,27 @@ void CTPRunScalers::printStream(std::ostream& stream) const rec.printStream(stream); } } +void CTPRunScalers::printO2(std::ostream& stream) const +{ + stream << "CTP Scalers (version:" << mVersion << ") Run:" << mRunNumber << std::endl; + stream << "O2 Counters:" << std::endl; + for (auto const& rec : mScalerRecordO2) { + rec.printStream(stream); + } +} +void CTPRunScalers::printFromZero(std::ostream& stream) const +{ + stream << "CTP Scalers (version:" << mVersion << ") Run:" << mRunNumber << std::endl; + stream << "O2 Counters:" << std::endl; + CTPScalerRecordO2 record0 = mScalerRecordO2[0]; + for (auto const& rec : mScalerRecordO2) { + rec.printFromZero(stream, record0); + } +} void CTPRunScalers::printClasses(std::ostream& stream) const { stream << "CTP classes:"; - for (int i = 0; i < mClassMask.size(); i++) { + for (uint32_t i = 0; i < mClassMask.size(); i++) { if (mClassMask[i]) { stream << " " << i; } @@ -106,6 +151,24 @@ std::vector CTPRunScalers::getClassIndexes() const } return indexes; } +// cls counted from 0 +int CTPRunScalers::getScalerIndexForClass(uint32_t cls) const +{ + if (cls >= 64) { + LOG(error) << "Class index out of range:" << cls; + return 255; + } + std::vector clslist = getClassIndexes(); + int i = 0; + for (auto const& clsl : clslist) { + if (cls == clsl) { + return i; + } + i++; + } + LOG(error) << " Class not found:" << cls; + return 255; +} int CTPRunScalers::readScalers(const std::string& rawscalers) { LOG(info) << "Loading CTP scalers."; @@ -113,7 +176,7 @@ int CTPRunScalers::readScalers(const std::string& rawscalers) int ret = 0; int level = 0; std::string line; - int nclasses = 0; + uint32_t nclasses = 0; int nlines = 0; while (std::getline(iss, line)) { o2::utils::Str::trim(line); @@ -133,7 +196,7 @@ int CTPRunScalers::readScalers(const std::string& rawscalers) } return 0; } -int CTPRunScalers::processScalerLine(const std::string& line, int& level, int& nclasses) +int CTPRunScalers::processScalerLine(const std::string& line, int& level, uint32_t& nclasses) { //std::cout << "Processing line" << std::endl; if (line.size() == 0) { @@ -166,13 +229,13 @@ int CTPRunScalers::processScalerLine(const std::string& line, int& level, int& n return 2; } else { mRunNumber = std::stol(tokens[0]); - int numofclasses = std::stoi(tokens[1]); + uint32_t numofclasses = std::stoi(tokens[1]); if ((numofclasses + 2) != ntokens) { LOG(error) << "Wrong syntax of second line in CTP scalers"; return 3; } mClassMask.reset(); - for (int i = 0; i < numofclasses; i++) { + for (uint32_t i = 0; i < numofclasses; i++) { int index = std::stoi(tokens[i + 2]); mClassMask[index] = 1; } @@ -234,23 +297,37 @@ int CTPRunScalers::convertRawToO2() overflows[i] = {0, 0, 0, 0, 0, 0}; } } + // Input overflows + std::array overflowsInputs = {48 * 0}; + errorCounters eCnts; // 1st o2 rec is just copy CTPScalerRecordO2 o2rec; - copyRawToO2ScalerRecord(mScalerRecordRaw[0], o2rec, overflows); + copyRawToO2ScalerRecord(mScalerRecordRaw[0], o2rec, overflows, overflowsInputs); mScalerRecordO2.push_back(o2rec); + int j = 1; for (uint32_t i = 1; i < mScalerRecordRaw.size(); i++) { //update overflows - updateOverflows(mScalerRecordRaw[i - 1], mScalerRecordRaw[i], overflows); + int ret = updateOverflows(mScalerRecordRaw[i - 1], mScalerRecordRaw[i], overflows); + // for(int k = 0; k < mClassMask.size(); k++) { + // if(mClassMask[k]) { + // LOG(info) << i << " " << k << " " << overflows[k][0] << " " << overflows[k][1] << " " << overflows[k][2] << " " << overflows[k][3] << " " << overflows[k][4] << " " << overflows[k][5]; + // } + // } // - CTPScalerRecordO2 o2rec; - copyRawToO2ScalerRecord(mScalerRecordRaw[i], o2rec, overflows); - mScalerRecordO2.push_back(o2rec); - // Check consistency - checkConsistency(mScalerRecordO2[i - 1], mScalerRecordO2[i]); + if (ret == 0) { + CTPScalerRecordO2 o2rec; + ret = updateOverflowsInps(mScalerRecordRaw[i - 1], mScalerRecordRaw[i], overflowsInputs); + copyRawToO2ScalerRecord(mScalerRecordRaw[i], o2rec, overflows, overflowsInputs); + mScalerRecordO2.push_back(o2rec); + // Check consistency + checkConsistency(mScalerRecordO2[j - 1], mScalerRecordO2[j], eCnts); + j++; + } } + eCnts.printStream(std::cout); return 0; } -int CTPRunScalers::copyRawToO2ScalerRecord(const CTPScalerRecordRaw& rawrec, CTPScalerRecordO2& o2rec, overflows_t& classesoverflows) +int CTPRunScalers::copyRawToO2ScalerRecord(const CTPScalerRecordRaw& rawrec, CTPScalerRecordO2& o2rec, overflows_t& classesoverflows, std::array& overflows) { if (rawrec.scalers.size() != (mClassMask.count())) { LOG(error) << "Inconsistent scaler record size:" << rawrec.scalers.size() << " Expected:" << mClassMask.count(); @@ -268,62 +345,111 @@ int CTPRunScalers::copyRawToO2ScalerRecord(const CTPScalerRecordRaw& rawrec, CTP o2scal.createCTPScalerO2FromRaw(rawscal, classesoverflows[k]); o2rec.scalers.push_back(o2scal); } + for (uint32_t i = 0; i < rawrec.scalersInps.size(); i++) { + uint64_t inpo2 = (uint64_t)(rawrec.scalersInps[i]) + 0xffffffffull * (uint64_t)(overflows[i]); + o2rec.scalersInps.push_back(inpo2); + } return 0; } -int CTPRunScalers::checkConsistency(const CTPScalerO2& scal0, const CTPScalerO2& scal1) const +int CTPRunScalers::checkConsistency(const CTPScalerO2& scal0, const CTPScalerO2& scal1, errorCounters& eCnts) const { int ret = 0; // Scaler should never decrease if (scal0.lmBefore > scal1.lmBefore) { - LOG(error) << "Scaler decreasing: Class:" << scal0.classIndex << " lmBefore 0:" << scal0.lmBefore << " lmBefore :" << scal1.lmBefore; + eCnts.lmB++; + if (eCnts.lmB < eCnts.MAXPRINT) { + LOG(error) << "Scaler decreasing: Class:" << scal0.classIndex << " lmBefore 0:" << scal0.lmBefore << " lmBefore :" << scal1.lmBefore; + } ret++; } if (scal0.l0Before > scal1.l0Before) { - LOG(error) << "Scaler decreasing: Class:" << scal0.classIndex << " lmBefore 0:" << scal0.l0Before << " lmBefore :" << scal1.l0Before; + eCnts.l0B++; + if (eCnts.l0B < eCnts.MAXPRINT) { + LOG(error) << "Scaler decreasing: Class:" << scal0.classIndex << " lmBefore 0:" << scal0.l0Before << " lmBefore :" << scal1.l0Before; + } ret++; } if (scal0.l1Before > scal1.l1Before) { - LOG(error) << "Scaler decreasing: Class:" << scal0.classIndex << " lmBefore 0:" << scal0.l1Before << " lmBefore :" << scal1.l1Before; + eCnts.l1B++; + if (eCnts.l1B < eCnts.MAXPRINT) { + LOG(error) << "Scaler decreasing: Class:" << scal0.classIndex << " lmBefore 0:" << scal0.l1Before << " lmBefore :" << scal1.l1Before; + } ret++; } if (scal0.lmAfter > scal1.lmAfter) { - LOG(error) << "Scaler decreasing: Class:" << scal0.classIndex << " lmAfter 0:" << scal0.lmAfter << " lmAfter :" << scal1.lmAfter; + eCnts.lmA++; + if (eCnts.lmA < eCnts.MAXPRINT) { + LOG(error) << "Scaler decreasing: Class:" << scal0.classIndex << " lmAfter 0:" << scal0.lmAfter << " lmAfter :" << scal1.lmAfter; + } ret++; } if (scal0.l0After > scal1.l0After) { - LOG(error) << "Scaler decreasing: Class:" << scal0.classIndex << " lmAfter 0:" << scal0.l0After << " lmAfter :" << scal1.l0After; + eCnts.l0A++; + if (eCnts.l0A < eCnts.MAXPRINT) { + LOG(error) << "Scaler decreasing: Class:" << scal0.classIndex << " lmAfter 0:" << scal0.l0After << " lmAfter :" << scal1.l0After; + } ret++; } if (scal0.l1After > scal1.l1After) { - LOG(error) << "Scaler decreasing: Class:" << scal0.classIndex << " lmAfter 0:" << scal0.l1After << " lmAfter :" << scal1.l1After; + eCnts.l1A++; + if (eCnts.l1A < eCnts.MAXPRINT) { + LOG(error) << "Scaler decreasing: Class:" << scal0.classIndex << " lmAfter 0:" << scal0.l1After << " lmAfter :" << scal1.l1After; + } ret++; } // // LMB >= LMA >= L0B >= L0A >= L1B >= L1A: 5 relations // broken for classes started at L0 // - if ((scal1.lmAfter - scal0.lmAfter) > (scal1.lmBefore - scal0.lmBefore)) { - LOG(error) << "LMA > LMB eerror:" << ((scal1.lmAfter - scal0.lmAfter) - (scal1.lmBefore - scal0.lmBefore)); + int64_t difThres = 6; + int64_t dif = (scal1.lmAfter - scal0.lmAfter) - (scal1.lmBefore - scal0.lmBefore); + if (dif <= difThres) { + eCnts.lmBlmAd1++; + } else if (dif > difThres) { + eCnts.lmBlmA++; + if (eCnts.lmBlmA < eCnts.MAXPRINT) { + LOG(error) << "LMA > LMB error:" << dif; + } ret++; } - if ((scal1.l0After - scal0.l0After) > (scal1.l0Before - scal0.l0Before)) { - LOG(error) << "L0A > L0B error:" << ((scal1.l0After - scal0.l0After) - (scal1.l0Before - scal0.l0Before)); + dif = (scal1.l0After - scal0.l0After) - (scal1.l0Before - scal0.l0Before); + if (dif <= difThres) { + eCnts.l0Bl0Ad1++; + } else if (dif > difThres) { + eCnts.l0Bl0A++; + if (eCnts.l0Bl0A < eCnts.MAXPRINT) { + LOG(error) << "L0A > L0B error:" << dif; + } ret++; } - if ((scal1.l1After - scal0.l1After) > (scal1.l1Before - scal0.l1Before)) { - LOG(error) << "L1A > L1B error:" << ((scal1.l0After - scal0.l0After) - (scal1.l0Before - scal0.l0Before)); + dif = (scal1.l0After - scal0.l0After) - (scal1.l0Before - scal0.l0Before); + if (dif <= difThres) { + eCnts.l1Bl1Ad1++; + } else if (dif > difThres) { + eCnts.l1Bl1A++; + if (eCnts.l1Bl1A < eCnts.MAXPRINT) { + LOG(error) << "L1A > L1B error:" << dif; + } ret++; } if ((scal1.l0Before - scal0.l0Before) > (scal1.lmAfter - scal0.lmAfter)) { // LOG(warning) << "L0B > LMA ok if L0 class."; // ret++; } - if ((scal1.l1Before - scal0.l1Before) > (scal1.l0After - scal0.l0After)) { - LOG(error) << "L1B > L0A Before error:" << ((scal1.l1Before - scal0.l1Before) - (scal1.l0After - scal0.l0After)); + dif = (scal1.l1Before - scal0.l1Before) - (scal1.l0After - scal0.l0After); + // LOG(info) << "L1B L0A " << dif << " " << scal1.l1Before << " " << scal1.l0After << " " << scal0.l1Before << " " << scal0.l0After; + if (dif <= difThres) { + eCnts.l0Al1Bd1++; + } else if (dif > difThres) { + eCnts.l0Al1B++; + if (eCnts.l0Al1B < eCnts.MAXPRINT) { + // LOG(error) << "L1B > L0A Before error:" << dif << " " << scal1.l1Before << " " << scal1.l0After << " " << scal0.l1Before << " " << scal0.l0After; + LOG(warning) << "L1B > L0A Before error:" << dif; + } ret++; } // - if (ret) { + if (ret < 0) { scal0.printStream(std::cout); scal1.printStream(std::cout); } @@ -335,18 +461,23 @@ int CTPRunScalers::updateOverflows(const CTPScalerRecordRaw& rec0, const CTPScal LOG(error) << "Inconsistent scaler record size:" << rec1.scalers.size() << " Expected:" << mClassMask.count(); return 1; } - for (int i = 0; i < rec0.scalers.size(); i++) { + if (rec0.intRecord.orbit > rec1.intRecord.orbit) { + LOG(warning) << "rec0 orbit:" << rec0.intRecord.orbit << "> rec1 orbit:" << rec1.intRecord.orbit << " skipping this record"; + return 1; + } + for (uint32_t i = 0; i < rec0.scalers.size(); i++) { int k = (getClassIndexes())[i]; updateOverflows(rec0.scalers[i], rec1.scalers[i], classesoverflows[k]); } return 0; } -int CTPRunScalers::checkConsistency(const CTPScalerRecordO2& rec0, const CTPScalerRecordO2& rec1) const +int CTPRunScalers::checkConsistency(const CTPScalerRecordO2& rec0, const CTPScalerRecordO2& rec1, errorCounters& eCnts) const { - for (int i = 0; i < rec0.scalers.size(); i++) { - checkConsistency(rec0.scalers[i], rec1.scalers[i]); + int ret = 0; + for (uint32_t i = 0; i < rec0.scalers.size(); i++) { + ret += checkConsistency(rec0.scalers[i], rec1.scalers[i], eCnts); } - return 0; + return ret; } int CTPRunScalers::updateOverflows(const CTPScalerRaw& scal0, const CTPScalerRaw& scal1, std::array& overflow) const { @@ -373,6 +504,33 @@ int CTPRunScalers::updateOverflows(const CTPScalerRaw& scal0, const CTPScalerRaw //std::cout << std::endl; return 0; } +// +int CTPRunScalers::updateOverflowsInps(const CTPScalerRecordRaw& rec0, const CTPScalerRecordRaw& rec1, std::array& overflow) const +{ + static int iPrint = 0; + uint32_t NINPS = 48; + if (rec0.scalersInps.size() < NINPS) { + if (iPrint < 1) { + LOG(warning) << "Input scalers not available. Size:" << rec0.scalersInps.size(); + iPrint++; + } + return 1; + } + if (rec1.scalersInps.size() < NINPS) { + if (iPrint < 1) { + LOG(warning) << "Input scalers not available. Size:" << rec0.scalersInps.size(); + iPrint++; + } + return 2; + } + for (uint32_t i = 0; i < NINPS; i++) { + if (rec0.scalersInps[i] > rec1.scalersInps[i]) { + overflow[i] += 1; + } + } + return 0; +} +// int CTPRunScalers::printRates() { if (mScalerRecordO2.size() == 0) { @@ -381,11 +539,15 @@ int CTPRunScalers::printRates() } LOG(info) << "Scaler rates for run:" << mRunNumber; CTPScalerRecordO2* scalrec0 = &mScalerRecordO2[0]; - for (int i = 1; i < mScalerRecordO2.size(); i++) { + uint32_t orbit0 = scalrec0->intRecord.orbit; + for (uint32_t i = 1; i < mScalerRecordO2.size(); i++) { CTPScalerRecordO2* scalrec1 = &mScalerRecordO2[i]; double_t tt = (double_t)(scalrec1->intRecord.orbit - scalrec0->intRecord.orbit); + double_t tinrun = (double_t)(scalrec1->intRecord.orbit - orbit0); tt = tt * 88e-6; - for (int j = 0; j < scalrec1->scalers.size(); j++) { + tinrun = tinrun * 88e-6; + std::cout << "==> Time wrt to SOR [s]:" << tinrun << " time intervale[s]:" << tt << std::endl; + for (uint32_t j = 0; j < scalrec1->scalers.size(); j++) { CTPScalerO2* s0 = &(scalrec0->scalers[j]); CTPScalerO2* s1 = &(scalrec1->scalers[j]); double_t rMB = (s1->lmBefore - s0->lmBefore) / tt; @@ -410,9 +572,11 @@ int CTPRunScalers::printIntegrals() LOG(info) << "ScalerRecord is empty, doing nothing"; return 0; } - LOG(info) << "Scaler Integrals for run:" << mRunNumber; + double_t time0 = mScalerRecordO2[0].epochTime; + double_t timeL = mScalerRecordO2[mScalerRecordO2.size() - 1].epochTime; + LOG(info) << "Scaler Integrals for run:" << mRunNumber << " duration:" << timeL - time0; - for (int i = 0; i < mScalerRecordO2[0].scalers.size(); i++) { + for (uint32_t i = 0; i < mScalerRecordO2[0].scalers.size(); i++) { std::cout << i << " LMB " << mScalerRecordO2[mScalerRecordO2.size() - 1].scalers[i].lmBefore - mScalerRecordO2[0].scalers[i].lmBefore << std::endl; std::cout << i << " LMA " << mScalerRecordO2[mScalerRecordO2.size() - 1].scalers[i].lmAfter - mScalerRecordO2[0].scalers[i].lmAfter << std::endl; std::cout << i << " L0B " << mScalerRecordO2[mScalerRecordO2.size() - 1].scalers[i].l0Before - mScalerRecordO2[0].scalers[i].l0Before << std::endl; @@ -422,8 +586,322 @@ int CTPRunScalers::printIntegrals() } return 0; } +// +// Input counting 1..48 +int CTPRunScalers::printInputRateAndIntegral(int inp) +{ + if (mScalerRecordO2.size() == 0) { + LOG(info) << "ScalerRecord is empty, doing nothing"; + return 1; + } + double_t time0 = mScalerRecordO2[0].epochTime; + double_t timeL = mScalerRecordO2[mScalerRecordO2.size() - 1].epochTime; + int integral = mScalerRecordO2[mScalerRecordO2.size() - 1].scalersInps[inp - 1] - mScalerRecordO2[0].scalersInps[inp - 1]; + std::cout << "Scaler Integrals for run:" << mRunNumber << " duration:" << timeL - time0; + std::cout << " Input " << inp << " integral:" << integral << " rate:" << integral / (timeL - time0) << std::endl; + return 0; +} +// Prints class before counters for lumi +// Class counting 1..64 +int CTPRunScalers::printClassBRateAndIntegralII(int iclsindex) +{ + if (mScalerRecordO2.size() == 0) { + LOG(info) << "ScalerRecord is empty, doing nothing"; + return 1; + } + double_t time0 = mScalerRecordO2[0].epochTime; + double_t timeL = mScalerRecordO2[mScalerRecordO2.size() - 1].epochTime; + int iscalerindex = getScalerIndexForClass(iclsindex); + if (iscalerindex != 255) { + int integral = mScalerRecordO2[mScalerRecordO2.size() - 1].scalers[iscalerindex].lmBefore - mScalerRecordO2[0].scalers[iscalerindex].lmBefore; + std::cout << "Scaler Integrals for run:" << mRunNumber << " duration:" << timeL - time0; + std::cout << " Class index" << iclsindex << " integral:" << integral << " rate:" << integral / (timeL - time0) << std::endl; + return 0; + } + return 1; +} +// Prints class before counters for lumi +// Scaler Index Class counting 1..64 +// getScalerIndexForClass(int cls) shpild be called before to convert class index to scaler class index +int CTPRunScalers::printClassBRateAndIntegral(int iclsscalerindex) +{ + if (mScalerRecordO2.size() == 0) { + LOG(info) << "ScalerRecord is empty, doing nothing"; + return 1; + } + double_t time0 = mScalerRecordO2[0].epochTime; + double_t timeL = mScalerRecordO2[mScalerRecordO2.size() - 1].epochTime; + { + int integral = mScalerRecordO2[mScalerRecordO2.size() - 1].scalers[iclsscalerindex - 1].lmBefore - mScalerRecordO2[0].scalers[iclsscalerindex - 1].lmBefore; + std::cout << "Scaler Integrals for run:" << mRunNumber << " duration:" << timeL - time0; + std::cout << " Class scaler index:" << iclsscalerindex << " integral:" << integral << " rate:" << integral / (timeL - time0) << std::endl; + } + return 0; +} +// +void CTPRunScalers::printLMBRateVsT() const +{ + for (uint32_t i = 1; i < mScalerRecordO2.size(); i++) { // loop over time + auto prev = &mScalerRecordO2[i - 1]; + auto curr = &mScalerRecordO2[i]; + double_t tt = (double_t)(curr->intRecord.orbit - prev->intRecord.orbit); + tt = tt * 88e-6; + + for (int j = 0; j < 1; j++) { // loop over classes + auto s0 = &(prev->scalers[j]); // type CTPScalerO2* + auto s1 = &(curr->scalers[j]); + double_t rMB = (s1->lmBefore - s0->lmBefore) / tt; // rate + auto delta = curr->epochTime - prev->epochTime; + double rMB2 = (s1->lmBefore - s0->lmBefore) / delta; + std::cout << "Class " << j << " " << (unsigned long long)(prev->epochTime) << " " << (unsigned long long)(curr->epochTime) << " DeltaTime " << delta << " tt " << tt << " interactions / s " << rMB << " in Hz " << rMB2 << "\n"; + } + } +} +// +uint64_t CTPRunScalers::getLumiNoPuCorr(int classindex, int type) const +{ + if (type < 7) { + const auto s0 = mScalerRecordO2[0].scalers[classindex]; + const auto s1 = mScalerRecordO2[mScalerRecordO2.size() - 1].scalers[classindex]; + switch (type) { + case 1: + return (s1.lmBefore - s0.lmBefore); + case 2: + return (s1.lmAfter - s0.lmAfter); + case 3: + return (s1.l0Before - s0.l0Before); + case 4: + return (s1.l0After - s0.l0After); + case 5: + return (s1.l1Before - s0.l1Before); + case 6: + return (s1.l1After - s0.l1After); + default: + LOG(error) << "Wrong type:" << type; + return -1; // wrong type + } + } else if (type == 7) { + auto s0 = mScalerRecordO2[0].scalersInps[classindex]; // type CTPScalerO2* + auto s1 = mScalerRecordO2[mScalerRecordO2.size() - 1].scalersInps[classindex]; + return (s1 - s0); + } else { + LOG(error) << "Wrong type:" << type; + return -1; // wrong type + } +}; +// +std::vector> CTPRunScalers::getRatesForIndex(int classindex, int type) const +{ + std::vector> scals; + for (int i = 0; i < mScalerRecordO2.size() - 1; i++) { + double_t diff = 0; + // double_t timeDiff = mScalerRecordO2[i + 1].epochTime - mScalerRecordO2[i].epochTime; + double_t timeDiff = (mScalerRecordO2[i + 1].intRecord.orbit - mScalerRecordO2[i].intRecord.orbit) * o2::constants::lhc::LHCOrbitMUS / 1.e6; + if (type < 7) { + const auto s0 = mScalerRecordO2[i].scalers[classindex]; + const auto s1 = mScalerRecordO2[i + 1].scalers[classindex]; + if (type == 1) { + diff = s1.lmBefore - s0.lmBefore; + } else if (type == 2) { + diff = s1.lmAfter - s0.lmAfter; + } else if (type == 3) { + diff = s1.l0Before - s0.l0Before; + } else if (type == 4) { + diff = s1.l0After - s0.l0After; + } else if (type == 5) { + diff = s1.l1Before - s0.l1Before; + } else if (type == 6) { + diff = s1.l1After - s0.l1After; + } else { + LOG(error) << "Wrong type:" << type; + return scals; // wrong type + } + } else if (type == 7) { + auto s0 = mScalerRecordO2[i].scalersInps[classindex]; // type CTPScalerO2* + auto s1 = mScalerRecordO2[i + 1].scalersInps[classindex]; + diff = s1 - s0; + } else { + LOG(error) << "Wrong type:" << type; + return scals; // wrong type + } + scals.emplace_back(std::pair{diff, timeDiff}); + } + return scals; +}; +// returns the pair of global (levelled) interaction rate, as well as instantaneous interpolated +// rate in Hz at a certain orbit number within the run +// type - 7 : inputs +// type - 1..6 : lmb,lma,l0b,l0a,l1b,l1a +std::pair CTPRunScalers::getRate(uint32_t orbit, int classindex, int type, bool qc) const +{ + if (mScalerRecordO2.size() <= 1) { + LOG(error) << "not enough data"; + return std::make_pair(-1., -1.); + } + + // assumption: mScalerRecordO2 is arranged in increasing + // orbit numbers + + // then we can use binary search to find the right entries + auto iter = std::lower_bound(mScalerRecordO2.begin(), mScalerRecordO2.end(), orbit, [&](CTPScalerRecordO2 const& a, uint32_t value) { return a.intRecord.orbit <= value; }); + auto nextindex = std::distance(mScalerRecordO2.begin(), iter); // this points to the first index that has orbit greater or equal to given orbit + + auto calcRate = [&](auto index1, auto index2) -> double { + const auto& snext = mScalerRecordO2[index2]; + const auto& sprev = mScalerRecordO2[index1]; + auto timedelta = (snext.intRecord.orbit - sprev.intRecord.orbit) * 88.e-6; // converts orbits into time + if (type < 7) { + const auto& s0 = sprev.scalers[classindex]; // type CTPScalerO2* + const auto& s1 = snext.scalers[classindex]; + switch (type) { + case 1: + return (s1.lmBefore - s0.lmBefore) / timedelta; + case 2: + return (s1.lmAfter - s0.lmAfter) / timedelta; + case 3: + return (s1.l0Before - s0.l0Before) / timedelta; + case 4: + return (s1.l0After - s0.l0After) / timedelta; + case 5: + return (s1.l1Before - s0.l1Before) / timedelta; + case 6: + return (s1.l1After - s0.l1After) / timedelta; + default: + LOG(error) << "Wrong type:" << type; + return -1; // wrong type + } + } else if (type == 7) { + auto s0 = sprev.scalersInps[classindex]; // type CTPScalerO2* + auto s1 = snext.scalersInps[classindex]; + return (s1 - s0) / timedelta; + } else { + LOG(error) << "Wrong type:" << type; + return -1; // wrong type + } + }; + // qc flag decides what to return if time outside run + if (nextindex == 0) { + // orbit is out of bounds + if (qc == 0) { + LOG(info) << "query orbit " << orbit << " before first record; Just returning the global rate"; + return std::make_pair(/*global mean rate*/ calcRate(0, mScalerRecordO2.size() - 1), /* current rate */ -1); + } else { + LOG(info) << "query orbit " << orbit << " before first record; Returning the first rate"; + return std::make_pair(/*global mean rate*/ calcRate(0, mScalerRecordO2.size() - 1), /* first rate */ calcRate(0, 1)); + } + } else if (nextindex == mScalerRecordO2.size()) { + if (qc == 0) { + LOG(info) << "query orbit " << orbit << " after last record; Just returning the global rate"; + return std::make_pair(/*global mean rate*/ calcRate(0, mScalerRecordO2.size() - 1), /* current rate */ -1); + } else { + LOG(info) << "query orbit " << orbit << " after last record; Returning the last rate"; + return std::make_pair(/*global mean rate*/ calcRate(0, mScalerRecordO2.size() - 1), /* last rate */ calcRate(mScalerRecordO2.size() - 2, mScalerRecordO2.size() - 1)); + } + } else { + return std::make_pair(/*global mean rate*/ calcRate(0, mScalerRecordO2.size() - 1), /* current rate */ calcRate(nextindex - 1, nextindex)); + } + return std::make_pair(-1., -1.); +} +// returns the pair of global (levelled) interaction rate, as well as instantaneous interpolated +// rate in Hz at a certain orbit number within the run +// type - 7 : inputs +// type - 1..6 : lmb,lma,l0b,l0a,l1b,l1a +std::pair CTPRunScalers::getRateGivenT(double timestamp, int classindex, int type, bool qc) const +{ + if (mScalerRecordO2.size() <= 1) { + LOG(error) << "not enough data"; + return std::make_pair(-1., -1.); + } + + // assumption: mScalerRecordO2 is arranged in increasing + // orbit numbers + + // then we can use binary search to find the right entries + auto iter = std::lower_bound(mScalerRecordO2.begin(), mScalerRecordO2.end(), timestamp, [&](CTPScalerRecordO2 const& a, double value) { return a.epochTime <= value; }); + // this points to the first index that has orbit greater to given orbit; + // If this is 0, it means that the above condition was false from the beginning, basically saying that the timestamp is below any of the ScalerRecords' orbits. + // If this is mScalerRecordO2.size(), it means mScalerRecordO2.end() was returned, condition was met throughout all ScalerRecords, basically saying the timestamp is above any of the ScalarRecordss orbits. + auto nextindex = std::distance(mScalerRecordO2.begin(), iter); + + auto calcRate = [&](auto index1, auto index2) -> double { + const auto& snext = mScalerRecordO2[index2]; + const auto& sprev = mScalerRecordO2[index1]; + auto timedelta = (snext.intRecord.orbit - sprev.intRecord.orbit) * 88.e-6; // converts orbits into time + // std::cout << "timedelta:" << timedelta << std::endl; + if (type < 7) { + const auto& s0 = sprev.scalers[classindex]; // type CTPScalerO2* + const auto& s1 = snext.scalers[classindex]; + switch (type) { + case 1: + return (s1.lmBefore - s0.lmBefore) / timedelta; + case 2: + return (s1.lmAfter - s0.lmAfter) / timedelta; + case 3: + return (s1.l0Before - s0.l0Before) / timedelta; + case 4: + return (s1.l0After - s0.l0After) / timedelta; + case 5: + return (s1.l1Before - s0.l1Before) / timedelta; + case 6: + return (s1.l1After - s0.l1After) / timedelta; + default: + LOG(error) << "Wrong type:" << type; + return -1; // wrong type + } + } else if (type == 7) { + // LOG(info) << "doing input:"; + auto s0 = sprev.scalersInps[classindex]; // type CTPScalerO2* + auto s1 = snext.scalersInps[classindex]; + return (s1 - s0) / timedelta; + } else { + LOG(error) << "Wrong type:" << type; + return -1; // wrong type + } + }; + if (nextindex == 0) { + // orbit is out of bounds + if (qc == 0) { + LOG(info) << "query timestamp " << (long)timestamp << " before first record; Just returning the global rate"; + return std::make_pair(/*global mean rate*/ calcRate(0, mScalerRecordO2.size() - 1), /* current rate */ -1); + } else { + LOG(info) << "query timestamp " << (long)timestamp << " before first record; Returning the first rate"; + return std::make_pair(/*global mean rate*/ calcRate(0, mScalerRecordO2.size() - 1), /* first rate */ calcRate(0, 1)); + } + } else if (nextindex == mScalerRecordO2.size()) { + if (qc == 0) { + LOG(info) << "query timestamp " << (long)timestamp << " after last record; Just returning the global rate"; + return std::make_pair(/*global mean rate*/ calcRate(0, mScalerRecordO2.size() - 1), /* current rate */ -1); + } else { + LOG(info) << "query timestamp " << (long)timestamp << " after last record; Returning the last rate"; + return std::make_pair(/*global mean rate*/ calcRate(0, mScalerRecordO2.size() - 1), /* last rate */ calcRate(mScalerRecordO2.size() - 2, mScalerRecordO2.size() - 1)); + } + } else { + return std::make_pair(/*global mean rate*/ calcRate(0, mScalerRecordO2.size() - 1), /* current rate */ calcRate(nextindex - 1, nextindex)); + } + return std::make_pair(-1., -1.); +} +// Offset orbit of all records +// +int CTPRunScalers::addOrbitOffset(uint32_t offset) +{ + int over = 0; + LOG(info) << "Subtracting from orbit " << offset; + for (auto& screc : mScalerRecordRaw) { + uint32_t orbit = screc.intRecord.orbit; + uint32_t orbitnew = 0; + orbitnew = orbit - offset; + if (orbit < offset) { + over++; + } + screc.intRecord.orbit = orbitnew; + } + if (over != 0 && over != mScalerRecordRaw.size()) { + LOG(warning) << "Orbit overflow inside run. Run:" << mRunNumber; + } + return 0; +} std::vector CTPRunScalers::scalerNames = { "runn0", "runn1", "runn2", "runn3", "runn4", "runn5", "runn6", "runn7", "runn8", "runn9", "runn10", "runn11", "runn12", "runn13", "runn14", "runn15", "ltg1_ORB", "ltg1_HB", "ltg1_HBr", "ltg1_HC", "ltg1_PH", "ltg1_PP", "ltg1_CAL", "ltg1_SOT", "ltg1_EOT", "ltg1_SOC", "ltg1_EOC", "ltg1_TF", "ltg1_FERST", "ltg1_RT", "ltg1_RS", "", "", "", "", "", "", "", "", "", "", "", "", "ltg1_GAP1", "ltg1_GAP2", "ltg1_TPC_sync", "ltg1_TPC_rst", "ltg1_TOF", "ltg2_ORB", "ltg2_HB", "ltg2_HBr", "ltg2_HC", "ltg2_PH", "ltg2_PP", "ltg2_CAL", "ltg2_SOT", "ltg2_EOT", "ltg2_SOC", "ltg2_EOC", "ltg2_TF", "ltg2_FERST", "ltg2_RT", "ltg2_RS", "", "", "", "", "", "", "", "", "", "", "", "", "ltg2_GAP1", "ltg2_GAP2", "ltg2_TPC_sync", "ltg2_TPC_rst", "ltg2_TOF", "ltg3_ORB", "ltg3_HB", "ltg3_HBr", "ltg3_HC", "ltg3_PH", "ltg3_PP", "ltg3_CAL", "ltg3_SOT", "ltg3_EOT", "ltg3_SOC", "ltg3_EOC", "ltg3_TF", "ltg3_FERST", "ltg3_RT", "ltg3_RS", "", "", "", "", "", "", "", "", "", "", "", "", "ltg3_GAP1", "ltg3_GAP2", "ltg3_TPC_sync", "ltg3_TPC_rst", "ltg3_TOF", "ltg4_ORB", "ltg4_HB", "ltg4_HBr", "ltg4_HC", "ltg4_PH", "ltg4_PP", "ltg4_CAL", "ltg4_SOT", "ltg4_EOT", "ltg4_SOC", "ltg4_EOC", "ltg4_TF", "ltg4_FERST", "ltg4_RT", "ltg4_RS", "", "", "", "", "", "", "", "", "", "", "", "", "ltg4_GAP1", "ltg4_GAP2", "ltg4_TPC_sync", "ltg4_TPC_rst", "ltg4_TOF", "ltg5_ORB", "ltg5_HB", "ltg5_HBr", "ltg5_HC", "ltg5_PH", "ltg5_PP", "ltg5_CAL", "ltg5_SOT", "ltg5_EOT", "ltg5_SOC", "ltg5_EOC", "ltg5_TF", "ltg5_FERST", "ltg5_RT", "ltg5_RS", "", "", "", "", "", "", "", "", "", "", "", "", "ltg5_GAP1", "ltg5_GAP2", "ltg5_TPC_sync", "ltg5_TPC_rst", "ltg5_TOF", "ltg6_ORB", "ltg6_HB", "ltg6_HBr", "ltg6_HC", "ltg6_PH", "ltg6_PP", "ltg6_CAL", "ltg6_SOT", "ltg6_EOT", "ltg6_SOC", "ltg6_EOC", "ltg6_TF", "ltg6_FERST", "ltg6_RT", "ltg6_RS", "", "", "", "", "", "", "", "", "", "", "", "", "ltg6_GAP1", "ltg6_GAP2", "ltg6_TPC_sync", "ltg6_TPC_rst", "ltg6_TOF", "ltg7_ORB", "ltg7_HB", "ltg7_HBr", "ltg7_HC", "ltg7_PH", "ltg7_PP", "ltg7_CAL", "ltg7_SOT", "ltg7_EOT", "ltg7_SOC", "ltg7_EOC", "ltg7_TF", "ltg7_FERST", "ltg7_RT", "ltg7_RS", "", "", "", "", "", "", "", "", "", "", "", "", "ltg7_GAP1", "ltg7_GAP2", "ltg7_TPC_sync", "ltg7_TPC_rst", "ltg7_TOF", "ltg8_ORB", "ltg8_HB", "ltg8_HBr", "ltg8_HC", "ltg8_PH", "ltg8_PP", "ltg8_CAL", "ltg8_SOT", "ltg8_EOT", "ltg8_SOC", "ltg8_EOC", "ltg8_TF", "ltg8_FERST", "ltg8_RT", "ltg8_RS", "", "", "", "", "", "", "", "", "", "", "", "", "ltg8_GAP1", "ltg8_GAP2", "ltg8_TPC_sync", "ltg8_TPC_rst", "ltg8_TOF", "ltg9_ORB", "ltg9_HB", "ltg9_HBr", "ltg9_HC", "ltg9_PH", "ltg9_PP", "ltg9_CAL", "ltg9_SOT", "ltg9_EOT", "ltg9_SOC", "ltg9_EOC", "ltg9_TF", "ltg9_FERST", "ltg9_RT", "ltg9_RS", "", "", "", "", "", "", "", "", "", "", "", "", "ltg9_GAP1", "ltg9_GAP2", "ltg9_TPC_sync", "ltg9_TPC_rst", "ltg9_TOF", "ltg10_ORB", "ltg10_HB", "ltg10_HBr", "ltg10_HC", "ltg10_PH", "ltg10_PP", "ltg10_CAL", "ltg10_SOT", "ltg10_EOT", "ltg10_SOC", "ltg10_EOC", "ltg10_TF", "ltg10_FERST", "ltg10_RT", "ltg10_RS", "", "", "", "", "", "", "", "", "", "", "", "", "ltg10_GAP1", "ltg10_GAP2", "ltg10_TPC_sync", "ltg10_TPC_rst", "ltg10_TOF", "ltg11_ORB", "ltg11_HB", "ltg11_HBr", "ltg11_HC", "ltg11_PH", "ltg11_PP", "ltg11_CAL", "ltg11_SOT", "ltg11_EOT", "ltg11_SOC", "ltg11_EOC", "ltg11_TF", "ltg11_FERST", "ltg11_RT", "ltg11_RS", "", "", "", "", "", "", "", "", "", "", "", "", "ltg11_GAP1", "ltg11_GAP2", "ltg11_TPC_sync", "ltg11_TPC_rst", "ltg11_TOF", "ltg12_ORB", "ltg12_HB", "ltg12_HBr", "ltg12_HC", "ltg12_PH", "ltg12_PP", "ltg12_CAL", "ltg12_SOT", "ltg12_EOT", "ltg12_SOC", "ltg12_EOC", "ltg12_TF", "ltg12_FERST", "ltg12_RT", "ltg12_RS", "", "", "", "", "", "", "", "", "", "", "", "", "ltg12_GAP1", "ltg12_GAP2", "ltg12_TPC_sync", "ltg12_TPC_rst", "ltg12_TOF", "ltg13_ORB", "ltg13_HB", "ltg13_HBr", "ltg13_HC", "ltg13_PH", "ltg13_PP", "ltg13_CAL", "ltg13_SOT", "ltg13_EOT", "ltg13_SOC", "ltg13_EOC", "ltg13_TF", "ltg13_FERST", "ltg13_RT", "ltg13_RS", "", "", "", "", "", "", "", "", "", "", "", "", "ltg13_GAP1", "ltg13_GAP2", "ltg13_TPC_sync", "ltg13_TPC_rst", "ltg13_TOF", "ltg14_ORB", "ltg14_HB", "ltg14_HBr", "ltg14_HC", "ltg14_PH", "ltg14_PP", "ltg14_CAL", "ltg14_SOT", "ltg14_EOT", "ltg14_SOC", "ltg14_EOC", "ltg14_TF", "ltg14_FERST", "ltg14_RT", "ltg14_RS", "", "", "", "", "", "", "", "", "", "", "", "", "ltg14_GAP1", "ltg14_GAP2", "ltg14_TPC_sync", "ltg14_TPC_rst", "ltg14_TOF", "ltg15_ORB", "ltg15_HB", "ltg15_HBr", "ltg15_HC", "ltg15_PH", "ltg15_PP", "ltg15_CAL", "ltg15_SOT", "ltg15_EOT", "ltg15_SOC", "ltg15_EOC", "ltg15_TF", "ltg15_FERST", "ltg15_RT", "ltg15_RS", "", "", "", "", "", "", "", "", "", "", "", "", "ltg15_GAP1", "ltg15_GAP2", "ltg15_TPC_sync", "ltg15_TPC_rst", "ltg15_TOF", "ltg16_ORB", "ltg16_HB", "ltg16_HBr", "ltg16_HC", "ltg16_PH", "ltg16_PP", "ltg16_CAL", "ltg16_SOT", "ltg16_EOT", "ltg16_SOC", "ltg16_EOC", "ltg16_TF", "ltg16_FERST", "ltg16_RT", "ltg16_RS", "", "", "", "", "", "", "", "", "", "", "", "", "ltg16_GAP1", "ltg16_GAP2", "ltg16_TPC_sync", "ltg16_TPC_rst", "ltg16_TOF", "ltg17_ORB", "ltg17_HB", "ltg17_HBr", "ltg17_HC", "ltg17_PH", "ltg17_PP", "ltg17_CAL", "ltg17_SOT", "ltg17_EOT", "ltg17_SOC", "ltg17_EOC", "ltg17_TF", "ltg17_FERST", "ltg17_RT", "ltg17_RS", "", "", "", "", "", "", "", "", "", "", "", "", "ltg17_GAP1", "ltg17_GAP2", "ltg17_TPC_sync", "ltg17_TPC_rst", "ltg17_TOF", "ltg18_ORB", "ltg18_HB", "ltg18_HBr", "ltg18_HC", "ltg18_PH", "ltg18_PP", "ltg18_CAL", "ltg18_SOT", "ltg18_EOT", "ltg18_SOC", "ltg18_EOC", "ltg18_TF", "ltg18_FERST", "ltg18_RT", "ltg18_RS", "", "", "", "", "", "", "", "", "", "", "", "", "ltg18_GAP1", "ltg18_GAP2", "ltg18_TPC_sync", "ltg18_TPC_rst", "ltg18_TOF", "bc40", "clk240", "extorb", "PLSRin", "FastLMin", "BUSYin", "SPAREin", "inp1", "inp2", "inp3", "inp4", "inp5", "inp6", "inp7", "inp8", "inp9", "inp10", "inp11", "inp12", "inp13", "inp14", "inp15", "inp16", "inp17", "inp18", "inp19", "inp20", "inp21", "inp22", "inp23", "inp24", "inp25", "inp26", "inp27", "inp28", "inp29", "inp30", "inp31", "inp32", "inp33", "inp34", "inp35", "inp36", "inp37", "inp38", "inp39", "inp40", "inp41", "inp42", "inp43", "inp44", "inp45", "inp46", "inp47", "inp48", "clamb1", "clamb2", "clamb3", "clamb4", "clamb5", "clamb6", "clamb7", "clamb8", "clamb9", "clamb10", "clamb11", "clamb12", "clamb13", "clamb14", "clamb15", "clamb16", "clamb17", "clamb18", "clamb19", "clamb20", "clamb21", "clamb22", "clamb23", "clamb24", "clamb25", "clamb26", "clamb27", "clamb28", "clamb29", "clamb30", "clamb31", "clamb32", "clamb33", "clamb34", "clamb35", "clamb36", "clamb37", "clamb38", "clamb39", "clamb40", "clamb41", "clamb42", "clamb43", "clamb44", "clamb45", "clamb46", "clamb47", "clamb48", "clamb49", "clamb50", "clamb51", "clamb52", "clamb53", "clamb54", "clamb55", "clamb56", "clamb57", "clamb58", "clamb59", "clamb60", "clamb61", "clamb62", "clamb63", "clamb64", "clama1", "clama2", "clama3", "clama4", "clama5", "clama6", "clama7", "clama8", "clama9", "clama10", "clama11", "clama12", "clama13", "clama14", "clama15", "clama16", "clama17", "clama18", "clama19", "clama20", "clama21", "clama22", "clama23", "clama24", "clama25", "clama26", "clama27", "clama28", "clama29", "clama30", "clama31", "clama32", "clama33", "clama34", "clama35", "clama36", "clama37", "clama38", "clama39", "clama40", "clama41", "clama42", "clama43", "clama44", "clama45", "clama46", "clama47", "clama48", "clama49", "clama50", "clama51", "clama52", "clama53", "clama54", "clama55", "clama56", "clama57", "clama58", "clama59", "clama60", "clama61", "clama62", "clama63", "clama64", "cla0b1", "cla0b2", "cla0b3", "cla0b4", "cla0b5", "cla0b6", "cla0b7", "cla0b8", "cla0b9", "cla0b10", "cla0b11", "cla0b12", "cla0b13", "cla0b14", "cla0b15", "cla0b16", "cla0b17", "cla0b18", "cla0b19", "cla0b20", "cla0b21", "cla0b22", "cla0b23", "cla0b24", "cla0b25", "cla0b26", "cla0b27", "cla0b28", "cla0b29", "cla0b30", "cla0b31", "cla0b32", "cla0b33", "cla0b34", "cla0b35", "cla0b36", "cla0b37", "cla0b38", "cla0b39", "cla0b40", "cla0b41", "cla0b42", "cla0b43", "cla0b44", "cla0b45", "cla0b46", "cla0b47", "cla0b48", "cla0b49", "cla0b50", "cla0b51", "cla0b52", "cla0b53", "cla0b54", "cla0b55", "cla0b56", "cla0b57", "cla0b58", "cla0b59", "cla0b60", "cla0b61", "cla0b62", "cla0b63", "cla0b64", "cla0a1", "cla0a2", "cla0a3", "cla0a4", "cla0a5", "cla0a6", "cla0a7", "cla0a8", "cla0a9", "cla0a10", "cla0a11", "cla0a12", "cla0a13", "cla0a14", "cla0a15", "cla0a16", "cla0a17", "cla0a18", "cla0a19", "cla0a20", "cla0a21", "cla0a22", "cla0a23", "cla0a24", "cla0a25", "cla0a26", "cla0a27", "cla0a28", "cla0a29", "cla0a30", "cla0a31", "cla0a32", "cla0a33", "cla0a34", "cla0a35", "cla0a36", "cla0a37", "cla0a38", "cla0a39", "cla0a40", "cla0a41", "cla0a42", "cla0a43", "cla0a44", "cla0a45", "cla0a46", "cla0a47", "cla0a48", "cla0a49", "cla0a50", "cla0a51", "cla0a52", "cla0a53", "cla0a54", "cla0a55", "cla0a56", "cla0a57", "cla0a58", "cla0a59", "cla0a60", "cla0a61", "cla0a62", "cla0a63", "cla0a64", "cla1b1", "cla1b2", "cla1b3", "cla1b4", "cla1b5", "cla1b6", "cla1b7", "cla1b8", "cla1b9", "cla1b10", "cla1b11", "cla1b12", "cla1b13", "cla1b14", "cla1b15", "cla1b16", "cla1b17", "cla1b18", "cla1b19", "cla1b20", "cla1b21", "cla1b22", "cla1b23", "cla1b24", "cla1b25", "cla1b26", "cla1b27", "cla1b28", "cla1b29", "cla1b30", "cla1b31", "cla1b32", "cla1b33", "cla1b34", "cla1b35", "cla1b36", "cla1b37", "cla1b38", "cla1b39", "cla1b40", "cla1b41", "cla1b42", "cla1b43", "cla1b44", "cla1b45", "cla1b46", "cla1b47", "cla1b48", "cla1b49", "cla1b50", "cla1b51", "cla1b52", "cla1b53", "cla1b54", "cla1b55", "cla1b56", "cla1b57", "cla1b58", "cla1b59", "cla1b60", "cla1b61", "cla1b62", "cla1b63", "cla1b64", "cla1a1", "cla1a2", "cla1a3", "cla1a4", "cla1a5", "cla1a6", "cla1a7", "cla1a8", "cla1a9", "cla1a10", "cla1a11", "cla1a12", "cla1a13", "cla1a14", "cla1a15", "cla1a16", "cla1a17", "cla1a18", "cla1a19", "cla1a20", "cla1a21", "cla1a22", "cla1a23", "cla1a24", "cla1a25", "cla1a26", "cla1a27", "cla1a28", "cla1a29", "cla1a30", "cla1a31", "cla1a32", "cla1a33", "cla1a34", "cla1a35", "cla1a36", "cla1a37", "cla1a38", "cla1a39", "cla1a40", "cla1a41", "cla1a42", "cla1a43", "cla1a44", "cla1a45", "cla1a46", "cla1a47", "cla1a48", "cla1a49", "cla1a50", "cla1a51", "cla1a52", "cla1a53", "cla1a54", "cla1a55", "cla1a56", "cla1a57", "cla1a58", "cla1a59", "cla1a60", "cla1a61", "cla1a62", "cla1a63", "cla1a64", "l0_trigger", "l1_trigger", "l2_trigger", "clum1", "clum2", "clum3", "clum4", "clum5", "clum6", "clu01", "clu02", "clu03", "clu04", "clu05", "clu06", "clu11", "clu12", "clu13", "clu14", "clu15", "clu16", "ltg1_busy", "ltg2_busy", "ltg3_busy", "ltg4_busy", "ltg5_busy", "ltg6_busy", "ltg7_busy", "ltg8_busy", "ltg9_busy", - "ltg10_busy", "ltg11_busy", "ltg12_busy", "ltg13_busy", "ltg14_busy", "ltg15_busy", "ltg16_busy", "ltg17_busy", "ltg18_busy"}; + "ltg10_busy", "ltg11_busy", "ltg12_busy", "ltg13_busy", "ltg14_busy", "ltg15_busy", "ltg16_busy", "ltg17_busy", "ltg18_busy", "orbitid", "clum7", "clum8", "clu07", "clu08", "clu17", "clu18", "clu_busy1", "clu_busy2", "clu_busy3", "clu_busy4", "clu_busy5", "clu_busy6", "clu_busy7", "clu_busy8"}; diff --git a/DataFormats/Detectors/Common/CMakeLists.txt b/DataFormats/Detectors/Common/CMakeLists.txt index 6b090771f292a..bbc0aaf9e8320 100644 --- a/DataFormats/Detectors/Common/CMakeLists.txt +++ b/DataFormats/Detectors/Common/CMakeLists.txt @@ -52,3 +52,14 @@ o2_add_test(DetID PUBLIC_LINK_LIBRARIES O2::DetectorsCommonDataFormats COMPONENT_NAME DetectorsCommonDataFormats LABELS dataformats) + +o2_add_test(CTFEntropyCoder + NAME CTFEntropyCoder + SOURCES test/testCTFEntropyCoder.cxx + PUBLIC_LINK_LIBRARIES O2::DetectorsCommonDataFormats O2::rANS + COMPONENT_NAME DetectorsCommonDataFormats + TARGETVARNAME TEST_CTF_ENTROPY_CODER + LABELS detectorsbase) +if(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "x86_64") + target_compile_options(${TEST_CTF_ENTROPY_CODER} PRIVATE -march=native) +endif() diff --git a/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/ANSHeader.h b/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/ANSHeader.h new file mode 100644 index 0000000000000..110af41cddd49 --- /dev/null +++ b/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/ANSHeader.h @@ -0,0 +1,70 @@ +// Copyright 2019-2023 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file ANSHeader.h +/// \author michael.lettrich@cern.ch +/// \brief representation of ANS Version number in a comparable way + +#ifndef _ALICEO2_ANSHEADER_H_ +#define _ALICEO2_ANSHEADER_H_ + +#include +#include +#include +#include + +namespace o2::ctf +{ + +struct ANSHeader { + uint8_t majorVersion; + uint8_t minorVersion; + + void clear() { majorVersion = minorVersion = 0; } + inline constexpr operator uint16_t() const noexcept + { + uint16_t major = majorVersion; + uint16_t minor = minorVersion; + return (major << 8) | minor; + }; + + inline operator std::string() const + { + return fmt::format("{}.{}", majorVersion, minorVersion); + }; + inline constexpr uint32_t version() const noexcept { return static_cast(*this); }; + inline constexpr bool operator==(const ANSHeader& other) const noexcept { return static_cast(*this) == static_cast(other); }; + inline constexpr bool operator!=(const ANSHeader& other) const noexcept { return static_cast(*this) != static_cast(other); }; + inline constexpr bool operator<(const ANSHeader& other) const noexcept { return static_cast(*this) < static_cast(other); }; + inline constexpr bool operator>(const ANSHeader& other) const noexcept { return static_cast(*this) > static_cast(other); }; + inline constexpr bool operator>=(const ANSHeader& other) const noexcept { return static_cast(*this) >= static_cast(other); }; + inline constexpr bool operator<=(const ANSHeader& other) const noexcept { return static_cast(*this) <= static_cast(other); }; + ClassDefNV(ANSHeader, 2); +}; + +inline constexpr ANSHeader ANSVersionUnspecified{0, 0}; +inline constexpr ANSHeader ANSVersionCompat{0, 1}; +inline constexpr ANSHeader ANSVersion1{1, 0}; + +inline ANSHeader ansVersionFromString(const std::string& ansVersionString) +{ + if (ansVersionString == "0.1" || ansVersionString == "compat") { + return ctf::ANSVersionCompat; + } else if (ansVersionString == "1.0") { + return ctf::ANSVersion1; + } else { + return ctf::ANSVersionUnspecified; + } +} + +} // namespace o2::ctf + +#endif /* _ALICEO2_ANSHEADER_H_ */ diff --git a/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/AlignParam.h b/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/AlignParam.h index a6939fc7b430d..5a0d2d64b0ff5 100644 --- a/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/AlignParam.h +++ b/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/AlignParam.h @@ -37,7 +37,12 @@ class AlignParam AlignParam(const char* symname, int algID, // volume symbolic name and its alignable ID double x, double y, double z, // delta translation double psi, double theta, double phi, // delta rotation - bool global = true); // global (preferable) or local delta definition + bool global = true, // global (preferable) or local delta definition + bool convertLocalToGlobal = true); // if local is provided, convert it to global + + AlignParam(const char* symname, int algID, TGeoMatrix& m, + bool global = true, // global (preferable) or local delta definition + bool convertLocalToGlobal = true); // if local is provided, convert it to global /// return symbolic name of the volume const std::string& getSymName() const { return mSymName; } @@ -50,7 +55,7 @@ class AlignParam double getZ() const { return mZ; } /// apply object to geoemetry - bool applyToGeometry() const; + bool applyToGeometry(int printLevel = -1) const; /// extract global delta matrix TGeoHMatrix createMatrix() const; @@ -68,6 +73,9 @@ class AlignParam void setAlignableID(int id) { mAlignableID = id; } /// ================ methods for direct setting of delta params + /// set parameters + void setParams(double x, double y, double z, double psi, double theta, double phi); + /// set parameters of global delta void setGlobalParams(double x, double y, double z, double psi, double theta, double phi); @@ -110,6 +118,11 @@ class AlignParam void print() const; + int rectify(double zero = 1e-13); + + bool isGlobal() const { return mIsGlobal; } + void setIsGlobal(bool v) { mIsGlobal = v; } + protected: bool matrixToAngles(const double* rot, double& psi, double& theta, double& phi) const; void anglesToMatrix(double psi, double theta, double phi, double* rot) const; @@ -119,8 +132,8 @@ class AlignParam private: std::string mSymName{}; + bool mIsGlobal = true; /// is this global delta? int mAlignableID = -1; /// alignable ID (set for sensors only) - double mX = 0.; ///< X translation of global delta double mY = 0.; ///< Y translation of global delta double mZ = 0.; ///< Z translation of global delta @@ -129,10 +142,10 @@ class AlignParam double mTheta = 0.; ///< "roll" : Euler angle of rotation around Y axis after 1st rotation (radians) double mPhi = 0.; ///< "yaw" : Euler angle of rotation around Z axis (radians) - ClassDefNV(AlignParam, 1); + ClassDefNV(AlignParam, 2); }; -} -} +} // namespace detectors +} // namespace o2 #endif diff --git a/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/DetID.h b/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/DetID.h index 5a3a134e737d2..2d2383783cfc3 100644 --- a/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/DetID.h +++ b/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/DetID.h @@ -77,14 +77,20 @@ class DetID static constexpr ID FDD = 14; static constexpr ID TST = 15; static constexpr ID CTP = 16; + static constexpr ID FOC = 17; #ifdef ENABLE_UPGRADES - static constexpr ID IT3 = 17; - static constexpr ID TRK = 18; - static constexpr ID FT3 = 19; - static constexpr ID FCT = 20; - static constexpr ID Last = FCT; + static constexpr ID IT3 = 18; + static constexpr ID TRK = 19; + static constexpr ID FT3 = 20; + static constexpr ID FCT = 21; + static constexpr ID TF3 = 22; + static constexpr ID RCH = 23; + static constexpr ID MI3 = 24; + static constexpr ID ECL = 25; + static constexpr ID FD3 = 26; + static constexpr ID Last = FD3; #else - static constexpr ID Last = CTP; ///< if extra detectors added, update this !!! + static constexpr ID Last = FOC; ///< if extra detectors added, update this !!! #endif static constexpr ID First = ITS; @@ -176,9 +182,9 @@ class DetID // detector names, will be defined in DataSources static constexpr const char* sDetNames[nDetectors + 1] = ///< defined detector names #ifdef ENABLE_UPGRADES - {"ITS", "TPC", "TRD", "TOF", "PHS", "CPV", "EMC", "HMP", "MFT", "MCH", "MID", "ZDC", "FT0", "FV0", "FDD", "TST", "CTP", "IT3", "TRK", "FT3", "FCT", nullptr}; + {"ITS", "TPC", "TRD", "TOF", "PHS", "CPV", "EMC", "HMP", "MFT", "MCH", "MID", "ZDC", "FT0", "FV0", "FDD", "TST", "CTP", "FOC", "IT3", "TRK", "FT3", "FCT", "TF3", "RCH", "MI3", "ECL", "FD3", nullptr}; #else - {"ITS", "TPC", "TRD", "TOF", "PHS", "CPV", "EMC", "HMP", "MFT", "MCH", "MID", "ZDC", "FT0", "FV0", "FDD", "TST", "CTP", nullptr}; + {"ITS", "TPC", "TRD", "TOF", "PHS", "CPV", "EMC", "HMP", "MFT", "MCH", "MID", "ZDC", "FT0", "FV0", "FDD", "TST", "CTP", "FOC", nullptr}; #endif static constexpr std::array @@ -186,10 +192,11 @@ class DetID {o2h::gDataOriginITS, o2h::gDataOriginTPC, o2h::gDataOriginTRD, o2h::gDataOriginTOF, o2h::gDataOriginPHS, o2h::gDataOriginCPV, o2h::gDataOriginEMC, o2h::gDataOriginHMP, o2h::gDataOriginMFT, o2h::gDataOriginMCH, o2h::gDataOriginMID, o2h::gDataOriginZDC, o2h::gDataOriginFT0, o2h::gDataOriginFV0, o2h::gDataOriginFDD, - o2h::gDataOriginTST, o2h::gDataOriginCTP + o2h::gDataOriginTST, o2h::gDataOriginCTP, o2h::gDataOriginFOC #ifdef ENABLE_UPGRADES , - o2h::gDataOriginIT3, o2h::gDataOriginTRK, o2h::gDataOriginFT3, o2h::gDataOriginFCT + o2h::gDataOriginIT3, o2h::gDataOriginTRK, o2h::gDataOriginFT3, o2h::gDataOriginFCT, o2h::gDataOriginTF3, + o2h::gDataOriginRCH, o2h::gDataOriginMI3, o2h::gDataOriginECL, o2h::gDataOriginFD3 #endif }; #endif // GPUCA_GPUCODE_DEVICE @@ -204,10 +211,12 @@ GPUconstexpr() DetID::mask_t sMasks[DetID::nDetectors] = ///< detectot masks {DetID::mask_t(math_utils::bit2Mask(DetID::ITS)), DetID::mask_t(math_utils::bit2Mask(DetID::TPC)), DetID::mask_t(math_utils::bit2Mask(DetID::TRD)), DetID::mask_t(math_utils::bit2Mask(DetID::TOF)), DetID::mask_t(math_utils::bit2Mask(DetID::PHS)), DetID::mask_t(math_utils::bit2Mask(DetID::CPV)), DetID::mask_t(math_utils::bit2Mask(DetID::EMC)), DetID::mask_t(math_utils::bit2Mask(DetID::HMP)), DetID::mask_t(math_utils::bit2Mask(DetID::MFT)), DetID::mask_t(math_utils::bit2Mask(DetID::MCH)), DetID::mask_t(math_utils::bit2Mask(DetID::MID)), DetID::mask_t(math_utils::bit2Mask(DetID::ZDC)), DetID::mask_t(math_utils::bit2Mask(DetID::FT0)), DetID::mask_t(math_utils::bit2Mask(DetID::FV0)), DetID::mask_t(math_utils::bit2Mask(DetID::FDD)), - DetID::mask_t(math_utils::bit2Mask(DetID::TST)), DetID::mask_t(math_utils::bit2Mask(DetID::CTP)) + DetID::mask_t(math_utils::bit2Mask(DetID::TST)), DetID::mask_t(math_utils::bit2Mask(DetID::CTP)), DetID::mask_t(math_utils::bit2Mask(DetID::FOC)) + #ifdef ENABLE_UPGRADES - , - DetID::mask_t(math_utils::bit2Mask(DetID::IT3)), DetID::mask_t(math_utils::bit2Mask(DetID::TRK)), DetID::mask_t(math_utils::bit2Mask(DetID::FT3)), DetID::mask_t(math_utils::bit2Mask(DetID::FCT)) + , + DetID::mask_t(math_utils::bit2Mask(DetID::IT3)), DetID::mask_t(math_utils::bit2Mask(DetID::TRK)), DetID::mask_t(math_utils::bit2Mask(DetID::FT3)), DetID::mask_t(math_utils::bit2Mask(DetID::FCT)), DetID::mask_t(math_utils::bit2Mask(DetID::TF3)), + DetID::mask_t(math_utils::bit2Mask(DetID::RCH)), DetID::mask_t(math_utils::bit2Mask(DetID::MI3)), DetID::mask_t(math_utils::bit2Mask(DetID::ECL)), DetID::mask_t(math_utils::bit2Mask(DetID::FD3)) #endif }; } // namespace detid_internal diff --git a/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/EncodedBlocks.h b/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/EncodedBlocks.h index df9f5ac6547fb..ba6b853f7fb23 100644 --- a/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/EncodedBlocks.h +++ b/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/EncodedBlocks.h @@ -16,18 +16,31 @@ #ifndef ALICEO2_ENCODED_BLOCKS_H #define ALICEO2_ENCODED_BLOCKS_H -#undef NDEBUG -#include +// #undef NDEBUG +// #include #include #include #include -#include "rANS/rans.h" -#include "rANS/utils.h" +#include + #include "TTree.h" #include "CommonUtils/StringUtils.h" #include "Framework/Logger.h" #include "DetectorsCommonDataFormats/CTFDictHeader.h" #include "DetectorsCommonDataFormats/CTFIOSize.h" +#include "DetectorsCommonDataFormats/ANSHeader.h" +#include "DetectorsCommonDataFormats/internal/Packer.h" +#include "DetectorsCommonDataFormats/Metadata.h" +#ifndef __CLING__ +#include "DetectorsCommonDataFormats/internal/ExternalEntropyCoder.h" +#include "DetectorsCommonDataFormats/internal/InplaceEntropyCoder.h" +#include "rANS/compat.h" +#include "rANS/histogram.h" +#include "rANS/serialize.h" +#include "rANS/factory.h" +#include "rANS/metrics.h" +#include "rANS/utils.h" +#endif namespace o2 { @@ -50,9 +63,20 @@ struct is_iterator inline constexpr bool is_iterator_v = is_iterator::value; + +inline constexpr bool mayEEncode(Metadata::OptStore opt) noexcept +{ + return (opt == Metadata::OptStore::EENCODE) || (opt == Metadata::OptStore::EENCODE_OR_PACK); +} + +inline constexpr bool mayPack(Metadata::OptStore opt) noexcept +{ + return (opt == Metadata::OptStore::PACK) || (opt == Metadata::OptStore::EENCODE_OR_PACK); +} + } // namespace detail +constexpr size_t PackingThreshold = 512; -using namespace o2::rans; constexpr size_t Alignment = 16; constexpr int WrappersSplitLevel = 99; @@ -76,10 +100,10 @@ inline T* relocatePointer(const char* oldBase, char* newBase, const T* ptr) } template = sizeof(source_T)), bool> = true> -inline size_t calculateNDestTElements(size_t sourceElems) noexcept +inline constexpr size_t calculateNDestTElements(size_t nElems) noexcept { - const size_t sizeOfSourceArray = sourceElems * sizeof(source_T); - return sizeOfSourceArray / sizeof(dest_T) + (sizeOfSourceArray % sizeof(dest_T) != 0); + const size_t srcBufferSize = nElems * sizeof(source_T); + return srcBufferSize / sizeof(dest_T) + (srcBufferSize % sizeof(dest_T) != 0); }; template = sizeof(source_T)), bool> = true> @@ -94,52 +118,6 @@ inline size_t calculatePaddedSize(size_t nElems) noexcept ///>>======================== Auxiliary classes =======================>> -struct ANSHeader { - uint8_t majorVersion; - uint8_t minorVersion; - - void clear() { majorVersion = minorVersion = 0; } - ClassDefNV(ANSHeader, 1); -}; - -struct Metadata { - enum class OptStore : uint8_t { // describe how the store the data described by this metadata - EENCODE, // entropy encoding applied - ROOTCompression, // original data repacked to array with slot-size = streamSize and saved with root compression - NONE, // original data repacked to array with slot-size = streamSize and saved w/o compression - NODATA // no data was provided - }; - size_t messageLength = 0; - size_t nLiterals = 0; - uint8_t messageWordSize = 0; - uint8_t coderType = 0; - uint8_t streamSize = 0; - uint8_t probabilityBits = 0; - OptStore opt = OptStore::EENCODE; - int32_t min = 0; - int32_t max = 0; - int nDictWords = 0; - int nDataWords = 0; - int nLiteralWords = 0; - - size_t getUncompressedSize() const { return messageLength * messageWordSize; } - size_t getCompressedSize() const { return (nDictWords + nDataWords + nLiteralWords) * streamSize; } - void clear() - { - min = max = 0; - messageLength = 0; - messageWordSize = 0; - nLiterals = 0; - coderType = 0; - streamSize = 0; - probabilityBits = 0; - nDictWords = 0; - nDataWords = 0; - nLiteralWords = 0; - } - ClassDefNV(Metadata, 2); -}; - /// registry struct for the buffer start and offsets of writable space struct Registry { char* head = nullptr; //! pointer on the head of the CTF @@ -160,6 +138,12 @@ struct Registry { return size - offsFreeStart; } + char* getFreeBlockEnd() const + { + assert(offsFreeStart <= size); + return getFreeBlockStart() + getFreeSize(); + } + ClassDefNV(Registry, 1); }; @@ -178,11 +162,21 @@ struct Block { inline const W* getData() const { return nData ? (payload + nDict) : nullptr; } inline const W* getDataPointer() const { return payload ? (payload + nDict) : nullptr; } // needed when nData is not set yet inline const W* getLiterals() const { return nLiterals ? (payload + nDict + nData) : nullptr; } + inline const W* getEndOfBlock() const + { + if (!registry) { + return nullptr; + } + // get last legal W*, since unaligned data is undefined behavior! + const size_t delta = reinterpret_cast(registry->getFreeBlockEnd()) % sizeof(W); + return reinterpret_cast(registry->getFreeBlockEnd() - delta); + } inline W* getCreatePayload() { return payload ? payload : (registry ? (payload = reinterpret_cast(registry->getFreeBlockStart())) : nullptr); } inline W* getCreateDict() { return payload ? payload : getCreatePayload(); } inline W* getCreateData() { return payload ? (payload + nDict) : getCreatePayload(); } inline W* getCreateLiterals() { return payload ? payload + (nDict + nData) : getCreatePayload(); } + inline W* getEndOfBlock() { return const_cast(static_cast(*this).getEndOfBlock()); }; inline auto getOffsDict() { return reinterpret_cast(getCreateDict()) - reinterpret_cast(registry->head); } inline auto getOffsData() { return reinterpret_cast(getCreateData()) - reinterpret_cast(registry->head); } @@ -290,8 +284,10 @@ struct Block { // resize block and free up unused buffer space. void realignBlock() { - size_t sz = estimateSize(getNStored()); - registry->offsFreeStart = (reinterpret_cast(payload) - registry->head) + sz; + if (payload) { + size_t sz = estimateSize(getNStored()); + registry->offsFreeStart = (reinterpret_cast(payload) - registry->head) + sz; + } } /// store binary blob data (buffer filled from head to tail) @@ -340,7 +336,15 @@ class EncodedBlocks public: typedef EncodedBlocks base; - void setHeader(const H& h) { mHeader = h; } +#ifndef __CLING__ + template + using dictionaryType = std::variant, rans::RenormedDenseHistogram>; +#endif + + void setHeader(const H& h) + { + mHeader = h; + } const H& getHeader() const { return mHeader; } H& getHeader() { return mHeader; } std::shared_ptr cloneHeader() const { return std::shared_ptr(new H(mHeader)); } // for dictionary creation @@ -361,15 +365,70 @@ class EncodedBlocks return mBlocks[i]; } - o2::rans::RenormedFrequencyTable getFrequencyTable(int i) const +#ifndef __CLING__ + template + dictionaryType getDictionary(int i, ANSHeader ansVersion = ANSVersionUnspecified) const { const auto& block = getBlock(i); const auto& metadata = getMetadata(i); - rans::FrequencyTable frequencyTable{block.getDict(), block.getDict() + block.getNDict(), metadata.min}; - return rans::renorm(std::move(frequencyTable), metadata.probabilityBits); - } + ansVersion = checkANSVersion(ansVersion); + + assert(static_cast(std::numeric_limits::min()) <= static_cast(metadata.max)); + assert(static_cast(std::numeric_limits::max()) >= static_cast(metadata.min)); + + // check consistency of metadata and type + [&]() { + const int64_t sourceMin = std::numeric_limits::min(); + const int64_t sourceMax = std::numeric_limits::max(); + + auto view = rans::trim(rans::HistogramView{block.getDict(), block.getDict() + block.getNDict(), metadata.min}); + const int64_t dictMin = view.getMin(); + const int64_t dictMax = view.getMax(); + assert(dictMin >= metadata.min); + assert(dictMax <= metadata.max); + + if ((dictMin < sourceMin) || (dictMax > sourceMax)) { + if (ansVersion == ANSVersionCompat && mHeader.majorVersion == 1 && mHeader.minorVersion == 0 && mHeader.dictTimeStamp < 1653192000000) { + LOGP(warn, "value range of dictionary and target datatype are incompatible: target type [{},{}] vs dictionary [{},{}], tolerate in compat mode for old dictionaries", sourceMin, sourceMax, dictMin, dictMax); + } else { + throw std::runtime_error(fmt::format("value range of dictionary and target datatype are incompatible: target type [{},{}] vs dictionary [{},{}]", sourceMin, sourceMax, dictMin, dictMax)); + } + } + }(); - void setANSHeader(const ANSHeader& h) { mANSHeader = h; } + if (ansVersion == ANSVersionCompat) { + rans::DenseHistogram histogram{block.getDict(), block.getDict() + block.getNDict(), metadata.min}; + return rans::compat::renorm(std::move(histogram), metadata.probabilityBits); + } else if (ansVersion == ANSVersion1) { + // dictionary is loaded from an explicit dict file and is stored densly + if (getANSHeader() == ANSVersionUnspecified) { + rans::DenseHistogram histogram{block.getDict(), block.getDict() + block.getNDict(), metadata.min}; + size_t renormingBits = rans::utils::sanitizeRenormingBitRange(metadata.probabilityBits); + LOG_IF(debug, renormingBits != metadata.probabilityBits) + << fmt::format("While reading metadata from external dictionary, rANSV1 is rounding renorming precision from {} to {}", metadata.probabilityBits, renormingBits); + return rans::renorm(std::move(histogram), renormingBits, rans::RenormingPolicy::ForceIncompressible); + } else { + // dictionary is elias-delta coded inside the block + if constexpr (sizeof(source_T) > 2) { + return rans::readRenormedSetDictionary(block.getDict(), block.getDict() + block.getNDict(), + static_cast(metadata.min), static_cast(metadata.max), + metadata.probabilityBits); + } else { + return rans::readRenormedDictionary(block.getDict(), block.getDict() + block.getNDict(), + static_cast(metadata.min), static_cast(metadata.max), + metadata.probabilityBits); + } + } + } else { + throw std::runtime_error(fmt::format("Failed to load serialized Dictionary. Unsupported ANS Version: {}", static_cast(ansVersion))); + } + }; +#endif + + void setANSHeader(const ANSHeader& h) + { + mANSHeader = h; + } const ANSHeader& getANSHeader() const { return mANSHeader; } ANSHeader& getANSHeader() { return mANSHeader; } @@ -409,6 +468,9 @@ class EncodedBlocks /// total allocated size in bytes size_t size() const { return mRegistry.size; } + /// used part of total allocated size in bytes (output size) + size_t outputsize() const { return mRegistry.offsFreeStart; } + /// size remaining for additional data size_t getFreeSize() const { return mRegistry.getFreeSize(); } @@ -435,25 +497,27 @@ class EncodedBlocks /// encode vector src to bloc at provided slot template - inline o2::ctf::CTFIOSize encode(const VE& src, int slot, uint8_t symbolTablePrecision, Metadata::OptStore opt, buffer_T* buffer = nullptr, const void* encoderExt = nullptr, float memfc = 1.f) + inline o2::ctf::CTFIOSize encode(const VE& src, int slot, uint8_t symbolTablePrecision, Metadata::OptStore opt, buffer_T* buffer = nullptr, const std::any& encoderExt = {}, float memfc = 1.f) { return encode(std::begin(src), std::end(src), slot, symbolTablePrecision, opt, buffer, encoderExt, memfc); } /// encode vector src to bloc at provided slot template - o2::ctf::CTFIOSize encode(const input_IT srcBegin, const input_IT srcEnd, int slot, uint8_t symbolTablePrecision, Metadata::OptStore opt, buffer_T* buffer = nullptr, const void* encoderExt = nullptr, float memfc = 1.f); + o2::ctf::CTFIOSize encode(const input_IT srcBegin, const input_IT srcEnd, int slot, uint8_t symbolTablePrecision, Metadata::OptStore opt, buffer_T* buffer = nullptr, const std::any& encoderExt = {}, float memfc = 1.f); /// decode block at provided slot to destination vector (will be resized as needed) template - o2::ctf::CTFIOSize decode(container_T& dest, int slot, const void* decoderExt = nullptr) const; + o2::ctf::CTFIOSize decode(container_T& dest, int slot, const std::any& decoderExt = {}) const; /// decode block at provided slot to destination pointer, the needed space assumed to be available template , bool> = true> - o2::ctf::CTFIOSize decode(D_IT dest, int slot, const void* decoderExt = nullptr) const; + o2::ctf::CTFIOSize decode(D_IT dest, int slot, const std::any& decoderExt = {}) const; +#ifndef __CLING__ /// create a special EncodedBlocks containing only dictionaries made from provided vector of frequency tables - static std::vector createDictionaryBlocks(const std::vector& vfreq, const std::vector& prbits); + static std::vector createDictionaryBlocks(const std::vector>& vfreq, const std::vector& prbits); +#endif /// print itself void print(const std::string& prefix = "", int verbosity = 1) const; @@ -468,6 +532,8 @@ class EncodedBlocks std::array mMetadata; // compressed block's details std::array, N> mBlocks; //! this is in fact stored, but to overcome TBuffer limits we have to define the branches per block!!! + inline static constexpr Metadata::OptStore FallbackStorageType{Metadata::OptStore::NONE}; + /// setup internal structure and registry for given buffer size (in bytes!!!) void init(size_t sz); @@ -495,8 +561,93 @@ class EncodedBlocks template static bool readTreeBranch(TTree& tree, const std::string& brname, D& dt, int ev = 0); - ClassDefNV(EncodedBlocks, 2); -}; + template + auto expandStorage(size_t slot, size_t nElemets, T* buffer = nullptr) -> decltype(auto); + + inline ANSHeader checkANSVersion(ANSHeader ansVersion) const + { + auto ctfANSHeader = getANSHeader(); + ANSHeader ret{ANSVersionUnspecified}; + + const bool isEqual{ansVersion == ctfANSHeader}; + const bool isHeaderUnspecified{ctfANSHeader == ANSVersionUnspecified}; + + if (isEqual) { + if (isHeaderUnspecified) { + throw std::runtime_error{fmt::format("Missmatch of ANSVersions, trying to encode/decode CTF with ANS Version Header {} with ANS Version {}", + static_cast(ctfANSHeader), + static_cast(ansVersion))}; + } else { + ret = ctfANSHeader; + } + } else { + if (isHeaderUnspecified) { + ret = ansVersion; + } else { + ret = ctfANSHeader; + } + } + + return ret; + }; + + template + o2::ctf::CTFIOSize entropyCodeRANSCompat(const input_IT srcBegin, const input_IT srcEnd, int slot, uint8_t symbolTablePrecision, buffer_T* buffer = nullptr, const std::any& encoderExt = {}, float memfc = 1.f); + + template + o2::ctf::CTFIOSize entropyCodeRANSV1(const input_IT srcBegin, const input_IT srcEnd, int slot, Metadata::OptStore opt, buffer_T* buffer = nullptr, const std::any& encoderExt = {}, float memfc = 1.f); + + template + o2::ctf::CTFIOSize encodeRANSV1External(const input_IT srcBegin, const input_IT srcEnd, int slot, const std::any& encoderExt, buffer_T* buffer = nullptr, double_t sizeEstimateSafetyFactor = 1); + + template + o2::ctf::CTFIOSize encodeRANSV1Inplace(const input_IT srcBegin, const input_IT srcEnd, int slot, Metadata::OptStore opt, buffer_T* buffer = nullptr, double_t sizeEstimateSafetyFactor = 1); + +#ifndef __CLING__ + template + o2::ctf::CTFIOSize pack(const input_IT srcBegin, const input_IT srcEnd, int slot, rans::Metrics::value_type> metrics, buffer_T* buffer = nullptr); + + template + inline o2::ctf::CTFIOSize pack(const input_IT srcBegin, const input_IT srcEnd, int slot, buffer_T* buffer = nullptr) + { + using source_type = typename std::iterator_traits::value_type; + + rans::Metrics metrics{}; + metrics.getDatasetProperties().numSamples = std::distance(srcBegin, srcEnd); + + if (metrics.getDatasetProperties().numSamples != 0) { + const auto [minIter, maxIter] = std::minmax_element(srcBegin, srcEnd); + metrics.getDatasetProperties().min = *minIter; + metrics.getDatasetProperties().max = *maxIter; + + // special case: if min === max, the range is 0 and the data can be reconstructed just via the metadata. + metrics.getDatasetProperties().alphabetRangeBits = + rans::utils::getRangeBits(metrics.getDatasetProperties().min, + metrics.getDatasetProperties().max); + } + + return pack(srcBegin, srcEnd, slot, metrics, buffer); + } +#endif + + template + o2::ctf::CTFIOSize store(const input_IT srcBegin, const input_IT srcEnd, int slot, Metadata::OptStore opt, buffer_T* buffer = nullptr); + + // decode + template + CTFIOSize decodeCompatImpl(dst_IT dest, int slot, const std::any& decoderExt) const; + + template + CTFIOSize decodeRansV1Impl(dst_IT dest, int slot, const std::any& decoderExt) const; + + template + CTFIOSize decodeUnpackImpl(dst_IT dest, int slot) const; + + template + CTFIOSize decodeCopyImpl(dst_IT dest, int slot) const; + + ClassDefNV(EncodedBlocks, 3); +}; // namespace ctf ///_____________________________________________________________________________ /// read from tree to non-flat object @@ -695,12 +846,13 @@ void EncodedBlocks::clear() template auto EncodedBlocks::getImage(const void* newHead) { + assert(newHead); auto image(*get(newHead)); // 1st make a shalow copy // now fix its pointers // we don't modify newHead, but still need to remove constness for relocation interface relocate(image.mRegistry.head, const_cast(reinterpret_cast(newHead)), reinterpret_cast(&image)); - return std::move(image); + return image; } ///_____________________________________________________________________________ @@ -708,8 +860,10 @@ auto EncodedBlocks::getImage(const void* newHead) template inline auto EncodedBlocks::create(void* head, size_t sz) { + const H defh; auto b = get(head); b->init(sz); + b->setHeader(defh); return b; } @@ -748,7 +902,7 @@ void EncodedBlocks::print(const std::string& prefix, int verbosity) con ndata += mBlocks[i].getNData(); nlit += mBlocks[i].getNLiterals(); } - LOG(info) << prefix << N << " blocks, input size: " << inpSize << ", output size: " << size() + LOG(info) << prefix << N << " blocks, input size: " << inpSize << ", output size: " << outputsize() << " NDictWords: " << ndict << " NDataWords: " << ndata << " NLiteralWords: " << nlit; } } @@ -756,9 +910,9 @@ void EncodedBlocks::print(const std::string& prefix, int verbosity) con ///_____________________________________________________________________________ template template -inline o2::ctf::CTFIOSize EncodedBlocks::decode(container_T& dest, // destination container - int slot, // slot of the block to decode - const void* decoderExt) const // optional externally provided decoder +inline o2::ctf::CTFIOSize EncodedBlocks::decode(container_T& dest, // destination container + int slot, // slot of the block to decode + const std::any& decoderExt) const // optional externally provided decoder { dest.resize(mMetadata[slot].messageLength); // allocate output buffer return decode(std::begin(dest), slot, decoderExt); @@ -767,54 +921,190 @@ inline o2::ctf::CTFIOSize EncodedBlocks::decode(container_T& dest, ///_____________________________________________________________________________ template template , bool>> -o2::ctf::CTFIOSize EncodedBlocks::decode(D_IT dest, // iterator to destination - int slot, // slot of the block to decode - const void* decoderExt) const // optional externally provided decoder +CTFIOSize EncodedBlocks::decode(D_IT dest, // iterator to destination + int slot, // slot of the block to decode + const std::any& decoderExt) const // optional externally provided decoder { + // get references to the right data + const auto& ansVersion = getANSHeader(); const auto& block = mBlocks[slot]; const auto& md = mMetadata[slot]; + LOGP(debug, "Slot{} | NStored={} Ndict={} nData={}, MD: messageLength:{} opt:{} min:{} max:{} offs:{} width:{} ", slot, block.getNStored(), block.getNDict(), block.getNData(), md.messageLength, (int)md.opt, md.min, md.max, md.literalsPackingOffset, md.literalsPackingWidth); - using dest_t = typename std::iterator_traits::value_type; + constexpr size_t word_size = sizeof(W); - // decode - if (block.getNStored()) { + if (ansVersion == ANSVersionCompat) { + if (!block.getNStored()) { + return {0, md.getUncompressedSize(), md.getCompressedSize() * word_size}; + } if (md.opt == Metadata::OptStore::EENCODE) { - if (!decoderExt && !block.getNDict()) { - LOG(error) << "Dictionaty is not saved for slot " << slot << " and no external decoder is provided"; - throw std::runtime_error("Dictionary is not saved and no external decoder provided"); - } - const o2::rans::LiteralDecoder64* decoder = reinterpret_cast*>(decoderExt); - std::unique_ptr> decoderLoc; - if (block.getNDict()) { // if dictionaty is saved, prefer it - decoderLoc = std::make_unique>(this->getFrequencyTable(slot)); - decoder = decoderLoc.get(); - } else { // verify that decoded corresponds to stored metadata - if (md.min != decoder->getMinSymbol()) { - LOG(error) << "Mismatch between min=" << md.min << " symbol in metadata and those in external decoder " - << decoder->getMinSymbol() << " for slot " << slot; - throw std::runtime_error("Mismatch between min symbol in metadata and the one in external decoder"); - } - } - // load incompressible symbols if they existed - std::vector literals; - if (block.getNLiterals()) { - // note: here we have to use md.nLiterals (original number of literal words) rather than md.nLiteralWords == block.getNLiterals() - // (number of W-words in the EncodedBlock occupied by literals) as we cast literals stored in W-word array - // to D-word array - literals = std::vector{reinterpret_cast(block.getLiterals()), reinterpret_cast(block.getLiterals()) + md.nLiterals}; + return decodeCompatImpl(dest, slot, decoderExt); + } else { + return decodeCopyImpl(dest, slot); + } + } else if (ansVersion == ANSVersion1) { + if (md.opt == Metadata::OptStore::PACK) { + return decodeUnpackImpl(dest, slot); + } + if (!block.getNStored()) { + return {0, md.getUncompressedSize(), md.getCompressedSize() * word_size}; + } + if (md.opt == Metadata::OptStore::EENCODE) { + return decodeRansV1Impl(dest, slot, decoderExt); + } else { + return decodeCopyImpl(dest, slot); + } + } else { + throw std::runtime_error("unsupported ANS Version"); + } +}; + +#ifndef __CLING__ +template +template +CTFIOSize EncodedBlocks::decodeCompatImpl(dst_IT dstBegin, int slot, const std::any& decoderExt) const +{ + + // get references to the right data + const auto& block = mBlocks[slot]; + const auto& md = mMetadata[slot]; + + using dst_type = typename std::iterator_traits::value_type; + using decoder_type = typename rans::compat::decoder_type; + + std::optional inplaceDecoder{}; + if (md.nDictWords > 0) { + inplaceDecoder = decoder_type{std::get>(this->getDictionary(slot))}; + } else if (!decoderExt.has_value()) { + throw std::runtime_error("neither dictionary nor external decoder provided"); + } + + auto getDecoder = [&]() -> const decoder_type& { + if (inplaceDecoder.has_value()) { + return inplaceDecoder.value(); + } else { + return std::any_cast(decoderExt); + } + }; + + const size_t NDecoderStreams = rans::compat::defaults::CoderPreset::nStreams; + + if (block.getNLiterals()) { + auto* literalsEnd = reinterpret_cast(block.getLiterals()) + md.nLiterals; + getDecoder().process(block.getData() + block.getNData(), dstBegin, md.messageLength, NDecoderStreams, literalsEnd); + } else { + getDecoder().process(block.getData() + block.getNData(), dstBegin, md.messageLength, NDecoderStreams); + } + return {0, md.getUncompressedSize(), md.getCompressedSize() * sizeof(W)}; +}; + +template +template +CTFIOSize EncodedBlocks::decodeRansV1Impl(dst_IT dstBegin, int slot, const std::any& decoderExt) const +{ + + // get references to the right data + const auto& block = mBlocks[slot]; + const auto& md = mMetadata[slot]; + + using dst_type = typename std::iterator_traits::value_type; + using decoder_type = typename rans::defaultDecoder_type; + + std::optional inplaceDecoder{}; + if (md.nDictWords > 0) { + std::visit([&](auto&& arg) { inplaceDecoder = decoder_type{arg}; }, this->getDictionary(slot)); + } else if (!decoderExt.has_value()) { + throw std::runtime_error("no dictionary nor external decoder provided"); + } + + auto getDecoder = [&]() -> const decoder_type& { + if (inplaceDecoder.has_value()) { + return inplaceDecoder.value(); + } else { + return std::any_cast(decoderExt); + } + }; + + // verify decoders + [&]() { + const decoder_type& decoder = getDecoder(); + const size_t decoderSymbolTablePrecision = decoder.getSymbolTablePrecision(); + + if (md.probabilityBits != decoderSymbolTablePrecision) { + throw std::runtime_error(fmt::format( + "Missmatch in decoder renorming precision vs metadata:{} Bits vs {} Bits.", + md.probabilityBits, decoderSymbolTablePrecision)); + } + + if (md.streamSize != rans::utils::getStreamingLowerBound_v) { + throw std::runtime_error("Streaming lower bound of dataset and decoder do not match"); + } + }(); + + // do the actual decoding + if (block.getNLiterals()) { + std::vector literals(md.nLiterals); + rans::unpack(block.getLiterals(), md.nLiterals, literals.data(), md.literalsPackingWidth, md.literalsPackingOffset); + getDecoder().process(block.getData() + block.getNData(), dstBegin, md.messageLength, md.nStreams, literals.end()); + } else { + getDecoder().process(block.getData() + block.getNData(), dstBegin, md.messageLength, md.nStreams); + } + return {0, md.getUncompressedSize(), md.getCompressedSize() * sizeof(W)}; +}; + +template +template +CTFIOSize EncodedBlocks::decodeUnpackImpl(dst_IT dest, int slot) const +{ + using dest_t = typename std::iterator_traits::value_type; + + const auto& block = mBlocks[slot]; + const auto& md = mMetadata[slot]; + + const size_t messageLength = md.messageLength; + const size_t packingWidth = md.probabilityBits; + const dest_t offset = md.min; + const auto* srcIt = block.getData(); + // we have a vector of one and the same value. All information is in the metadata + if (packingWidth == 0) { + const dest_t value = [&]() -> dest_t { + // Bugfix: We tried packing values with a width of 0 Bits; + if (md.nDataWords > 0) { + LOGP(debug, "packing bug recovery: MD nStreams:{} messageLength:{} nLiterals:{} messageWordSize:{} coderType:{} streamSize:{} probabilityBits:{} (int)opt:{} min:{} max:{} literalsPackingOffset:{} literalsPackingWidth:{} nDictWords:{} nDataWords:{} nLiteralWords:{}", + value, md.nStreams, md.messageLength, md.nLiterals, md.messageWordSize, md.coderType, md.streamSize, md.probabilityBits, (int)md.opt, md.min, md.max, md.literalsPackingOffset, md.literalsPackingWidth, md.nDictWords, md.nDataWords, md.nLiteralWords); + return offset + static_cast(*srcIt); } - decoder->process(block.getData() + block.getNData(), dest, md.messageLength, literals); - } else { // data was stored as is - using destPtr_t = typename std::iterator_traits::pointer; - destPtr_t srcBegin = reinterpret_cast(block.payload); - destPtr_t srcEnd = srcBegin + md.messageLength * sizeof(dest_t); - std::copy(srcBegin, srcEnd, dest); - // std::memcpy(dest, block.payload, md.messageLength * sizeof(dest_t)); + // normal case: + return offset; + }(); + for (size_t i = 0; i < messageLength; ++i) { + *dest++ = value; } + } else { + rans::unpack(srcIt, messageLength, dest, packingWidth, offset); } - return {0, md.getUncompressedSize(), md.getCompressedSize()}; -} + return {0, md.getUncompressedSize(), md.getCompressedSize() * sizeof(W)}; +}; + +template +template +CTFIOSize EncodedBlocks::decodeCopyImpl(dst_IT dest, int slot) const +{ + // get references to the right data + const auto& block = mBlocks[slot]; + const auto& md = mMetadata[slot]; + + using dest_t = typename std::iterator_traits::value_type; + using decoder_t = typename rans::compat::decoder_type; + using destPtr_t = typename std::iterator_traits::pointer; + + destPtr_t srcBegin = reinterpret_cast(block.payload); + destPtr_t srcEnd = srcBegin + md.messageLength * sizeof(dest_t); + std::copy(srcBegin, srcEnd, dest); + + return {0, md.getUncompressedSize(), md.getCompressedSize() * sizeof(W)}; +}; ///_____________________________________________________________________________ template @@ -825,147 +1115,417 @@ o2::ctf::CTFIOSize EncodedBlocks::encode(const input_IT srcBegin, uint8_t symbolTablePrecision, // encoding into Metadata::OptStore opt, // option for data compression buffer_T* buffer, // optional buffer (vector) providing memory for encoded blocks - const void* encoderExt, // optional external encoder + const std::any& encoderExt, // optional external encoder float memfc) // memory allocation margin factor { - - using storageBuffer_t = W; - using input_t = typename std::iterator_traits::value_type; - using ransEncoder_t = typename rans::LiteralEncoder64; - using ransState_t = typename ransEncoder_t::coder_t; - using ransStream_t = typename ransEncoder_t::stream_t; - - // assert at compile time that output types align so that padding is not necessary. - static_assert(std::is_same_v); - static_assert(std::is_same_v); - // fill a new block assert(slot == mRegistry.nFilledBlocks); mRegistry.nFilledBlocks++; const size_t messageLength = std::distance(srcBegin, srcEnd); // cover three cases: - // * empty source message: no entropy coding + // * empty source message: no co // * source message to pass through without any entropy coding // * source message where entropy coding should be applied // case 1: empty source message if (messageLength == 0) { - mMetadata[slot] = Metadata{0, 0, sizeof(input_t), sizeof(ransState_t), sizeof(ransStream_t), symbolTablePrecision, Metadata::OptStore::NODATA, 0, 0, 0, 0, 0}; + mMetadata[slot] = Metadata{}; + mMetadata[slot].opt = Metadata::OptStore::NODATA; return {}; } + if (detail::mayEEncode(opt)) { + const ANSHeader& ansVersion = getANSHeader(); + if (ansVersion == ANSVersionCompat) { + return entropyCodeRANSCompat(srcBegin, srcEnd, slot, symbolTablePrecision, buffer, encoderExt, memfc); + } else if (ansVersion == ANSVersion1) { + return entropyCodeRANSV1(srcBegin, srcEnd, slot, opt, buffer, encoderExt, memfc); + } else { + throw std::runtime_error(fmt::format("Unsupported ANS Coder Version: {}.{}", ansVersion.majorVersion, ansVersion.minorVersion)); + } + } else if (detail::mayPack(opt)) { + return pack(srcBegin, srcEnd, slot, buffer); + } else { + return store(srcBegin, srcEnd, slot, opt, buffer); + } +}; - auto* thisBlock = &mBlocks[slot]; - auto* thisMetadata = &mMetadata[slot]; +template +template +[[nodiscard]] auto EncodedBlocks::expandStorage(size_t slot, size_t nElements, T* buffer) -> decltype(auto) +{ + // after previous relocation this (hence its data members) are not guaranteed to be valid + auto* old = get(buffer->data()); + auto* thisBlock = &(old->mBlocks[slot]); + auto* thisMetadata = &(old->mMetadata[slot]); // resize underlying buffer of block if necessary and update all pointers. - auto expandStorage = [&](int additionalElements) { - auto* const blockHead = get(thisBlock->registry->head); // extract pointer from the block, as "this" might be invalid - const size_t additionalSize = blockHead->estimateBlockSize(additionalElements); // size in bytes!!! - if (additionalSize >= thisBlock->registry->getFreeSize()) { - LOG(debug) << "Slot " << slot << ": free size: " << thisBlock->registry->getFreeSize() << ", need " << additionalSize << " for " << additionalElements << " words"; - if (buffer) { - blockHead->expand(*buffer, blockHead->size() + (additionalSize - blockHead->getFreeSize())); - thisMetadata = &(get(buffer->data())->mMetadata[slot]); - thisBlock = &(get(buffer->data())->mBlocks[slot]); // in case of resizing this and any this.xxx becomes invalid - } else { - throw std::runtime_error("no room for encoded block in provided container"); - } + auto* const blockHead = get(thisBlock->registry->head); // extract pointer from the block, as "this" might be invalid + const size_t additionalSize = blockHead->estimateBlockSize(nElements); // additionalSize is in bytes!!! + if (additionalSize >= thisBlock->registry->getFreeSize()) { + LOGP(debug, "Slot {} with {} available words needs to allocate {} bytes for a total of {} words.", slot, thisBlock->registry->getFreeSize(), additionalSize, nElements); + if (buffer) { + blockHead->expand(*buffer, blockHead->size() + (additionalSize - blockHead->getFreeSize())); + thisMetadata = &(get(buffer->data())->mMetadata[slot]); + thisBlock = &(get(buffer->data())->mBlocks[slot]); // in case of resizing this and any this.xxx becomes invalid + } else { + throw std::runtime_error("failed to allocate additional space in provided external buffer"); } - }; + } + return std::make_pair(thisBlock, thisMetadata); +}; - // case 3: message where entropy coding should be applied - if (opt == Metadata::OptStore::EENCODE) { - // build symbol statistics - constexpr size_t SizeEstMarginAbs = 10 * 1024; - const float SizeEstMarginRel = 1.5 * memfc; +template +template +o2::ctf::CTFIOSize EncodedBlocks::entropyCodeRANSCompat(const input_IT srcBegin, const input_IT srcEnd, int slot, uint8_t symbolTablePrecision, buffer_T* buffer, const std::any& encoderExt, float memfc) +{ + using storageBuffer_t = W; + using input_t = typename std::iterator_traits::value_type; + using ransEncoder_t = typename rans::compat::encoder_type; + using ransState_t = typename ransEncoder_t::coder_type::state_type; + using ransStream_t = typename ransEncoder_t::stream_type; - const auto [inplaceEncoder, frequencyTable] = [&]() { - if (encoderExt) { - return std::make_tuple(ransEncoder_t{}, rans::FrequencyTable{}); + // assert at compile time that output types align so that padding is not necessary. + static_assert(std::is_same_v); + static_assert(std::is_same_v); + + auto* thisBlock = &mBlocks[slot]; + auto* thisMetadata = &mMetadata[slot]; + + // build symbol statistics + constexpr size_t SizeEstMarginAbs = 10 * 1024; + const float SizeEstMarginRel = 1.5 * memfc; + + const size_t messageLength = std::distance(srcBegin, srcEnd); + rans::DenseHistogram frequencyTable{}; + rans::compat::encoder_type inplaceEncoder{}; + + try { + std::tie(inplaceEncoder, frequencyTable) = [&]() { + if (encoderExt.has_value()) { + return std::make_tuple(ransEncoder_t{}, rans::DenseHistogram{}); } else { - rans::FrequencyTable frequencyTable = rans::makeFrequencyTableFromSamples(srcBegin, srcEnd); - RenormedFrequencyTable renormedFrequencyTable = rans::renorm(frequencyTable, symbolTablePrecision); - return std::make_tuple(ransEncoder_t{renormedFrequencyTable}, frequencyTable); + auto histogram = rans::makeDenseHistogram::fromSamples(srcBegin, srcEnd); + auto encoder = rans::compat::makeEncoder::fromHistogram(histogram, symbolTablePrecision); + return std::make_tuple(std::move(encoder), std::move(histogram)); } }(); - ransEncoder_t const* const encoder = encoderExt ? reinterpret_cast(encoderExt) : &inplaceEncoder; - - // estimate size of encode buffer - int dataSize = rans::calculateMaxBufferSize(messageLength, encoder->getAlphabetRangeBits(), sizeof(input_t)); // size in bytes - // preliminary expansion of storage based on dict size + estimated size of encode buffer - dataSize = SizeEstMarginAbs + int(SizeEstMarginRel * (dataSize / sizeof(storageBuffer_t))) + (sizeof(input_t) < sizeof(storageBuffer_t)); // size in words of output stream - expandStorage(frequencyTable.size() + dataSize); - // store dictionary first - if (!frequencyTable.empty()) { - thisBlock->storeDict(frequencyTable.size(), frequencyTable.data()); - LOGP(debug, "StoreDict {} bytes, offs: {}:{}", frequencyTable.size() * sizeof(W), thisBlock->getOffsDict(), thisBlock->getOffsDict() + frequencyTable.size() * sizeof(W)); + } catch (const rans::HistogramError& error) { + LOGP(warning, "Failed to build Dictionary for rANS encoding, using fallback option"); + return store(srcBegin, srcEnd, slot, this->FallbackStorageType, buffer); + } + const ransEncoder_t& encoder = encoderExt.has_value() ? std::any_cast(encoderExt) : inplaceEncoder; + + // estimate size of encode buffer + int dataSize = rans::compat::calculateMaxBufferSizeB(messageLength, rans::compat::getAlphabetRangeBits(encoder.getSymbolTable())); // size in bytes + // preliminary expansion of storage based on dict size + estimated size of encode buffer + dataSize = SizeEstMarginAbs + int(SizeEstMarginRel * (dataSize / sizeof(storageBuffer_t))) + (sizeof(input_t) < sizeof(storageBuffer_t)); // size in words of output stream + + const auto view = rans::trim(rans::makeHistogramView(frequencyTable)); + std::tie(thisBlock, thisMetadata) = expandStorage(slot, view.size() + dataSize, buffer); + + // store dictionary first + + if (!view.empty()) { + thisBlock->storeDict(view.size(), view.data()); + LOGP(debug, "StoreDict {} bytes, offs: {}:{}", view.size() * sizeof(W), thisBlock->getOffsDict(), thisBlock->getOffsDict() + view.size() * sizeof(W)); + } + // vector of incompressible literal symbols + std::vector literals; + // directly encode source message into block buffer. + storageBuffer_t* const blockBufferBegin = thisBlock->getCreateData(); + const size_t maxBufferSize = thisBlock->registry->getFreeSize(); // note: "this" might be not valid after expandStorage call!!! + const auto [encodedMessageEnd, literalsEnd] = encoder.process(srcBegin, srcEnd, blockBufferBegin, std::back_inserter(literals)); + rans::utils::checkBounds(encodedMessageEnd, blockBufferBegin + maxBufferSize / sizeof(W)); + dataSize = encodedMessageEnd - thisBlock->getDataPointer(); + thisBlock->setNData(dataSize); + thisBlock->realignBlock(); + LOGP(debug, "StoreData {} bytes, offs: {}:{}", dataSize * sizeof(W), thisBlock->getOffsData(), thisBlock->getOffsData() + dataSize * sizeof(W)); + // update the size claimed by encode message directly inside the block + + // store incompressible symbols if any + const size_t nLiteralSymbols = literals.size(); + const size_t nLiteralWords = [&]() { + if (!literals.empty()) { + const size_t nSymbols = literals.size(); + // introduce padding in case literals don't align; + const size_t nLiteralSymbolsPadded = calculatePaddedSize(nSymbols); + literals.resize(nLiteralSymbolsPadded, {}); + + const size_t nLiteralStorageElems = calculateNDestTElements(nSymbols); + std::tie(thisBlock, thisMetadata) = expandStorage(slot, nLiteralStorageElems, buffer); + thisBlock->storeLiterals(nLiteralStorageElems, reinterpret_cast(literals.data())); + LOGP(debug, "StoreLiterals {} bytes, offs: {}:{}", nLiteralStorageElems * sizeof(W), thisBlock->getOffsLiterals(), thisBlock->getOffsLiterals() + nLiteralStorageElems * sizeof(W)); + return nLiteralStorageElems; } - // vector of incompressible literal symbols - std::vector literals; - // directly encode source message into block buffer. - storageBuffer_t* const blockBufferBegin = thisBlock->getCreateData(); - const size_t maxBufferSize = thisBlock->registry->getFreeSize(); // note: "this" might be not valid after expandStorage call!!! - const auto encodedMessageEnd = encoder->process(srcBegin, srcEnd, blockBufferBegin, literals); - rans::utils::checkBounds(encodedMessageEnd, blockBufferBegin + maxBufferSize / sizeof(W)); - dataSize = encodedMessageEnd - thisBlock->getDataPointer(); - thisBlock->setNData(dataSize); - thisBlock->realignBlock(); - LOGP(debug, "StoreData {} bytes, offs: {}:{}", dataSize * sizeof(W), thisBlock->getOffsData(), thisBlock->getOffsData() + dataSize * sizeof(W)); - // update the size claimed by encode message directly inside the block - - // store incompressible symbols if any - const size_t nLiteralSymbols = literals.size(); - const size_t nLiteralWords = [&]() { - if (!literals.empty()) { - const size_t nSymbols = literals.size(); - // introduce padding in case literals don't align; - const size_t nLiteralSymbolsPadded = calculatePaddedSize(nSymbols); - literals.resize(nLiteralSymbolsPadded, {}); - - const size_t nLiteralStorageElems = calculateNDestTElements(nSymbols); - expandStorage(nLiteralStorageElems); - thisBlock->storeLiterals(nLiteralStorageElems, reinterpret_cast(literals.data())); - LOGP(debug, "StoreLiterals {} bytes, offs: {}:{}", nLiteralStorageElems * sizeof(W), thisBlock->getOffsLiterals(), thisBlock->getOffsLiterals() + nLiteralStorageElems * sizeof(W)); - return nLiteralStorageElems; - } - return size_t(0); - }(); + return size_t(0); + }(); + + LOGP(debug, "Min, {} Max, {}, size, {}, nSamples {}", view.getMin(), view.getMax(), view.size(), frequencyTable.getNumSamples()); + + *thisMetadata = detail::makeMetadataRansCompat(encoder.getNStreams(), + messageLength, + nLiteralSymbols, + encoder.getSymbolTable().getPrecision(), + view.getMin(), + view.getMax(), + view.size(), + dataSize, + nLiteralWords); + + return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize() * sizeof(W)}; +} + +template +template +o2::ctf::CTFIOSize EncodedBlocks::entropyCodeRANSV1(const input_IT srcBegin, const input_IT srcEnd, int slot, Metadata::OptStore opt, buffer_T* buffer, const std::any& encoderExt, float memfc) +{ + CTFIOSize encoderStatistics{}; + + const size_t nSamples = std::distance(srcBegin, srcEnd); + if (detail::mayPack(opt) && nSamples < PackingThreshold) { + encoderStatistics = pack(srcBegin, srcEnd, slot, buffer); + } else { - *thisMetadata = Metadata{messageLength, - nLiteralSymbols, - sizeof(input_t), - sizeof(ransState_t), - sizeof(ransStream_t), - static_cast(encoder->getSymbolTablePrecision()), - opt, - encoder->getMinSymbol(), - encoder->getMaxSymbol(), - static_cast(frequencyTable.size()), - dataSize, - static_cast(nLiteralWords)}; - } else { // store original data w/o EEncoding - // FIXME(milettri): we should be able to do without an intermediate vector; - // provided iterator is not necessarily pointer, need to use intermediate vector!!! - - // introduce padding in case literals don't align; - const size_t nSourceElemsPadded = calculatePaddedSize(messageLength); - std::vector tmp(nSourceElemsPadded, {}); - std::copy(srcBegin, srcEnd, std::begin(tmp)); - - const size_t nBufferElems = calculateNDestTElements(messageLength); - expandStorage(nBufferElems); - thisBlock->storeData(nBufferElems, reinterpret_cast(tmp.data())); - - *thisMetadata = Metadata{messageLength, 0, sizeof(input_t), sizeof(ransState_t), sizeof(storageBuffer_t), symbolTablePrecision, opt, 0, 0, 0, static_cast(nBufferElems), 0}; + if (encoderExt.has_value()) { + encoderStatistics = encodeRANSV1External(srcBegin, srcEnd, slot, encoderExt, buffer, memfc); + } else { + encoderStatistics = encodeRANSV1Inplace(srcBegin, srcEnd, slot, opt, buffer, memfc); + } } - return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize()}; + return encoderStatistics; } +template +template +CTFIOSize EncodedBlocks::encodeRANSV1External(const input_IT srcBegin, const input_IT srcEnd, int slot, const std::any& encoderExt, buffer_T* buffer, double_t sizeEstimateSafetyFactor) +{ + using storageBuffer_t = W; + using input_t = typename std::iterator_traits::value_type; + using ransEncoder_t = typename internal::ExternalEntropyCoder::encoder_type; + using ransState_t = typename ransEncoder_t::coder_type::state_type; + using ransStream_t = typename ransEncoder_t::stream_type; + + // assert at compile time that output types align so that padding is not necessary. + static_assert(std::is_same_v); + static_assert(std::is_same_v); + + auto* thisBlock = &mBlocks[slot]; + auto* thisMetadata = &mMetadata[slot]; + + const size_t messageLength = std::distance(srcBegin, srcEnd); + internal::ExternalEntropyCoder encoder{std::any_cast(encoderExt)}; + + const size_t payloadSizeWords = encoder.template computePayloadSizeEstimate(messageLength); + std::tie(thisBlock, thisMetadata) = expandStorage(slot, payloadSizeWords, buffer); + + // encode payload + auto encodedMessageEnd = encoder.encode(srcBegin, srcEnd, thisBlock->getCreateData(), thisBlock->getEndOfBlock()); + const size_t dataSize = std::distance(thisBlock->getCreateData(), encodedMessageEnd); + thisBlock->setNData(dataSize); + thisBlock->realignBlock(); + LOGP(debug, "StoreData {} bytes, offs: {}:{}", dataSize * sizeof(storageBuffer_t), thisBlock->getOffsData(), thisBlock->getOffsData() + dataSize * sizeof(storageBuffer_t)); + // update the size claimed by encoded message directly inside the block + + // encode literals + size_t literalsSize = 0; + if (encoder.getNIncompressibleSamples() > 0) { + const size_t literalsBufferSizeWords = encoder.template computePackedIncompressibleSize(); + std::tie(thisBlock, thisMetadata) = expandStorage(slot, literalsBufferSizeWords, buffer); + auto literalsEnd = encoder.writeIncompressible(thisBlock->getCreateLiterals(), thisBlock->getEndOfBlock()); + literalsSize = std::distance(thisBlock->getCreateLiterals(), literalsEnd); + thisBlock->setNLiterals(literalsSize); + thisBlock->realignBlock(); + LOGP(debug, "StoreLiterals {} bytes, offs: {}:{}", literalsSize * sizeof(storageBuffer_t), thisBlock->getOffsLiterals(), thisBlock->getOffsLiterals() + literalsSize * sizeof(storageBuffer_t)); + } + + // write metadata + const auto& symbolTable = encoder.getEncoder().getSymbolTable(); + *thisMetadata = detail::makeMetadataRansV1(encoder.getEncoder().getNStreams(), + rans::utils::getStreamingLowerBound_v, + messageLength, + encoder.getNIncompressibleSamples(), + symbolTable.getPrecision(), + symbolTable.getOffset(), + symbolTable.getOffset() + symbolTable.size(), + encoder.getIncompressibleSymbolOffset(), + encoder.getIncompressibleSymbolPackingBits(), + 0, + dataSize, + literalsSize); + + return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize() * sizeof(W)}; +}; + +template +template +CTFIOSize EncodedBlocks::encodeRANSV1Inplace(const input_IT srcBegin, const input_IT srcEnd, int slot, Metadata::OptStore opt, buffer_T* buffer, double_t sizeEstimateSafetyFactor) +{ + using storageBuffer_t = W; + using input_t = typename std::iterator_traits::value_type; + using ransEncoder_t = typename rans::denseEncoder_type; + using ransState_t = typename ransEncoder_t::coder_type::state_type; + using ransStream_t = typename ransEncoder_t::stream_type; + + // assert at compile time that output types align so that padding is not necessary. + static_assert(std::is_same_v); + static_assert(std::is_same_v); + + auto* thisBlock = &mBlocks[slot]; + auto* thisMetadata = &mMetadata[slot]; + + internal::InplaceEntropyCoder encoder{}; + rans::SourceProxy proxy{srcBegin, srcEnd, [](input_IT begin, input_IT end) { + const size_t nSamples = std::distance(begin, end); + return (!std::is_pointer_v && (nSamples < rans::utils::pow2(23))); + }}; + + try { + if (proxy.isCached()) { + encoder = internal::InplaceEntropyCoder{proxy.beginCache(), proxy.endCache()}; + } else { + encoder = internal::InplaceEntropyCoder{proxy.beginIter(), proxy.endIter()}; + } + } catch (const rans::HistogramError& error) { + LOGP(warning, "Failed to build Dictionary for rANS encoding, using fallback option"); + if (proxy.isCached()) { + return store(proxy.beginCache(), proxy.endCache(), slot, this->FallbackStorageType, buffer); + } else { + return store(proxy.beginIter(), proxy.endIter(), slot, this->FallbackStorageType, buffer); + } + } + + const rans::Metrics& metrics = encoder.getMetrics(); + /* + if constexpr (sizeof(input_t) > 2) { + const auto& dp = metrics.getDatasetProperties(); + LOGP(info, "Metrics:{{slot: {}, numSamples: {}, min: {}, max: {}, alphabetRangeBits: {}, nUsedAlphabetSymbols: {}, preferPacking: {}}}", slot, dp.numSamples, dp.min, dp.max, dp.alphabetRangeBits, dp.nUsedAlphabetSymbols, metrics.getSizeEstimate().preferPacking()); + } + */ + if (detail::mayPack(opt) && metrics.getSizeEstimate().preferPacking()) { + if (proxy.isCached()) { + return pack(proxy.beginCache(), proxy.endCache(), slot, metrics, buffer); + } else { + return pack(proxy.beginIter(), proxy.endIter(), slot, metrics, buffer); + }; + } + + encoder.makeEncoder(); + + const rans::SizeEstimate sizeEstimate = metrics.getSizeEstimate(); + const size_t bufferSizeWords = rans::utils::nBytesTo((sizeEstimate.getCompressedDictionarySize() + + sizeEstimate.getCompressedDatasetSize() + + sizeEstimate.getIncompressibleSize()) * + sizeEstimateSafetyFactor); + std::tie(thisBlock, thisMetadata) = expandStorage(slot, bufferSizeWords, buffer); + + // encode dict + auto encodedDictEnd = encoder.writeDictionary(thisBlock->getCreateDict(), thisBlock->getEndOfBlock()); + const size_t dictSize = std::distance(thisBlock->getCreateDict(), encodedDictEnd); + thisBlock->setNDict(dictSize); + thisBlock->realignBlock(); + LOGP(debug, "StoreDict {} bytes, offs: {}:{}", dictSize * sizeof(storageBuffer_t), thisBlock->getOffsDict(), thisBlock->getOffsDict() + dictSize * sizeof(storageBuffer_t)); + + // encode payload + auto encodedMessageEnd = thisBlock->getCreateData(); + if (proxy.isCached()) { + encodedMessageEnd = encoder.encode(proxy.beginCache(), proxy.endCache(), thisBlock->getCreateData(), thisBlock->getEndOfBlock()); + } else { + encodedMessageEnd = encoder.encode(proxy.beginIter(), proxy.endIter(), thisBlock->getCreateData(), thisBlock->getEndOfBlock()); + } + const size_t dataSize = std::distance(thisBlock->getCreateData(), encodedMessageEnd); + thisBlock->setNData(dataSize); + thisBlock->realignBlock(); + LOGP(debug, "StoreData {} bytes, offs: {}:{}", dataSize * sizeof(storageBuffer_t), thisBlock->getOffsData(), thisBlock->getOffsData() + dataSize * sizeof(storageBuffer_t)); + // update the size claimed by encoded message directly inside the block + + // encode literals + size_t literalsSize{}; + if (encoder.getNIncompressibleSamples() > 0) { + auto literalsEnd = encoder.writeIncompressible(thisBlock->getCreateLiterals(), thisBlock->getEndOfBlock()); + literalsSize = std::distance(thisBlock->getCreateLiterals(), literalsEnd); + thisBlock->setNLiterals(literalsSize); + thisBlock->realignBlock(); + LOGP(debug, "StoreLiterals {} bytes, offs: {}:{}", literalsSize * sizeof(storageBuffer_t), thisBlock->getOffsLiterals(), thisBlock->getOffsLiterals() + literalsSize * sizeof(storageBuffer_t)); + } + + // write metadata + *thisMetadata = detail::makeMetadataRansV1(encoder.getNStreams(), + rans::utils::getStreamingLowerBound_v, + std::distance(srcBegin, srcEnd), + encoder.getNIncompressibleSamples(), + encoder.getSymbolTablePrecision(), + *metrics.getCoderProperties().min, + *metrics.getCoderProperties().max, + metrics.getDatasetProperties().min, + metrics.getDatasetProperties().alphabetRangeBits, + dictSize, + dataSize, + literalsSize); + + return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize() * sizeof(W)}; +}; // namespace ctf + +template +template +o2::ctf::CTFIOSize EncodedBlocks::pack(const input_IT srcBegin, const input_IT srcEnd, int slot, rans::Metrics::value_type> metrics, buffer_T* buffer) +{ + using storageBuffer_t = W; + using input_t = typename std::iterator_traits::value_type; + + const size_t messageLength = metrics.getDatasetProperties().numSamples; + const auto alphabetRangeBits = metrics.getDatasetProperties().alphabetRangeBits; + + auto* thisBlock = &mBlocks[slot]; + auto* thisMetadata = &mMetadata[slot]; + size_t packedSize = 0; + + if (messageLength == 0) { + *thisMetadata = detail::makeMetadataPack(0, 0, 0, 0); + } else if (metrics.getDatasetProperties().alphabetRangeBits == 0) { + *thisMetadata = detail::makeMetadataPack(messageLength, 0, *srcBegin, 0); + } else { + internal::Packer packer{metrics}; + size_t packingBufferWords = packer.template getPackingBufferSize(messageLength); + std::tie(thisBlock, thisMetadata) = expandStorage(slot, packingBufferWords, buffer); + auto packedMessageEnd = packer.pack(srcBegin, srcEnd, thisBlock->getCreateData(), thisBlock->getEndOfBlock()); + packedSize = std::distance(thisBlock->getCreateData(), packedMessageEnd); + *thisMetadata = detail::makeMetadataPack(messageLength, packer.getPackingWidth(), packer.getOffset(), packedSize); + thisBlock->setNData(packedSize); + thisBlock->realignBlock(); + } + + LOGP(debug, "StoreData {} bytes, offs: {}:{}", packedSize * sizeof(storageBuffer_t), thisBlock->getOffsData(), thisBlock->getOffsData() + packedSize * sizeof(storageBuffer_t)); + return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize() * sizeof(W)}; +}; + +template +template +o2::ctf::CTFIOSize EncodedBlocks::store(const input_IT srcBegin, const input_IT srcEnd, int slot, Metadata::OptStore opt, buffer_T* buffer) +{ + using storageBuffer_t = W; + using input_t = typename std::iterator_traits::value_type; + + const size_t messageLength = std::distance(srcBegin, srcEnd); + // introduce padding in case literals don't align; + const size_t nSourceElemsPadded = calculatePaddedSize(messageLength); + std::vector tmp(nSourceElemsPadded, {}); + std::copy(srcBegin, srcEnd, std::begin(tmp)); + + const size_t nBufferElems = calculateNDestTElements(messageLength); + auto [thisBlock, thisMetadata] = expandStorage(slot, nBufferElems, buffer); + thisBlock->storeData(nBufferElems, reinterpret_cast(tmp.data())); + + *thisMetadata = detail::makeMetadataStore(messageLength, opt, nBufferElems); + + return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize() * sizeof(W)}; +}; + /// create a special EncodedBlocks containing only dictionaries made from provided vector of frequency tables template -std::vector EncodedBlocks::createDictionaryBlocks(const std::vector& vfreq, const std::vector& vmd) +std::vector EncodedBlocks::createDictionaryBlocks(const std::vector>& vfreq, const std::vector& vmd) { + if (vfreq.size() != N) { throw std::runtime_error(fmt::format("mismatch between the size of frequencies vector {} and number of blocks {}", vfreq.size(), N)); } @@ -976,9 +1536,12 @@ std::vector EncodedBlocks::createDictionaryBlocks(const std::vect std::vector vdict(sz); // memory space for dictionary auto dictBlocks = create(vdict.data(), sz); for (int ib = 0; ib < N; ib++) { - if (vfreq[ib].size()) { - LOG(info) << "adding dictionary of " << vfreq[ib].size() << " words for block " << ib << ", min/max= " << vfreq[ib].getMinSymbol() << "/" << vfreq[ib].getMaxSymbol(); - dictBlocks->mBlocks[ib].storeDict(vfreq[ib].size(), vfreq[ib].data()); + const auto& thisHistogram = vfreq[ib]; + const auto view = rans::trim(rans::makeHistogramView(thisHistogram)); + + if (!view.empty()) { + LOG(info) << "adding dictionary of " << view.size() << " words for block " << ib << ", min/max= " << view.getMin() << "/" << view.getMax(); + dictBlocks->mBlocks[ib].storeDict(view.size(), view.data()); dictBlocks = get(vdict.data()); // !!! rellocation might have invalidated dictBlocks pointer dictBlocks->mMetadata[ib] = vmd[ib]; dictBlocks->mMetadata[ib].opt = Metadata::OptStore::ROOTCompression; // we will compress the dictionary with root! @@ -988,8 +1551,9 @@ std::vector EncodedBlocks::createDictionaryBlocks(const std::vect } dictBlocks->mRegistry.nFilledBlocks++; } - return std::move(vdict); + return vdict; } +#endif template void EncodedBlocks::dump(const std::string& prefix, int ncol) const diff --git a/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/FileMetaData.h b/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/FileMetaData.h index e696e3798c47d..1408e0e11714a 100644 --- a/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/FileMetaData.h +++ b/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/FileMetaData.h @@ -16,6 +16,7 @@ #include #include +#include namespace o2 { @@ -26,25 +27,25 @@ class DataTakingContext; namespace dataformats { -struct FileMetaData { // https://docs.google.com/document/d/1nH9EZEFBSpuZwOWs3RBcfy-6aRChgAqClBv6G06MjH4/edit - std::string LHCPeriod{}; // 1, LHC data taking period + detector name, in case of individual detector data stream, required - std::string lurl{}; // 3, the local EPN path to the CTF or calibration file, required - std::string type{}; // 4, raw or calib or other; default is other, optional - std::string guid{}; // 7, default is auto-generated, optional - std::string surl{}; // 8, the remote storage path where we store the data file, optional - std::string curl{}; // 9, the Grid catalogue path, optional - std::string md5{}; //10, default the checksum of the lurl file; only filled after a successful transfer, if needed, optional - std::string xxhash{}; //11, default calculated from the lurl file, only filled after a successful transfer, if needed, optional - std::string seName{}; // 12, default is taken from the configuration file, optional - std::string seioDaemons{}; // 13, default is taken from the configuration file, optional - std::string priority{}; // 14, low or high; default is low, optional - std::string detComposition{}; // 17, contains a list of detectors, optional - std::string run{}; // 2, run number, required - long ctime{}; // 5, default the timestamp of the lurl file, optional - size_t size{}; // 6, default the size of the lurl file, optional - int persistent{}; // 16, default is forever, optional +struct FileMetaData { // https://docs.google.com/document/d/1nH9EZEFBSpuZwOWs3RBcfy-6aRChgAqClBv6G06MjH4/edit + std::string LHCPeriod{}; // 1, LHC data taking period + detector name, in case of individual detector data stream, required + std::string lurl{}; // 3, the local EPN path to the CTF or calibration file, required + std::string type{}; // 4, raw or calib or other; default is other, optional + std::string guid{}; // 7, default is auto-generated, optional + std::string surl{}; // 8, the remote storage path where we store the data file, optional + std::string curl{}; // 9, the Grid catalogue path, optional + std::string md5{}; // 10, default the checksum of the lurl file; only filled after a successful transfer, if needed, optional + std::string xxhash{}; // 11, default calculated from the lurl file, only filled after a successful transfer, if needed, optional + std::string seName{}; // 12, default is taken from the configuration file, optional + std::string seioDaemons{}; // 13, default is taken from the configuration file, optional + std::string priority{}; // 14, low or high; default is low, optional + std::string detComposition{}; // 17, contains a list of detectors, optional + std::string run{}; // 2, run number, required + long ctime{}; // 5, default the timestamp of the lurl file, optional + size_t size{}; // 6, default the size of the lurl file, optional + int persistent{}; // 16, default is forever, optional std::vector tfOrbits; // 15, comma-sep. list of 1st orbits of TFs, optional - bool fillFileData(const std::string& fname); + bool fillFileData(const std::string& fname, bool fillmd5 = false, const std::string& tmpEnding = ""); void setDataTakingContext(const o2::framework::DataTakingContext& dtc); std::string asString() const; }; diff --git a/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/Metadata.h b/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/Metadata.h new file mode 100644 index 0000000000000..975522767dce1 --- /dev/null +++ b/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/Metadata.h @@ -0,0 +1,206 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file Metadata.h +/// \brief Metadata required to decode a Block + +/// The Metadata required to decoded the CTF of particular detector. + +#ifndef ALICEO2_METADATA_H +#define ALICEO2_METADATA_H + +#include +#include + +namespace o2::ctf +{ + +struct Metadata { + enum class OptStore : uint8_t { // describe how the store the data described by this metadata + EENCODE, // entropy encoding applied + ROOTCompression, // original data repacked to array with slot-size = streamSize and saved with root compression + NONE, // original data repacked to array with slot-size = streamSize and saved w/o compression + NODATA, // no data was provided + PACK, // use Bitpacking + EENCODE_OR_PACK // decide at runtime if to encode or pack + }; + uint8_t nStreams = 0; // Amount of concurrent Streams used by the encoder. only used by rANS version >=1. + size_t messageLength = 0; // Message length (multiply with messageWordSize to get size in Bytes). + size_t nLiterals = 0; // Number of samples that were stored as literals. + uint8_t messageWordSize = 0; // size in Bytes of a symbol in the encoded message. + uint8_t coderType = 0; // what type of CTF Coder is used? (32 vs 64 bit coders). + uint8_t streamSize = 0; // number of Bytes emmitted during rANS stream out (ransCompat) or lower renorming bound (ransV1). + uint8_t probabilityBits = 0; // The encoder renormed the distribution of source symbols to sum up to 2^probabilityBits. + OptStore opt = OptStore::EENCODE; // The type of storage operation that was conducted. + int32_t min = 0; // min symbol of the source dataset. + int32_t max = 0; // max symbol of the source dataset. + int32_t literalsPackingOffset = 0; // Offset from 0 used for bit packing of literal (incompressible) symbols. only used by rANS version >=1. + uint8_t literalsPackingWidth = 0; // Amount of bits used to pack literal (incompressible) symbols. only used by rANS version >=1. + int nDictWords = 0; // Amount of words used to store the encoding dictionary. + int nDataWords = 0; // Amount of words used to store the actual data. + int nLiteralWords = 0; // Amount of words used to store literal (incompressible) samples. + + /** + * @brief Uncompressed size of stored data in bytes + * + * @return size_t Uncompressed size in bytes + */ + size_t getUncompressedSize() const { return messageLength * messageWordSize; } + + /** + * @brief Size of the stored, compressed data in multiples of the underlying buffer word size + * + * @return size_t The size in multiples of the underlying buffer word size + * @warning This size is in number of words of the underlying storage buffer. + * Multiply with the size of the storage buffer type to get the correct size in bytes. + */ + size_t getCompressedSize() const { return nDictWords + nDataWords + nLiteralWords; } + void clear() + { + nStreams = 0; + messageLength = 0; + nLiterals = 0; + messageWordSize = 0; + coderType = 0; + streamSize = 0; + probabilityBits = 0; + min = 0; + max = 0; + literalsPackingOffset = 0; + literalsPackingWidth = 0; + nDictWords = 0; + nDataWords = 0; + nLiteralWords = 0; + } + ClassDefNV(Metadata, 3); +}; + +namespace detail +{ + +template +[[nodiscard]] inline constexpr Metadata makeMetadataRansCompat(size_t nStreams, size_t messageLength, + size_t nLiterals, size_t symbolTablePrecision, + source_T min, source_T max, size_t dictWords, + size_t dataWords, size_t literalWords) noexcept +{ + return Metadata{ + static_cast(nStreams), + messageLength, + nLiterals, + static_cast(sizeof(source_T)), + static_cast(sizeof(state_T)), + static_cast(sizeof(stream_T)), + static_cast(symbolTablePrecision), + Metadata::OptStore::EENCODE, + static_cast(min), + static_cast(max), + static_cast(0), + static_cast(sizeof(source_T)), + static_cast(dictWords), + static_cast(dataWords), + static_cast(literalWords)}; +}; + +template +[[nodiscard]] inline constexpr Metadata makeMetadataRansDict(size_t symbolTablePrecision, source_T min, + source_T max, size_t dictWords, ctf::Metadata::OptStore optStore) noexcept +{ + return Metadata{ + static_cast(0), + static_cast(0), + static_cast(0), + static_cast(sizeof(source_T)), + static_cast(0), + static_cast(0), + static_cast(symbolTablePrecision), + optStore, + static_cast(min), + static_cast(max), + static_cast(0), + static_cast(0), + static_cast(dictWords), + static_cast(0), + static_cast(0)}; +}; + +template +[[nodiscard]] inline constexpr Metadata makeMetadataRansV1(size_t nStreams, size_t streamingLowerBound, size_t messageLength, + size_t nLiterals, size_t symbolTablePrecision, + source_T dictMin, source_T dictMax, + source_T literalsOffset, size_t literalsPackingWidth, size_t dictWords, + size_t dataWords, size_t literalWords) noexcept +{ + return Metadata{ + static_cast(nStreams), + messageLength, + nLiterals, + static_cast(sizeof(source_T)), + static_cast(sizeof(state_T)), + static_cast(streamingLowerBound), + static_cast(symbolTablePrecision), + Metadata::OptStore::EENCODE, + static_cast(dictMin), + static_cast(dictMax), + static_cast(literalsOffset), + static_cast(literalsPackingWidth), + static_cast(dictWords), + static_cast(dataWords), + static_cast(literalWords)}; +}; + +template +[[nodiscard]] inline constexpr Metadata makeMetadataPack(size_t messageLength, size_t packingWidth, + source_T packingOffset, size_t dataWords) noexcept +{ + return Metadata{ + static_cast(1), + messageLength, + static_cast(0), + static_cast(sizeof(source_T)), + static_cast(0), + static_cast(0), + static_cast(packingWidth), + Metadata::OptStore::PACK, + static_cast(packingOffset), + static_cast(0), + static_cast(0), + static_cast(0), + static_cast(0), + static_cast(dataWords), + static_cast(0)}; +}; + +template +[[nodiscard]] inline constexpr Metadata makeMetadataStore(size_t messageLength, Metadata::OptStore opStore, size_t dataWords) noexcept +{ + return Metadata{ + static_cast(0), + messageLength, + static_cast(0), + static_cast(sizeof(source_T)), + static_cast(0), + static_cast(sizeof(buffer_T)), + static_cast(0), + opStore, + static_cast(0), + static_cast(0), + static_cast(0), + static_cast(0), + static_cast(0), + static_cast(dataWords), + static_cast(0)}; +}; + +} // namespace detail +} // namespace o2::ctf + +#endif \ No newline at end of file diff --git a/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/SimTraits.h b/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/SimTraits.h index eeb7480f9cf51..37c4b790d181b 100644 --- a/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/SimTraits.h +++ b/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/SimTraits.h @@ -88,13 +88,19 @@ class SimTraits /*FV0*/ VS{ "FV0Hit" }, /*FDD*/ VS{ "FDDHit" }, /*TST*/ VS{ "TSTHit" }, // former ACO - /*CTP*/ VS{ "CTPHit" } + /*CTP*/ VS{ "CTPHit" }, + /*FOC*/ VS{ "FOCHit" } #ifdef ENABLE_UPGRADES , /*IT3*/ VS{ "IT3Hit" }, /*TRK*/ VS{ "TRKHit" }, /*FT3*/ VS{ "FT3Hit" }, - /*FCT*/ VS{ "FCTHit" } + /*FCT*/ VS{ "FCTHit" }, + /*TF3*/ VS{ "TF3Hit" }, + /*RCH*/ VS{ "RCHHit" }, + /*MI3*/ VS{ "MI3Hit" }, + /*ECL*/ VS{ "ECLHit" }, + /*FD */ VS{ "FDHit" } #endif }; // clang-format on @@ -162,7 +168,10 @@ namespace tpc { class HitGroup; } - +namespace focal +{ +class Hit; +} namespace detectors { @@ -226,6 +235,10 @@ template <> struct DetIDToHitTypes { using HitType = o2::tpc::HitGroup; }; +template <> +struct DetIDToHitTypes { + using HitType = o2::focal::Hit; +}; #ifdef ENABLE_UPGRADES template <> struct DetIDToHitTypes { diff --git a/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/internal/ExternalEntropyCoder.h b/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/internal/ExternalEntropyCoder.h new file mode 100644 index 0000000000000..5335514278e26 --- /dev/null +++ b/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/internal/ExternalEntropyCoder.h @@ -0,0 +1,119 @@ +// Copyright 2019-2023 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file CTFEntropyCoder.h +/// \author michael.lettrich@cern.ch +/// \brief Interface for externally provided rANS entropy coders + +#ifndef ALICEO2_EXTERNALENTROPYCODER_H_ +#define ALICEO2_EXTERNALENTROPYCODER_H_ + +#include + +#include "DetectorsCommonDataFormats/internal/Packer.h" + +#include "rANS/encode.h" +#include "rANS/factory.h" +#include "rANS/histogram.h" +#include "rANS/metrics.h" +#include "rANS/serialize.h" + +namespace o2::ctf::internal +{ + +template +class ExternalEntropyCoder +{ + public: + using source_type = source_T; + using encoder_type = typename rans::denseEncoder_type; + + ExternalEntropyCoder(const encoder_type& encoder); + + [[nodiscard]] inline const encoder_type& getEncoder() const noexcept { return *mEncoder; }; + + template + [[nodiscard]] inline size_t computePayloadSizeEstimate(size_t nElements, double_t safetyFactor = 1); + + template + [[nodiscard]] dst_IT encode(src_IT srcBegin, src_IT srcEnd, dst_IT dstBegin, dst_IT dstEnd); + + [[nodiscard]] inline size_t getNIncompressibleSamples() const noexcept { return mIncompressibleBuffer.size(); }; + + [[nodiscard]] inline source_type getIncompressibleSymbolOffset() const noexcept { return mIncompressiblePacker.getOffset(); }; + + [[nodiscard]] inline size_t getIncompressibleSymbolPackingBits() const noexcept { return mIncompressiblePacker.getPackingWidth(); }; + + template + [[nodiscard]] size_t computePackedIncompressibleSize() const noexcept; + + template + [[nodiscard]] dst_T* writeIncompressible(dst_T* dstBegin, dst_T* dstEnd) const; + + private: + const encoder_type* mEncoder{}; + std::vector mIncompressibleBuffer{}; + Packer mIncompressiblePacker{}; +}; + +template +ExternalEntropyCoder::ExternalEntropyCoder(const encoder_type& encoder) : mEncoder{&encoder} +{ + if (!getEncoder().getSymbolTable().hasEscapeSymbol()) { + throw std::runtime_error("External entropy encoder must be able to handle incompressible symbols."); + } +}; + +template +template +[[nodiscard]] inline size_t ExternalEntropyCoder::computePayloadSizeEstimate(size_t nElements, double_t safetyFactor) +{ + constexpr size_t Overhead = 10 * rans::utils::pow2(10); // 10KB overhead safety margin + const double_t RelativeSafetyFactor = 2.0 * safetyFactor; + const size_t messageSizeB = nElements * sizeof(source_type); + return rans::utils::nBytesTo(std::ceil(RelativeSafetyFactor * messageSizeB) + Overhead); +} + +template +template +[[nodiscard]] dst_IT ExternalEntropyCoder::encode(src_IT srcBegin, src_IT srcEnd, dst_IT dstBegin, dst_IT dstEnd) +{ + const size_t incompressibleSymbolFrequency = [&]() { + const auto& symbolTable = mEncoder->getSymbolTable(); + const double_t incompressibleSymbolProbability = static_cast(symbolTable.getEscapeSymbol().getFrequency()) / rans::utils::pow2(symbolTable.getPrecision()); + return std::ceil(std::distance(srcBegin, srcEnd) * incompressibleSymbolProbability); + }(); + + mIncompressibleBuffer.reserve(incompressibleSymbolFrequency); + auto [encodedMessageEnd, literalsEnd] = mEncoder->process(srcBegin, srcEnd, dstBegin, std::back_inserter(mIncompressibleBuffer)); + rans::utils::checkBounds(encodedMessageEnd, dstEnd); + mIncompressiblePacker = Packer{mIncompressibleBuffer.data(), mIncompressibleBuffer.data() + mIncompressibleBuffer.size()}; + + return encodedMessageEnd; +}; + +template +template +[[nodiscard]] inline size_t ExternalEntropyCoder::computePackedIncompressibleSize() const noexcept +{ + return mIncompressiblePacker.template getPackingBufferSize(mIncompressibleBuffer.size()); +}; + +template +template +[[nodiscard]] inline dst_T* ExternalEntropyCoder::writeIncompressible(dst_T* dstBegin, dst_T* dstEnd) const +{ + return mIncompressiblePacker.pack(mIncompressibleBuffer.data(), mIncompressibleBuffer.size(), dstBegin, dstEnd); +}; + +} // namespace o2::ctf::internal + +#endif /* ALICEO2_EXTERNALENTROPYCODER_H_ */ \ No newline at end of file diff --git a/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/internal/InplaceEntropyCoder.h b/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/internal/InplaceEntropyCoder.h new file mode 100644 index 0000000000000..8601469b34142 --- /dev/null +++ b/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/internal/InplaceEntropyCoder.h @@ -0,0 +1,312 @@ +// Copyright 2019-2023 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file CTFEntropyCoder.h +/// \author michael.lettrich@cern.ch +/// \brief ANS Entropy Coding and packing specialization for CTF Coders + +#ifndef ALICEO2_INPLACEENTROPYCODER_H_ +#define ALICEO2_INPLACEENTROPYCODER_H_ + +#include +#include +#include + +#include "DetectorsCommonDataFormats/internal/Packer.h" + +#include "rANS/encode.h" +#include "rANS/factory.h" +#include "rANS/histogram.h" +#include "rANS/metrics.h" +#include "rANS/serialize.h" + +namespace o2::ctf::internal +{ + +template +class InplaceEntropyCoder +{ + using dense_histogram_type = rans::DenseHistogram; + using adaptive_histogram_type = rans::AdaptiveHistogram; + using sparse_histogram_type = rans::SparseHistogram; + + using dense_encoder_type = rans::denseEncoder_type; + using adaptive_encoder_type = rans::adaptiveEncoder_type; + using sparse_encoder_type = rans::sparseEncoder_type; + + using dict_buffer_type = std::vector; + + public: + using source_type = source_T; + using metrics_type = rans::Metrics; + using packer_type = Packer; + using histogram_type = std::variant; + using encoder_type = std::variant; + using incompressible_buffer_type = std::vector; + + InplaceEntropyCoder() = default; + + template + InplaceEntropyCoder(source_IT srcBegin, source_IT srcEnd); + + template + InplaceEntropyCoder(source_IT srcBegin, source_IT srcEnd, source_type min, source_type max); + + void makeEncoder(); + + // getters + + [[nodiscard]] inline const metrics_type& getMetrics() const noexcept { return mMetrics; }; + + [[nodiscard]] inline size_t getNIncompressibleSamples() const noexcept { return mIncompressibleBuffer.size(); }; + + [[nodiscard]] size_t getNStreams() const; + + [[nodiscard]] size_t getSymbolTablePrecision() const; + + template + [[nodiscard]] size_t getPackedIncompressibleSize() const noexcept; + + // operations + template + [[nodiscard]] dst_IT encode(src_IT srcBegin, src_IT srcEnd, dst_IT dstBegin, dst_IT dstEnd); + + template + [[nodiscard]] dst_IT writeDictionary(dst_IT dstBegin, dst_IT dstEnd); + + template + [[nodiscard]] dst_T* writeIncompressible(dst_T* dstBegin, dst_T* dstEnd); + + private: + template ::value_type) < 4), bool> = true> + void init(source_IT srcBegin, source_IT srcEnd, source_type min, source_type max); + + template ::value_type) == 4), bool> = true> + void init(source_IT srcBegin, source_IT srcEnd, source_type min, source_type max); + + template ::value_type) < 4), bool> = true> + void init(source_IT srcBegin, source_IT srcEnd); + + template ::value_type) == 4), bool> = true> + void init(source_IT srcBegin, source_IT srcEnd); + + template + void serializeDictionary(const container_T&); + + std::optional mHistogram{}; + metrics_type mMetrics{}; + std::optional mEncoder{}; + incompressible_buffer_type mIncompressibleBuffer{}; + dict_buffer_type mDictBuffer{}; + packer_type mIncompressiblePacker{}; +}; + +template +template +InplaceEntropyCoder::InplaceEntropyCoder(src_IT srcBegin, src_IT srcEnd) +{ + static_assert(std::is_same_v::value_type>); + + const size_t nSamples = std::distance(srcBegin, srcEnd); + if constexpr (std::is_pointer_v) { + if (sizeof(source_type) > 2 && nSamples > 0) { + const auto [min, max] = rans::internal::minmax(gsl::span(srcBegin, srcEnd)); + init(srcBegin, srcEnd, min, max); + } else { + init(srcBegin, srcEnd); + } + } else { + init(srcBegin, srcEnd); + } + + mIncompressiblePacker = Packer(mMetrics); +}; + +template +template +InplaceEntropyCoder::InplaceEntropyCoder(source_IT srcBegin, source_IT srcEnd, source_type min, source_type max) +{ + static_assert(std::is_same_v::value_type>); + init(srcBegin, srcEnd, min, max); + mIncompressiblePacker = Packer(mMetrics); +}; + +template +[[nodiscard]] inline size_t InplaceEntropyCoder::getNStreams() const +{ + size_t nStreams{}; + std::visit([&, this](auto&& encoder) { nStreams = encoder.getNStreams(); }, *mEncoder); + return nStreams; +} + +template +[[nodiscard]] inline size_t InplaceEntropyCoder::getSymbolTablePrecision() const +{ + size_t precision{}; + std::visit([&, this](auto&& encoder) { precision = encoder.getSymbolTable().getPrecision(); }, *mEncoder); + return precision; +} + +template +void InplaceEntropyCoder::makeEncoder() +{ + std::visit([this](auto&& histogram) { + auto renormed = rans::renorm(std::move(histogram), mMetrics); + + if (std::holds_alternative(*mHistogram)) { + serializeDictionary(renormed); + } + + const size_t rangeBits = rans::utils::getRangeBits(*mMetrics.getCoderProperties().min, *mMetrics.getCoderProperties().max); + const size_t nUsedAlphabetSymbols = mMetrics.getDatasetProperties().nUsedAlphabetSymbols; + + if (rangeBits <= 18) { + // dense symbol tables if they fit into cache, or source data covers the range of the alphabet well + mEncoder = encoder_type{std::in_place_type, renormed}; + } else if (nUsedAlphabetSymbols < rans::utils::pow2(14)) { + // sparse symbol table makes sense if it fits into L3 Cache + mEncoder = encoder_type{std::in_place_type, renormed}; + } else { + // adaptive symbol table otherwise + mEncoder = encoder_type{std::in_place_type, renormed}; + } + }, + *mHistogram); +}; + +template +template +[[nodiscard]] dst_IT InplaceEntropyCoder::encode(src_IT srcBegin, src_IT srcEnd, dst_IT dstBegin, dst_IT dstEnd) +{ + static_assert(std::is_same_v::value_type>); + + dst_IT messageEnd = dstBegin; + + std::visit([&, this](auto&& encoder) { + if (encoder.getSymbolTable().hasEscapeSymbol()) { + mIncompressibleBuffer.reserve(*mMetrics.getCoderProperties().nIncompressibleSamples); + auto [encodedMessageEnd, literalsEnd] = encoder.process(srcBegin, srcEnd, dstBegin, std::back_inserter(mIncompressibleBuffer)); + messageEnd = encodedMessageEnd; + } else { + messageEnd = encoder.process(srcBegin, srcEnd, dstBegin); + } + rans::utils::checkBounds(messageEnd, dstEnd); + }, + *mEncoder); + + return messageEnd; +}; + +template +template +[[nodiscard]] inline dst_IT InplaceEntropyCoder::writeDictionary(dst_IT dstBegin, dst_IT dstEnd) +{ + static_assert(std::is_pointer_v); + + using dst_type = std::remove_pointer_t; + + dst_IT ret{}; + if (mDictBuffer.empty()) { + std::visit([&, this](auto&& encoder) { ret = rans::compressRenormedDictionary(encoder.getSymbolTable(), dstBegin); }, *mEncoder); + } else { + // copy + std::memcpy(dstBegin, mDictBuffer.data(), mDictBuffer.size()); + + // determine location of end + auto end = reinterpret_cast(dstBegin) + mDictBuffer.size(); + // realign pointer + constexpr size_t alignment = std::alignment_of_v; + end += (alignment - reinterpret_cast(end) % alignment) % alignment; + // and convert it back to ret + ret = reinterpret_cast(end); + } + + rans::utils::checkBounds(ret, dstEnd); + return ret; +}; + +template +template +inline dst_T* InplaceEntropyCoder::writeIncompressible(dst_T* dstBegin, dst_T* dstEnd) +{ + return mIncompressiblePacker.pack(mIncompressibleBuffer.data(), mIncompressibleBuffer.size(), dstBegin, dstEnd); +}; + +template +template +[[nodiscard]] inline size_t InplaceEntropyCoder::getPackedIncompressibleSize() const noexcept +{ + return mIncompressiblePacker.template getPackingBufferSize(getNIncompressibleSamples()); +} + +template +template ::value_type) < 4), bool>> +void InplaceEntropyCoder::init(source_IT srcBegin, source_IT srcEnd, source_type min, source_type max) +{ + mHistogram.emplace(histogram_type{rans::makeDenseHistogram::fromSamples(srcBegin, srcEnd)}); + mMetrics = metrics_type{std::get(*mHistogram), min, max}; +}; + +template +template ::value_type) == 4), bool>> +void InplaceEntropyCoder::init(source_IT srcBegin, source_IT srcEnd, source_type min, source_type max) +{ + const size_t nSamples = std::distance(srcBegin, srcEnd); + const size_t rangeBits = rans::utils::getRangeBits(min, max); + + if ((rangeBits <= 18) || ((nSamples / rans::utils::pow2(rangeBits)) >= 0.80)) { + // either the range of source symbols is distrubuted such that it fits into L3 Cache + // Or it is possible for the data to cover a very significant fraction of the total [min,max] range + mHistogram = histogram_type{std::in_place_type, rans::makeDenseHistogram::fromSamples(srcBegin, srcEnd, min, max)}; + mMetrics = metrics_type{std::get(*mHistogram), min, max}; + } else if (nSamples / rans::utils::pow2(rangeBits) <= 0.3) { + // or the range of source symbols is spread very thinly accross a large range + mHistogram = histogram_type{std::in_place_type, rans::makeSparseHistogram::fromSamples(srcBegin, srcEnd)}; + mMetrics = metrics_type{std::get(*mHistogram), min, max}; + } else { + // no strong evidence of either extreme case + mHistogram = histogram_type{std::in_place_type, rans::makeAdaptiveHistogram::fromSamples(srcBegin, srcEnd)}; + mMetrics = metrics_type{std::get(*mHistogram), min, max}; + } +}; + +template +template ::value_type) < 4), bool>> +void InplaceEntropyCoder::init(source_IT srcBegin, source_IT srcEnd) +{ + mHistogram = histogram_type{std::in_place_type, rans::makeDenseHistogram::fromSamples(srcBegin, srcEnd)}; + mMetrics = metrics_type{std::get(*mHistogram)}; +}; + +template +template ::value_type) == 4), bool>> +void InplaceEntropyCoder::init(source_IT srcBegin, source_IT srcEnd) +{ + mHistogram = histogram_type{std::in_place_type, rans::makeSparseHistogram::fromSamples(srcBegin, srcEnd)}; + mMetrics = metrics_type{std::get(*mHistogram)}; +}; + +template +template +void InplaceEntropyCoder::serializeDictionary(const container_T& renormedHistogram) +{ + + mDictBuffer.resize(mMetrics.getSizeEstimate().getCompressedDictionarySize(), 0); + auto end = rans::compressRenormedDictionary(renormedHistogram, mDictBuffer.data()); + rans::utils::checkBounds(end, mDictBuffer.data() + mDictBuffer.size()); + mDictBuffer.resize(std::distance(mDictBuffer.data(), end)); + + assert(mDictBuffer.size() > 0); +}; + +} // namespace o2::ctf::internal + +#endif /* ALICEO2_INPLACEENTROPYCODER_H_ */ \ No newline at end of file diff --git a/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/internal/Packer.h b/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/internal/Packer.h new file mode 100644 index 0000000000000..fa11e9f6dac6d --- /dev/null +++ b/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/internal/Packer.h @@ -0,0 +1,124 @@ +// Copyright 2019-2023 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file Packer.h +/// \author michael.lettrich@cern.ch +/// \brief Interfaces for BitPacking using librans + +#ifndef ALICEO2_PACKER_H_ +#define ALICEO2_PACKER_H_ + +#ifndef __CLING__ +#include "rANS/pack.h" +#include "rANS/metrics.h" +#endif + +namespace o2::ctf::internal +{ + +template +class Packer +{ + public: + using source_type = source_T; + + Packer() = default; +#ifndef __CLING__ + explicit Packer(rans::Metrics& metrics) : mOffset{metrics.getDatasetProperties().min}, + mPackingWidth{metrics.getDatasetProperties().alphabetRangeBits} {}; +#endif + + template + Packer(source_IT srcBegin, source_IT srcEnd); + + [[nodiscard]] inline source_type getOffset() const noexcept { return mOffset; }; + + [[nodiscard]] inline size_t getPackingWidth() const noexcept { return mPackingWidth; }; + + template + [[nodiscard]] size_t getPackingBufferSize(size_t messageLength) const noexcept; + + template + [[nodiscard]] dst_T* pack(source_IT srcBegin, source_IT srcEnd, dst_T* dstBegin, dst_T* dstEnd) const; + + template + [[nodiscard]] dst_T* pack(const source_T* __restrict srcBegin, size_t extent, dst_T* dstBegin, dst_T* dstEnd) const; + + private: + source_type mOffset{}; + size_t mPackingWidth{}; +}; + +template +template +Packer::Packer(source_IT srcBegin, source_IT srcEnd) +{ +#ifndef __CLING__ + static_assert(rans::utils::isCompatibleIter_v); + if (srcBegin != srcEnd) { + + const auto [min, max] = [&]() { + if constexpr (std::is_pointer_v) { + return rans::utils::minmax(gsl::span(srcBegin, srcEnd)); + } else { + const auto [minIter, maxIter] = std::minmax_element(srcBegin, srcEnd); + return std::make_pair(*minIter, *maxIter); + } + }(); + + mOffset = min; + mPackingWidth = rans::utils::getRangeBits(min, max); + } +#endif +}; + +template +template +[[nodiscard]] inline size_t Packer::getPackingBufferSize(size_t messageLength) const noexcept +{ +#ifndef __CLING__ + return rans::computePackingBufferSize(messageLength, mPackingWidth); +#else + return 0; +#endif +}; + +template +template +[[nodiscard]] inline dst_T* Packer::pack(const source_T* __restrict srcBegin, size_t extent, dst_T* dstBegin, dst_T* dstEnd) const +{ + return pack(srcBegin, srcBegin + extent, dstBegin, dstEnd); +} + +template +template +[[nodiscard]] dst_T* Packer::pack(source_IT srcBegin, source_IT srcEnd, dst_T* dstBegin, dst_T* dstEnd) const +{ + static_assert(std::is_same_v::value_type>); + size_t extent = std::distance(srcBegin, srcEnd); + + if (extent == 0) { + return dstBegin; + } +#ifndef __CLING__ + rans::BitPtr packEnd = rans::pack(srcBegin, extent, dstBegin, mPackingWidth, mOffset); + auto* end = packEnd.toPtr(); + ++end; // one past end iterator; + rans::utils::checkBounds(end, dstEnd); + return end; +#else + return nullptr; +#endif +}; + +} // namespace o2::ctf::internal + +#endif /* ALICEO2_PACKER_H_ */ \ No newline at end of file diff --git a/DataFormats/Detectors/Common/src/AlignParam.cxx b/DataFormats/Detectors/Common/src/AlignParam.cxx index 5cf804fc085fe..2061726a29c66 100644 --- a/DataFormats/Detectors/Common/src/AlignParam.cxx +++ b/DataFormats/Detectors/Common/src/AlignParam.cxx @@ -12,12 +12,12 @@ /// \file AlignParam.cxx /// \brief Implementation of the base alignment parameters class -#include #include #include #include #include +#include "Framework/Logger.h" #include "DetectorsCommonDataFormats/AlignParam.h" using namespace o2::detectors; @@ -26,8 +26,9 @@ using namespace o2::detectors; AlignParam::AlignParam(const char* symname, int algID, // volume symbolic name and its alignable ID double x, double y, double z, // delta translation double psi, double theta, double phi, // delta rotation - bool global) // global (preferable) or local delta definition - : mSymName(symname), mAlignableID(algID) + bool global, // global (preferable) or local delta definition + bool convertLocalToGlobal) // if local is provided, convert it to global + : mSymName(symname), mIsGlobal(global || convertLocalToGlobal), mAlignableID(algID) { /// standard constructor with 3 translation + 3 rotation parameters /// If the user explicitly sets the global variable to false then the @@ -35,20 +36,33 @@ AlignParam::AlignParam(const char* symname, int algID, // volume symbolic /// This requires to have a gGeoMenager active instance, otherwise the /// constructor will fail (no object created) - if (global) { - setGlobalParams(x, y, z, psi, theta, phi); - } else { + setParams(x, y, z, psi, theta, phi); + if (!global && convertLocalToGlobal) { setLocalParams(x, y, z, psi, theta, phi); } } +//___________________________________________________ +AlignParam::AlignParam(const char* symname, int algID, TGeoMatrix& m, bool global, bool convertLocalToGlobal) + : mSymName(symname), mIsGlobal(global || convertLocalToGlobal), mAlignableID(algID) +{ + setTranslation(m); + if (!setRotation(m)) { + const double* rot = m.GetRotationMatrix(); + throw std::runtime_error(fmt::format("Failed to extract roll-pitch-yall angles from [[{},{},{}], [{},{},{}], [{},{},{}] for {}", rot[0], rot[1], rot[2], rot[3], rot[4], rot[5], rot[6], rot[7], rot[8], symname)); + } + if (!global && convertLocalToGlobal && !setLocalParams(mX, mY, mZ, mPsi, mTheta, mPhi)) { + throw std::runtime_error(fmt::format("Alignment creation for {} failed: geomManager is absent", symname)); + } +} + //___________________________________________________ TGeoHMatrix AlignParam::createMatrix() const { /// create a copy of alignment global delta matrix TGeoHMatrix mat; setMatrixTranslation(mX, mY, mZ, mat); - setMatrixRotation(mPhi, mTheta, mPsi, mat); + setMatrixRotation(mPsi, mTheta, mPhi, mat); return mat; } @@ -209,6 +223,10 @@ bool AlignParam::createLocalMatrix(TGeoHMatrix& m) const // In case that the TGeo was not initialized or not closed, // returns false and the object parameters are not set. // + m = createMatrix(); + if (!mIsGlobal) { + return true; + } if (!gGeoManager || !gGeoManager->IsClosed()) { LOG(error) << "Can't get the local alignment object parameters! gGeoManager doesn't exist or it is still open!"; return false; @@ -233,7 +251,6 @@ bool AlignParam::createLocalMatrix(TGeoHMatrix& m) const LOG(error) << "Volume name or path " << symname << " is not valid!"; return false; } - m = createMatrix(); TGeoHMatrix gprime, gprimeinv; gprime = *node->GetMatrix(); gprimeinv = gprime.Inverse(); @@ -244,7 +261,7 @@ bool AlignParam::createLocalMatrix(TGeoHMatrix& m) const } //_____________________________________________________________________________ -bool AlignParam::applyToGeometry() const +bool AlignParam::applyToGeometry(int printLevel) const { /// Apply the current alignment object to the TGeo geometry /// This method returns FALSE if the symname of the object was not @@ -288,18 +305,26 @@ bool AlignParam::applyToGeometry() const } // double threshold = 0.001; + TGeoHMatrix* align = new TGeoHMatrix(createMatrix()); + if (mIsGlobal) { + align->Multiply(node->GetMatrix()); + align->MultiplyLeft(node->GetMatrix(node->GetLevel() - 1)->Inverse()); + } else { + align->MultiplyLeft(node->GetOriginalMatrix()); + } - TGeoHMatrix gprime = *node->GetMatrix(); - TGeoHMatrix align = createMatrix(); - gprime.MultiplyLeft(&align); - TGeoHMatrix* ginv = new TGeoHMatrix; // TGeoPhysicalNode takes and manages raw pointer, need naked new! - TGeoHMatrix* g = node->GetMatrix(node->GetLevel() - 1); - *ginv = g->Inverse(); - *ginv *= gprime; - - LOG(debug) << "Aligning volume " << symname; - - node->Align(ginv); + node->Align(align); + + if (getLevel() <= printLevel) { + LOGP(info, "{:*^100}", symname); + LOGP(info, " - Alignment parameter:"); + print(); + LOGP(info, " - Alignment matrix:"); + align->Print(); + LOGP(info, " - Node:"); + node->Print(); + LOGP(info, "{:~^100}", symname); + } return true; } @@ -333,8 +358,8 @@ int AlignParam::getLevel() const void AlignParam::print() const { // print parameters - printf("%s : %6d | X: %+e Y: %+e Z: %+e | pitch: %+e roll: %+e yaw: %e\n", getSymName().c_str(), getAlignableID(), getX(), - getY(), getZ(), getPsi(), getTheta(), getPhi()); + printf("%s (Lvl:%2d): %6d | %s | tra: X: %+e Y: %+e Z: %+e | pitch: %+e roll: %+e yaw: %e\n", getSymName().c_str(), getLevel(), getAlignableID(), (mIsGlobal) ? "G" : "L", + getX(), getY(), getZ(), getPsi(), getTheta(), getPhi()); } //_____________________________________________________________________________ @@ -345,6 +370,14 @@ void AlignParam::setGlobalParams(double x, double y, double z, double psi, doubl setRotation(psi, theta, phi); } +//_____________________________________________________________________________ +void AlignParam::setParams(double x, double y, double z, double psi, double theta, double phi) +{ + /// set parameters of global delta + setTranslation(x, y, z); + setRotation(psi, theta, phi); +} + //_____________________________________________________________________________ void AlignParam::setRotation(double psi, double theta, double phi) { @@ -434,3 +467,33 @@ bool AlignParam::setLocalRotation(const TGeoMatrix& m) rotm.SetRotation(m.GetRotationMatrix()); return setLocalParams(rotm); } + +//_____________________________________________________________________________ +int AlignParam::rectify(double zero) +{ + int nonZero = 6; + if (std::abs(mX) < zero) { + mX = 0.; + } + if (std::abs(mY) < zero) { + mY = 0.; + nonZero--; + } + if (std::abs(mZ) < zero) { + mZ = 0.; + nonZero--; + } + if (std::abs(mPsi) < zero) { + mPsi = 0.; + nonZero--; + } + if (std::abs(mTheta) < zero) { + mTheta = 0.; + nonZero--; + } + if (std::abs(mPhi) < zero) { + mPhi = 0.; + nonZero--; + } + return nonZero; +} diff --git a/DataFormats/Detectors/Common/src/CTFHeader.cxx b/DataFormats/Detectors/Common/src/CTFHeader.cxx index e704b46c1e240..4332f8aecd3d6 100644 --- a/DataFormats/Detectors/Common/src/CTFHeader.cxx +++ b/DataFormats/Detectors/Common/src/CTFHeader.cxx @@ -18,7 +18,7 @@ using DetID = o2::detectors::DetID; /// describe itsel as a string std::string CTFHeader::describe() const { - return fmt::format("Run:{:07d} TF:{} Orbit:{:08d} CteationTime:{} Detectors: {}", run, tfCounter, firstTForbit, creationTime, DetID::getNames(detectors)); + return fmt::format("Run:{:07d} TF:{} Orbit:{:08d} CreationTime:{} Detectors: {}", run, tfCounter, firstTForbit, creationTime, DetID::getNames(detectors)); } void CTFHeader::print() const diff --git a/DataFormats/Detectors/Common/src/DetID.cxx b/DataFormats/Detectors/Common/src/DetID.cxx index a98e34815d894..6b5fe0ed5c762 100644 --- a/DataFormats/Detectors/Common/src/DetID.cxx +++ b/DataFormats/Detectors/Common/src/DetID.cxx @@ -17,7 +17,7 @@ #include "CommonUtils/StringUtils.h" #include #include -#include "FairLogger.h" +#include using namespace o2::detectors; @@ -27,7 +27,7 @@ constexpr const char* DetID::sDetNames[DetID::nDetectors + 1]; // redundant declarations constexpr DetID::ID DetID::ITS, DetID::TPC, DetID::TRD, DetID::TOF, DetID::PHS, DetID::CPV, DetID::EMC, - DetID::HMP, DetID::MFT, DetID::MCH, DetID::MID, DetID::ZDC, DetID::FT0, DetID::FV0, DetID::FDD, DetID::TST, DetID::CTP, DetID::First, DetID::Last; + DetID::HMP, DetID::MFT, DetID::MCH, DetID::MID, DetID::ZDC, DetID::FT0, DetID::FV0, DetID::FDD, DetID::TST, DetID::CTP, DetID::FOC, DetID::First, DetID::Last; #ifdef ENABLE_UPGRADES constexpr DetID::ID DetID::IT3; diff --git a/DataFormats/Detectors/Common/src/FileMetaData.cxx b/DataFormats/Detectors/Common/src/FileMetaData.cxx index 7a025648a32d3..f9230fd8ac7a3 100644 --- a/DataFormats/Detectors/Common/src/FileMetaData.cxx +++ b/DataFormats/Detectors/Common/src/FileMetaData.cxx @@ -13,6 +13,7 @@ #include "DetectorsCommonDataFormats/FileMetaData.h" #include "Framework/DataTakingContext.h" +#include "CommonUtils/StringUtils.h" #include #include #include @@ -20,14 +21,22 @@ using namespace o2::dataformats; -bool FileMetaData::fillFileData(const std::string& fname) +bool FileMetaData::fillFileData(const std::string& fname, bool fillmd5, const std::string& tmpEnding) { + // fill metadata for fname, accounting that the fname might be temporary one while the real one is fnameFinal try { lurl = std::filesystem::canonical(fname).string(); size = std::filesystem::file_size(lurl); - std::unique_ptr md5ptr{TMD5::FileChecksum(fname.c_str())}; - md5 = md5ptr->AsString(); + if (fillmd5) { + std::unique_ptr md5ptr{TMD5::FileChecksum(fname.c_str())}; + if (md5ptr) { + md5 = md5ptr->AsString(); + } + } ctime = std::chrono::time_point_cast(std::chrono::system_clock::now()).time_since_epoch().count(); + if (tmpEnding.size()) { + o2::utils::Str::rtrim(lurl, tmpEnding); + } } catch (std::exception const& e) { LOG(error) << "Failed to fill metadata for file " << fname << ", reason: " << e.what(); return false; diff --git a/DataFormats/Detectors/Common/test/testCTFEntropyCoder.cxx b/DataFormats/Detectors/Common/test/testCTFEntropyCoder.cxx new file mode 100644 index 0000000000000..6b688fcebc7f7 --- /dev/null +++ b/DataFormats/Detectors/Common/test/testCTFEntropyCoder.cxx @@ -0,0 +1,350 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// @file testCTFEntropyCoder +/// @author Michael Lettrich +/// @brief Test entropy coding using rANS algorithm + +#define BOOST_TEST_MODULE Test CTFEntropyCoder class +#define BOOST_TEST_MAIN +#define BOOST_TEST_DYN_LINK + +#undef NDEBUG +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "DetectorsCommonDataFormats/internal/Packer.h" +#include "DetectorsCommonDataFormats/internal/ExternalEntropyCoder.h" +#include "DetectorsCommonDataFormats/internal/InplaceEntropyCoder.h" +#include "rANS/histogram.h" +#include "rANS/metrics.h" +#include "rANS/factory.h" +#include "rANS/iterator.h" + +using namespace o2; + +using buffer_type = uint32_t; +using source_types = boost::mp11::mp_list; + +template +class SourceMessage +{ + public: + SourceMessage(size_t messageSize, source_T max = std::numeric_limits::max(), source_T min = std::numeric_limits::min()) : mMin{min}, mMax{max} + { + if (mSourceMessage.empty()) { + std::mt19937 mt(0); // same seed we want always the same distrubution of random numbers; + assert(max >= min); + const size_t draws = (max - min) + 1; + const double probability = 0.5; + std::binomial_distribution dist(draws, probability); + mSourceMessage.resize(messageSize); + std::generate(mSourceMessage.begin(), mSourceMessage.end(), [&dist, &mt, min]() -> source_T { return static_cast(dist(mt)) + min; }); + } + } + + inline constexpr source_T getMin() const noexcept { return mMin; }; + inline constexpr source_T getMax() const noexcept { return mMax; }; + inline constexpr auto& get() const noexcept { return mSourceMessage; }; + + private: + source_T mMin{}; + source_T mMax{}; + std::vector mSourceMessage{}; +}; + +class SourceMessageProxy +{ + public: + SourceMessageProxy() = default; + + template + const auto& getMessage() const noexcept + { + if constexpr (std::is_same_v) { + return sourceMessage8u.get(); + } else if constexpr (std::is_same_v) { + return sourceMessage8.get(); + } else if constexpr (std::is_same_v) { + return sourceMessage16u.get(); + } else if constexpr (std::is_same_v) { + return sourceMessage16.get(); + } else if constexpr (std::is_same_v) { + return sourceMessage32u.get(); + } else if constexpr (std::is_same_v) { + return sourceMessage32.get(); + } else { + throw std::runtime_error{"unsupported source type"}; + } + }; + + private: + inline static constexpr size_t MessageSize = rans::utils::pow2(10); + SourceMessage sourceMessage8u{MessageSize}; + SourceMessage sourceMessage8{MessageSize}; + SourceMessage sourceMessage16u{MessageSize}; + SourceMessage sourceMessage16{MessageSize}; + SourceMessage sourceMessage32u{MessageSize, rans::utils::pow2(27)}; + SourceMessage sourceMessage32{MessageSize, rans::utils::pow2(26), -static_cast(rans::utils::pow2(26))}; +}; + +inline const SourceMessageProxy MessageProxy{}; + +template +void encodeInplace(source_IT begin, source_IT end) +{ + using source_type = typename std::iterator_traits::value_type; + + ctf::internal::InplaceEntropyCoder entropyCoder{begin, end}; + // BOOST_CHECK_THROW(entropyCoder.getEncoder(), std::runtime_error); + entropyCoder.makeEncoder(); + + const rans::Metrics& metrics = entropyCoder.getMetrics(); + const rans::SizeEstimate sizeEstimate = metrics.getSizeEstimate(); + + LOGP(info, "dataset[{},{}], coder[{},{}]", metrics.getDatasetProperties().min, metrics.getDatasetProperties().max, *metrics.getCoderProperties().min, *metrics.getCoderProperties().max); + + std::vector encodeBuffer(sizeEstimate.getCompressedDatasetSize(), 0); + std::vector literalSymbolsBuffer(sizeEstimate.getIncompressibleSize(), 0); + std::vector dictBuffer(sizeEstimate.getCompressedDictionarySize(), 0); + + auto encoderEnd = entropyCoder.encode(begin, end, encodeBuffer.data(), encodeBuffer.data() + encodeBuffer.size()); + [[maybe_unused]] auto literalsEnd = entropyCoder.writeIncompressible(literalSymbolsBuffer.data(), literalSymbolsBuffer.data() + literalSymbolsBuffer.size()); + auto dictEnd = entropyCoder.writeDictionary(dictBuffer.data(), dictBuffer.data() + dictBuffer.size()); + // decode + const auto& coderProperties = metrics.getCoderProperties(); + auto decoder = rans::makeDecoder<>::fromRenormed(rans::readRenormedDictionary(dictBuffer.data(), dictEnd, + *coderProperties.min, *coderProperties.max, + *coderProperties.renormingPrecisionBits)); + std::vector literals(entropyCoder.getNIncompressibleSamples()); + + const auto& datasetPropterties = metrics.getDatasetProperties(); + rans::unpack(literalSymbolsBuffer.data(), literals.size(), literals.data(), + datasetPropterties.alphabetRangeBits, datasetPropterties.min); + + size_t messageLength = std::distance(begin, end); + std::vector sourceBuffer(messageLength, 0); + + decoder.process(encoderEnd, sourceBuffer.data(), messageLength, entropyCoder.getNStreams(), literals.end()); + + BOOST_CHECK_EQUAL_COLLECTIONS(sourceBuffer.begin(), sourceBuffer.end(), begin, end); +}; + +BOOST_AUTO_TEST_CASE_TEMPLATE(testInplaceEncoderEmpty, source_T, source_types) +{ + std::vector testMessage{}; + encodeInplace(testMessage.data(), testMessage.data() + testMessage.size()); +}; + +BOOST_AUTO_TEST_CASE_TEMPLATE(testInplaceEncoderPTR, source_T, source_types) +{ + const auto& testMessage = MessageProxy.getMessage(); + encodeInplace(testMessage.data(), testMessage.data() + testMessage.size()); +}; + +BOOST_AUTO_TEST_CASE_TEMPLATE(testInplaceEncoderIter, source_T, source_types) +{ + const auto& testMessage = MessageProxy.getMessage(); + encodeInplace(testMessage.begin(), testMessage.end()); +}; + +template +class ShiftFunctor + +{ + public: + template + inline value_T operator()(iterA_T iterA, iterB_T iterB) const + { + return *iterB + (static_cast(*iterA) << shift); + }; + + template + inline void operator()(iterA_T iterA, iterB_T iterB, value_T value) const + { + *iterA = value >> shift; + *iterB = value & ((0x1 << shift) - 0x1); + }; +}; + +template +auto makeInputIterators(iterA_T iterA, iterB_T iterB, size_t nElements, F functor) +{ + using namespace o2::rans::utils; + + return std::make_tuple(rans::CombinedInputIterator{iterA, iterB, functor}, + rans::CombinedInputIterator{advanceIter(iterA, nElements), advanceIter(iterB, nElements), functor}); +}; + +BOOST_AUTO_TEST_CASE(testInplaceEncoderCombinedIterator) +{ + + const auto& testMessage1 = MessageProxy.getMessage(); + const auto& testMessage2 = MessageProxy.getMessage(); + + auto [begin, end] = makeInputIterators(testMessage1.data(), testMessage2.data(), testMessage1.size(), ShiftFunctor()>{}); + + encodeInplace(begin, end); +}; + +class ExternalEncoderDecoderProxy +{ + public: + ExternalEncoderDecoderProxy() + { + SourceMessageProxy proxy{}; + + auto renormed8u = rans::renorm(rans::makeDenseHistogram::fromSamples(proxy.getMessage().begin(), proxy.getMessage().end()), rans::RenormingPolicy::ForceIncompressible); + auto renormed8 = rans::renorm(rans::makeDenseHistogram::fromSamples(proxy.getMessage().begin(), proxy.getMessage().end()), rans::RenormingPolicy::ForceIncompressible); + auto renormed16u = rans::renorm(rans::makeDenseHistogram::fromSamples(proxy.getMessage().begin(), proxy.getMessage().end()), rans::RenormingPolicy::ForceIncompressible); + auto renormed16 = rans::renorm(rans::makeDenseHistogram::fromSamples(proxy.getMessage().begin(), proxy.getMessage().end()), rans::RenormingPolicy::ForceIncompressible); + auto renormed32u = rans::renorm(rans::makeDenseHistogram::fromSamples(proxy.getMessage().begin(), proxy.getMessage().end()), rans::RenormingPolicy::ForceIncompressible); + auto renormed32 = rans::renorm(rans::makeDenseHistogram::fromSamples(proxy.getMessage().begin(), proxy.getMessage().end()), rans::RenormingPolicy::ForceIncompressible); + + encoder8u = rans::makeDenseEncoder<>::fromRenormed(renormed8u); + encoder8 = rans::makeDenseEncoder<>::fromRenormed(renormed8); + encoder16u = rans::makeDenseEncoder<>::fromRenormed(renormed16u); + encoder16 = rans::makeDenseEncoder<>::fromRenormed(renormed16); + encoder32u = rans::makeDenseEncoder<>::fromRenormed(renormed32u); + encoder32 = rans::makeDenseEncoder<>::fromRenormed(renormed32); + + decoder8u = rans::makeDecoder<>::fromRenormed(renormed8u); + decoder8 = rans::makeDecoder<>::fromRenormed(renormed8); + decoder16u = rans::makeDecoder<>::fromRenormed(renormed16u); + decoder16 = rans::makeDecoder<>::fromRenormed(renormed16); + decoder32u = rans::makeDecoder<>::fromRenormed(renormed32u); + decoder32 = rans::makeDecoder<>::fromRenormed(renormed32); + } + + template + const auto& getEncoder() const noexcept + { + if constexpr (std::is_same_v) { + return encoder8u; + } else if constexpr (std::is_same_v) { + return encoder8; + } else if constexpr (std::is_same_v) { + return encoder16u; + } else if constexpr (std::is_same_v) { + return encoder16; + } else if constexpr (std::is_same_v) { + return encoder32u; + } else if constexpr (std::is_same_v) { + return encoder32; + } else { + throw std::runtime_error{"unsupported encoder type"}; + } + }; + + template + const auto& getDecoder() const noexcept + { + if constexpr (std::is_same_v) { + return decoder8u; + } else if constexpr (std::is_same_v) { + return decoder8; + } else if constexpr (std::is_same_v) { + return decoder16u; + } else if constexpr (std::is_same_v) { + return decoder16; + } else if constexpr (std::is_same_v) { + return decoder32u; + } else if constexpr (std::is_same_v) { + return decoder32; + } else { + throw std::runtime_error{"unsupported encoder type"}; + } + }; + + private: + rans::denseEncoder_type encoder8u{}; + rans::denseEncoder_type encoder8{}; + rans::denseEncoder_type encoder16u{}; + rans::denseEncoder_type encoder16{}; + rans::denseEncoder_type encoder32u{}; + rans::denseEncoder_type encoder32{}; + + rans::defaultDecoder_type decoder8u{}; + rans::defaultDecoder_type decoder8{}; + rans::defaultDecoder_type decoder16u{}; + rans::defaultDecoder_type decoder16{}; + rans::defaultDecoder_type decoder32u{}; + rans::defaultDecoder_type decoder32{}; +}; + +ExternalEncoderDecoderProxy ExternalEncoders{}; + +template +void encodeExternal(source_IT begin, source_IT end) +{ + using source_type = typename std::iterator_traits::value_type; + + ctf::internal::ExternalEntropyCoder entropyCoder{ExternalEncoders.getEncoder()}; + + const size_t sourceExtent = std::distance(begin, end); + std::vector encodeBuffer(entropyCoder.template computePayloadSizeEstimate(sourceExtent), 0); + auto encoderEnd = entropyCoder.encode(begin, end, encodeBuffer.data(), encodeBuffer.data() + encodeBuffer.size()); + + std::vector literalSymbolsBuffer(entropyCoder.template computePackedIncompressibleSize(), 0); + [[maybe_unused]] auto literalsEnd = entropyCoder.writeIncompressible(literalSymbolsBuffer.data(), literalSymbolsBuffer.data() + literalSymbolsBuffer.size()); + + // decode + auto decoder = ExternalEncoders.getDecoder(); + std::vector literals((entropyCoder.getNIncompressibleSamples())); + + rans::unpack(literalSymbolsBuffer.data(), literals.size(), literals.data(), + entropyCoder.getIncompressibleSymbolPackingBits(), entropyCoder.getIncompressibleSymbolOffset()); + + size_t messageLength = std::distance(begin, end); + std::vector sourceBuffer(messageLength, 0); + + decoder.process(encoderEnd, sourceBuffer.data(), messageLength, entropyCoder.getEncoder().getNStreams(), literals.end()); + + BOOST_CHECK_EQUAL_COLLECTIONS(sourceBuffer.begin(), sourceBuffer.end(), begin, end); +}; + +BOOST_AUTO_TEST_CASE_TEMPLATE(testExternalEncoderEmpty, source_T, source_types) +{ + std::vector testMessage{}; + encodeExternal(testMessage.data(), testMessage.data() + testMessage.size()); +}; + +BOOST_AUTO_TEST_CASE_TEMPLATE(testExternalEncoderPTR, source_T, source_types) +{ + const auto& testMessage = MessageProxy.getMessage(); + encodeExternal(testMessage.data(), testMessage.data() + testMessage.size()); +}; + +BOOST_AUTO_TEST_CASE_TEMPLATE(testExternalEncoderIter, source_T, source_types) +{ + const auto& testMessage = MessageProxy.getMessage(); + encodeExternal(testMessage.begin(), testMessage.end()); +}; + +BOOST_AUTO_TEST_CASE(testExternalEncoderCombinedIterator) +{ + + const auto& testMessage1 = MessageProxy.getMessage(); + const auto& testMessage2 = MessageProxy.getMessage(); + + auto [begin, end] = makeInputIterators(testMessage1.data(), testMessage2.data(), testMessage1.size(), ShiftFunctor()>{}); + + encodeExternal(begin, end); +}; \ No newline at end of file diff --git a/DataFormats/Detectors/EMCAL/CMakeLists.txt b/DataFormats/Detectors/EMCAL/CMakeLists.txt index 6e18044fb0b87..9c93bae30ddf6 100644 --- a/DataFormats/Detectors/EMCAL/CMakeLists.txt +++ b/DataFormats/Detectors/EMCAL/CMakeLists.txt @@ -10,38 +10,42 @@ # or submit itself to any jurisdiction. o2_add_library(DataFormatsEMCAL - SOURCES src/EMCALBlockHeader.cxx - src/TriggerRecord.cxx - src/Constants.cxx - src/Cluster.cxx - src/AnalysisCluster.cxx - src/Cell.cxx - src/Digit.cxx - src/EventHandler.cxx - src/CTF.cxx - src/ErrorTypeFEE.cxx - PUBLIC_LINK_LIBRARIES O2::CommonDataFormat - O2::Headers - O2::MathUtils - O2::DetectorsBase - O2::SimulationDataFormat - Boost::serialization) + SOURCES src/EMCALBlockHeader.cxx + src/TriggerRecord.cxx + src/Constants.cxx + src/Cluster.cxx + src/AnalysisCluster.cxx + src/Cell.cxx + src/Digit.cxx + src/EventHandler.cxx + src/CTF.cxx + src/ErrorTypeFEE.cxx + src/CellLabel.cxx + src/ClusterLabel.cxx + src/CompressedTriggerData.cxx + PUBLIC_LINK_LIBRARIES O2::CommonDataFormat + O2::Headers + O2::MathUtils + O2::SimulationDataFormat + Boost::serialization) o2_target_root_dictionary(DataFormatsEMCAL - HEADERS include/DataFormatsEMCAL/EMCALBlockHeader.h - include/DataFormatsEMCAL/TriggerRecord.h - include/DataFormatsEMCAL/Constants.h - include/DataFormatsEMCAL/Cell.h - include/DataFormatsEMCAL/Digit.h - include/DataFormatsEMCAL/Cluster.h - include/DataFormatsEMCAL/AnalysisCluster.h - include/DataFormatsEMCAL/EventHandler.h - include/DataFormatsEMCAL/MCLabel.h - include/DataFormatsEMCAL/CTF.h - include/DataFormatsEMCAL/ErrorTypeFEE.h) + HEADERS include/DataFormatsEMCAL/EMCALBlockHeader.h + include/DataFormatsEMCAL/TriggerRecord.h + include/DataFormatsEMCAL/Constants.h + include/DataFormatsEMCAL/Cell.h + include/DataFormatsEMCAL/Digit.h + include/DataFormatsEMCAL/Cluster.h + include/DataFormatsEMCAL/AnalysisCluster.h + include/DataFormatsEMCAL/EventHandler.h + include/DataFormatsEMCAL/MCLabel.h + include/DataFormatsEMCAL/CTF.h + include/DataFormatsEMCAL/ErrorTypeFEE.h + include/DataFormatsEMCAL/CellLabel.h + include/DataFormatsEMCAL/ClusterLabel.h) o2_add_test(Cell - SOURCES test/testCell.cxx - COMPONENT_NAME DataFormats-EMCAL - PUBLIC_LINK_LIBRARIES O2::DataFormatsEMCAL - LABELS emcal dataformats) + SOURCES test/testCell.cxx + COMPONENT_NAME DataFormats-EMCAL + PUBLIC_LINK_LIBRARIES O2::DataFormatsEMCAL + LABELS emcal dataformats) diff --git a/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/AnalysisCluster.h b/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/AnalysisCluster.h index 52c1fa39d0be2..e19fd17dea2ce 100644 --- a/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/AnalysisCluster.h +++ b/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/AnalysisCluster.h @@ -12,7 +12,7 @@ #ifndef ALICEO2_EMCAL_ANALYSISCLUSTER_H_ #define ALICEO2_EMCAL_ANALYSISCLUSTER_H_ -#include "FairLogger.h" +#include #include #include #include "Rtypes.h" @@ -182,6 +182,9 @@ class AnalysisCluster float getCoreEnergy() const { return mCoreEnergy; } void setCoreEnergy(float energy) { mCoreEnergy = energy; } + float getFCross() const { return mFCross; } + void setFCross(float fCross) { mFCross = fCross; } + /// /// Returns TLorentzVector with momentum of the cluster. Only valid for clusters /// identified as photons or pi0 (overlapped gamma) produced on the vertex @@ -223,12 +226,13 @@ class AnalysisCluster float mTime = 0.; ///< Time of the digit/cell with maximal energy deposition bool mIsExotic = false; //! +#include +#include #include "DataFormatsEMCAL/Constants.h" namespace o2 @@ -27,104 +29,105 @@ namespace emcal /// \since March 6, 2019 /// \ingroup EMCALDataFormat /// -/// # Base format for EMCAL cell information in the Compressed Timeframe +/// # Cell content /// /// The cell class contains the relevant information for each tower per event /// - Tower ID /// - Energy of the raw fit /// - Time of the raw fit /// - Type of the cell +/// +/// # Compression for CTF +/// /// While cell type and tower ID have a predefined range based on the hardware /// design, energy and time have a finite resolution influenced by the resolution /// of the digitizer. This is used in order to compress the information stored /// in the compressed timeframe by not storing the full double values but instead /// assigning a certain amount of bits to each information. Therefore for certain -/// information (energy, time) precision loss has to be taken into account. -/// -/// # Internal structure and resolution -/// -/// The internal structure is a bit field compressing the information to -/// 48 bits. The definition of the bit field as well as the value range and the resolution -/// is listed in the table below: +/// information (energy, time) precision loss has to be taken into account. The number +/// of bits assigned to each data member in the encoding are as follows: /// -/// | Bits | Content | Resolution | Range | -/// |-------|---------------|---------------|-----------------------------| -/// | 0-14 | Tower ID | - | 0 to 17644 | -/// | 15-26 | Time (ns) | 0.73 ns | -600 to 900 ns | -/// | 27-40 | Energy (GeV) | 0.0153 GeV | 0 to 250 GeV | -/// | 41-42 | Cell type | - | 0=LG, 1=HG, 2=LEMon, 4=TRU | +/// | Content | Number of bits |Resolution | Range | +/// |---------------|----------------|--------------|-----------------------------| +/// | Tower ID | 15 | - | 0 to 17644 | +/// | Time (ns) | 11 | 0.73 ns | -600 to 900 ns | +/// | Energy (GeV) | 14 | 0.0153 GeV | 0 to 250 GeV | +/// | Cell type | 2 | - | 0=LG, 1=HG, 2=LEMon, 4=TRU | /// /// The remaining bits are 0 class Cell { public: - Cell(); - Cell(short tower, float energy, float time, ChannelType_t ctype = ChannelType_t::LOW_GAIN); + enum class EncoderVersion { + EncodingV0, + EncodingV1, + EncodingV2 + }; + /// \brief Default constructor + Cell() = default; + + /// \brief Constructor + /// \param tower Tower ID + /// \param energy Energy + /// \param timestamp Cell time + /// \param ctype Channel type + Cell(short tower, float energy, float timestamp, ChannelType_t ctype = ChannelType_t::LOW_GAIN); + + /// \brief Constructor, from encoded bit representation + /// \param tower Tower bitsets + /// \param energy Energy bits + /// \param timestamp Cell time bits + /// \param ctype Channel type bits + /// \param version Encoding version + Cell(uint16_t towerBits, uint16_t energyBits, uint16_t timestampBits, uint16_t channelBits, EncoderVersion version = EncoderVersion::EncodingV1); + + /// \brief Destructor ~Cell() = default; // override - void setTower(short tower) { getDataRepresentation()->mTowerID = tower; } - short getTower() const { return getDataRepresentation()->mTowerID; } + /// \brief Set the tower ID + /// \param tower Tower ID + void setTower(short tower) { mTowerID = tower; } + + /// \brief Get the tower ID + /// \return Tower ID + short getTower() const { return mTowerID; } /// \brief Set the time stamp - /// \param time Time in ns - /// - /// The time stamp is expressed in ns and has - /// a resolution of 1 ns. The time range which can - /// be stored is from -1023 to 1023 ns. In case the - /// range is exceeded the time is set to the limit - /// of the range. - void setTimeStamp(float time); + /// \param timestamp Time in ns + void setTimeStamp(float timestamp) { mTimestamp = timestamp; } /// \brief Get the time stamp /// \return Time in ns - /// - /// Time has a resolution of 1 ns and can cover - /// a range from -1023 to 1023 ns - float getTimeStamp() const; + float getTimeStamp() const { return mTimestamp; } /// \brief Set the energy of the cell /// \brief Energy of the cell in GeV - /// - /// The energy range covered by the cell - /// is 0 - 250 GeV, with a resolution of - /// 0.0153 GeV. In case an energy exceeding - /// the limits is provided the energy is - /// set to the limits (0 in case of negative - /// energy, 250. in case of energies > 250 GeV) - void setEnergy(float energy); + void setEnergy(float energy) { mEnergy = energy; } /// \brief Get the energy of the cell /// \return Energy of the cell - /// - /// The energy is truncated to a range - /// covering 0 to 250 GeV with a resolution - /// of 0.0153 GeV - float getEnergy() const; + float getEnergy() const { return mEnergy; } /// \brief Set the amplitude of the cell /// \param amplitude Cell amplitude - /// - /// See setEnergy for more information void setAmplitude(float amplitude) { setEnergy(amplitude); } /// \brief Get cell amplitude - /// \return cell Amplitude - /// - /// Set getEnergy for more information + /// \return Cell amplitude in GeV float getAmplitude() const { return getEnergy(); } /// \brief Set the type of the cell /// \param ctype Type of the cell (HIGH_GAIN, LOW_GAIN, LEDMON, TRU) - void setType(ChannelType_t ctype) { getDataRepresentation()->mCellStatus = static_cast(ctype); } + void setType(ChannelType_t ctype) { mChannelType = ctype; } /// \brief Get the type of the cell /// \return Type of the cell (HIGH_GAIN, LOW_GAIN, LEDMON, TRU) - ChannelType_t getType() const { return static_cast(getDataRepresentation()->mCellStatus); } + ChannelType_t getType() const { return mChannelType; } /// \brief Check whether the cell is of a given type /// \param ctype Type of the cell (HIGH_GAIN, LOW_GAIN, LEDMON, TRU) /// \return True if the type of the cell matches the requested type, false otherwise - bool isChannelType(ChannelType_t ctype) const { return getType() == ctype; } + bool isChannelType(ChannelType_t ctype) const { return mChannelType == ctype; } /// \brief Mark cell as low gain cell void setLowGain() { setType(ChannelType_t::LOW_GAIN); } @@ -154,38 +157,101 @@ class Cell /// \return True if the cell type is TRU, false otherwise Bool_t getTRU() const { return isChannelType(ChannelType_t::TRU); } + /// \brief Apply compression as done during writing to / reading from CTF + /// \param version Encoder version + void truncate(EncoderVersion version = EncoderVersion::EncodingV1); + void PrintStream(std::ostream& stream) const; - /// used for CTF encoding/decoding: access to packed data - void setPacked(uint16_t tower, uint16_t t, uint16_t en, uint16_t status) + /// \brief Initialize cell class from bit representation (for CTF decoding) + /// \param towerIDBits Encoded tower ID + /// \param timestampBits Encoded timestamp + /// \param energyBits Encoded energy + /// \param celltypeBits Encoded cell type + /// \param version Encoder version + void initialiseFromEncoded(uint16_t towerIDBits, uint16_t timestampBits, uint16_t energyBits, uint16_t celltypeBits, EncoderVersion version = EncoderVersion::EncodingV1) { - auto dt = getDataRepresentation(); - dt->mTowerID = tower; - dt->mTime = t; - dt->mEnergy = en; - dt->mCellStatus = status; + setEnergyEncoded(energyBits, static_cast(celltypeBits), version); + setTimestampEncoded(timestampBits); + setTowerIDEncoded(towerIDBits); + setChannelTypeEncoded(celltypeBits); } - auto getPackedTowerID() const { return getDataRepresentation()->mTowerID; } - auto getPackedTime() const { return getDataRepresentation()->mTime; } - auto getPackedEnergy() const { return getDataRepresentation()->mEnergy; } - auto getPackedCellStatus() const { return getDataRepresentation()->mCellStatus; } + /// \brief Get encoded bit representation of tower ID (for CTF) + /// \return Encoded bit representation + /// + /// Same as getTower - no compression applied for tower ID + uint16_t getTowerIDEncoded() const; + + /// \brief Get encoded bit representation of timestamp (for CTF) + /// \return Encoded bit representation + /// + /// The time stamp is expressed in ns and has + /// a resolution of 1 ns. The time range which can + /// be stored is from -1023 to 1023 ns. In case the + /// range is exceeded the time is set to the limit + /// of the range. + uint16_t getTimeStampEncoded() const; + + /// \brief Get encoded bit representation of energy (for CTF) + /// \param version Encoding verions + /// \return Encoded bit representation + /// + /// The energy range covered by the cell + /// is 0 - 250 GeV, with a resolution of + /// 0.0153 GeV. In case an energy exceeding + /// the limits is provided the energy is + /// set to the limits (0 in case of negative + /// energy, 250. in case of energies > 250 GeV) + uint16_t getEnergyEncoded(EncoderVersion version = EncoderVersion::EncodingV2) const; + + /// \brief Get encoded bit representation of cell type (for CTF) + /// \return Encoded bit representation + uint16_t getCellTypeEncoded() const; + + void initializeFromPackedBitfieldV0(const char* bitfield); + + static float getEnergyFromPackedBitfieldV0(const char* bitfield); + static float getTimeFromPackedBitfieldV0(const char* bitfield); + static ChannelType_t getCellTypeFromPackedBitfieldV0(const char* bitfield); + static short getTowerFromPackedBitfieldV0(const char* bitfield); + + static uint16_t encodeTime(float timestamp); + static uint16_t encodeEnergyV0(float energy); + static uint16_t encodeEnergyV1(float energy, ChannelType_t celltype); + static uint16_t encodeEnergyV2(float energy, ChannelType_t celltype); + static uint16_t V0toV1(uint16_t energybits, ChannelType_t celltype); + static uint16_t V0toV2(uint16_t energybits, ChannelType_t celltype); + static uint16_t V1toV2(uint16_t energybits, ChannelType_t celltype); + static float decodeTime(uint16_t timestampBits); + static float decodeEnergyV0(uint16_t energybits); + static float decodeEnergyV1(uint16_t energybits, ChannelType_t celltype); + static float decodeEnergyV2(uint16_t energybits, ChannelType_t celltype); private: - struct __attribute__((packed)) CellData { - uint16_t mTowerID : 15; ///< bits 0-14 Tower ID - uint16_t mTime : 11; ///< bits 15-25: Time (signed, can become negative after calibration) - uint16_t mEnergy : 14; ///< bits 26-39: Energy - uint16_t mCellStatus : 2; ///< bits 40-41: Cell status - uint16_t mZerod : 6; ///< bits 42-47: Zerod - }; + /// \brief Set cell energy from encoded bit representation (from CTF) + /// \param energyBits Bit representation of energy + /// \param cellTypeBits Bit representation of cell type + void setEnergyEncoded(uint16_t energyBits, uint16_t cellTypeBits, EncoderVersion version = EncoderVersion::EncodingV1); + + /// \brief Set cell time from encoded bit representation (from CTF) + /// \param timestampBits Bit representation of timestamp + void setTimestampEncoded(uint16_t timestampBits); + + /// \brief Set tower ID from encoded bit representation (from CTF) + /// \param towerIDBits Bit representation of towerID + void setTowerIDEncoded(uint16_t towerIDBits); - CellData* getDataRepresentation() { return reinterpret_cast(mCellWords); } - const CellData* getDataRepresentation() const { return reinterpret_cast(mCellWords); } + /// \brief Set cell type from encoded bit representation (from CTF) + /// \param channelTypeBits Bit representation of cell type + void setChannelTypeEncoded(uint16_t channelTypeBits); - uint16_t mCellWords[3]; ///< data word + float mEnergy = FLT_MIN; ///< Energy + float mTimestamp = FLT_MIN; ///< Timestamp + short mTowerID = SHRT_MAX; ///< Tower ID + ChannelType_t mChannelType = ChannelType_t::HIGH_GAIN; ///< Cell type - ClassDefNV(Cell, 1); + ClassDefNV(Cell, 3); }; /// \brief Stream operator for EMCAL cell diff --git a/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/CellLabel.h b/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/CellLabel.h new file mode 100644 index 0000000000000..543e49fb06dd8 --- /dev/null +++ b/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/CellLabel.h @@ -0,0 +1,80 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef ALICEO2_EMCAL_CELLLABEL_H_ +#define ALICEO2_EMCAL_CELLLABEL_H_ + +#include +#include +#include +#include + +namespace o2 +{ + +namespace emcal +{ + +/// \class CellLabel +/// \brief cell class for MC particle IDs and their respective amplitude fraction +/// \ingroup EMCALDataFormat +/// \author Marvin Hemmer , Goethe university Frankfurt +/// \since December 13, 2023 +/// + +class CellLabel +{ + public: + // CellLabel() = default; + + /// \brief Constructor using std::vector by moving NOT copying + /// \param labels list of mc labels + /// \param amplitudeFractions list of amplitude fractions + CellLabel(std::vector labels, std::vector amplitudeFractions); + + /// \brief Constructor using gsl::span + /// \param labels list of mc labels + /// \param amplitudeFractions list of amplitude fractions + CellLabel(gsl::span labels, gsl::span amplitudeFractions); + + // ~CellLabel() = default; + // CellLabel(const CellLabel& clus) = default; + // CellLabel& operator=(const CellLabel& source) = default; + + /// \brief Getter of label size + /// \param index index which label to get + size_t GetLabelSize(void) const { return mLabels.size(); } + + /// \brief Getter for label + /// \param index index which label to get + int32_t GetLabel(size_t index) const { return mLabels[index]; } + + /// \brief Getter for labels + std::vector GetLabels() const { return mLabels; } + + /// \brief Getter for amplitude fraction + /// \param index index which amplitude fraction to get + float GetAmplitudeFraction(size_t index) const { return mAmplitudeFraction[index]; } + + /// \brief Getter for amplitude fractions + std::vector GetAmplitudeFractions() const { return mAmplitudeFraction; } + + /// \brief Getter for label with leading amplitude fraction + int32_t GetLeadingMCLabel() const; + + protected: + std::vector mLabels; ///< List of MC particles that generated the cluster, ordered in deposited energy. + std::vector mAmplitudeFraction; ///< List of the fraction of the cell energy coming from a MC particle. Index aligns with mLabels! +}; + +} // namespace emcal +} // namespace o2 +#endif // ALICEO2_EMCAL_CELLLABEL_H_ diff --git a/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/ClusterLabel.h b/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/ClusterLabel.h new file mode 100644 index 0000000000000..b6db76f91ff34 --- /dev/null +++ b/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/ClusterLabel.h @@ -0,0 +1,95 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef ALICEO2_EMCAL_CLUSTERLABEL_H_ +#define ALICEO2_EMCAL_CLUSTERLABEL_H_ + +#include +#include +#include +#include "Rtypes.h" + +namespace o2 +{ + +namespace emcal +{ + +/// \class ClusterLabel +/// \brief cluster class for MC particle IDs and their respective energy fraction +/// \ingroup EMCALDataFormat +/// \author Marvin Hemmer , Goethe university Frankfurt +/// \since December 13, 2023 +/// + +class ClusterLabel +{ + public: + /// \struct labelWithE + /// \brief Wrapper structure to make cluster label sortable in energy fraction + struct labelWithE { + + /// \brief Constructor + labelWithE() : label(0), energyFraction(0.) {} + + /// \brief Constructor + /// \param l MC label + /// \param e Energy fraction + labelWithE(int l, float e) : label(l), energyFraction(e) {} + + /// \brief Comparison lower operator comparing cells based on energy + /// + /// std::sort will require operator>= to compile. + /// + /// \param rhs Label to compare to + /// \return True if this cell is has a lower energy, false otherwise + bool operator>=(labelWithE const& rhs) const + { + return energyFraction >= rhs.energyFraction; + } + + float energyFraction; ///< Energy Fraction + int label; ///< MC label + }; + + // ClusterLabel() = default; + // ~ClusterLabel() = default; + // ClusterLabel(const ClusterLabel& clus) = default; + // ClusterLabel& operator=(const ClusterLabel& source) = default; + + /// \brief Clear the member variables + void clear(); + + /// \brief Add label and energy fraction to the + /// \param label MC label + /// \param energyFraction Energy fraction + void addValue(int label, float energyFraction); + + /// \brief Normalize the energy fraction + /// \param factor normalization factor + void normalize(float factor); + + /// \brief Getter for vector of labels + std::vector getLabels(); + + /// \brief Getter for vector of energy fractions + std::vector getEnergyFractions(); + + /// \brief Sort the labels and energy fraction in descending order (largest energy fraction to smallest) + void orderLabels(); + + protected: + std::vector mClusterLabels; ///< List of MC particles that generated the cluster, paired with energy fraction +}; + +} // namespace emcal +} // namespace o2 +#endif // ALICEO2_EMCAL_CLUSTERLABEL_H_ diff --git a/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/CompressedTriggerData.h b/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/CompressedTriggerData.h new file mode 100644 index 0000000000000..5fbf2187ab5dd --- /dev/null +++ b/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/CompressedTriggerData.h @@ -0,0 +1,68 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +#ifndef ALICEO2_EMCAL_COMPRESSEDTRIGGERDATA_H +#define ALICEO2_EMCAL_COMPRESSEDTRIGGERDATA_H + +#include +#include + +namespace o2::emcal +{ + +/// \struct CompressedTRU +/// \brief Compressed reconstructed TRU information +/// \ingroup EMCALDataFormat +struct CompressedTRU { + uint8_t mTRUIndex; ///< TRU index + uint8_t mTriggerTime; ///< Trigger time of the TRU + bool mFired; ///< Fired status of the TRU + uint8_t mNumberOfPatches; ///< Number of patches found for the TRU +}; + +/// \struct CompressedTriggerPatch +/// \brief Compressed reconstructed L0 trigger patch information +/// \ingroup EMCALDataFormat +struct CompressedTriggerPatch { + uint8_t mTRUIndex; ///< Index of the TRU where the trigger patch has been found + uint8_t mPatchIndexInTRU; ///< Index of the trigger patch in the TRU + uint8_t mTime; ///< Reconstructed time of the trigger patch + uint16_t mADC; ///< ADC sum of the trigger patch +}; + +/// \struct CompressedL0TimeSum +/// \brief Compressed L0 timesum information +/// \ingroup EMCALDataFormat +struct CompressedL0TimeSum { + uint16_t mIndex; ///< Absolute ID of the FastOR + uint16_t mTimesum; ///< ADC value of the time-sum (4-integral) +}; + +/// \brief Output stream operator of the CompressedTRU +/// \param stream Stream to write to +/// \param tru TRU data to be streamed +/// \return Stream after writing +std::ostream& operator<<(std::ostream& stream, const CompressedTRU& tru); + +/// \brief Output stream operator of the CompressedTriggerPatch +/// \param stream Stream to write to +/// \param patch Trigger patch to be streamed +/// \return Stream after writing +std::ostream& operator<<(std::ostream& stream, const CompressedTriggerPatch& patch); + +/// \brief Output stream operator of the CompressedL0TimeSum +/// \param stream Stream to write to +/// \param timesum FastOR L0 timesum to be streamed +/// \return Stream after writing +std::ostream& operator<<(std::ostream& stream, const CompressedL0TimeSum& timesum); + +} // namespace o2::emcal + +#endif // ALICEO2_EMCAL_COMPRESSEDTRIGGERDATA_H \ No newline at end of file diff --git a/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/Constants.h b/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/Constants.h index 103a9419b5e08..015c77deccc46 100644 --- a/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/Constants.h +++ b/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/Constants.h @@ -103,6 +103,11 @@ constexpr int MAX_RANGE_ADC = 0x3FF; ///< Dynamic range of the ADCs (1 constexpr double EMCAL_TRU_ADCENERGY = 0.0786; ///< resolution of the TRU digitizer, @TODO check exact value } // namespace constants +namespace triggerbits +{ +constexpr uint32_t Inc = 0x1 << 20; ///< trigger bit marking incomplete event +} + enum FitAlgorithm { Standard = 0, ///< Standard raw fitter Gamma2 = 1, ///< Gamma2 raw fitter @@ -110,6 +115,35 @@ enum FitAlgorithm { NONE = 3 }; +enum STUtype_t { + ESTU = 0, ///< EMCAL STU + DSTU = 1 ///< DCAL STU +}; + +namespace STUparam //[0]->EMCAL STU, [1]->DCAL STU +{ +constexpr int FeeID[2] = {44, 45}; ///< FEE_ID in RDH +constexpr int NTRU[2] = {32, 14}; ///< number of TRUs +constexpr int CFG_nWords[2] = {17, 17}; ///< number of configuration words +constexpr int L1JetIndex_nWords[2] = {11, 11}; ///< number of words with Jet indices +constexpr int L0index_nWords[2] = {96, 42}; ///< number of words with null data +constexpr int L1GammaIndex_nWords[2] = {128, 56}; ///< number of words with Gamma indices +constexpr int Raw_nWords[2] = {1536, 672}; ///< number of words with ADC +constexpr int SubregionsEta[2] = {12, 12}; ///< number of subregions over eta +constexpr int SubregionsPhi[2] = {16, 10}; ///< number of subregions over phi +constexpr int PaloadSizeFull[2] = {1928, 866}; ///< number of words in full payload = 1944/882-16 +constexpr int PaloadSizeShort[2] = {391, 193}; ///< number of words in shorts payload = 407/209-16 +} // namespace STUparam + +namespace TRUparam +{ +constexpr int Nchannels = 96; ///< number of FastORs per TRU +constexpr int NchannelsOverEta = 8; ///< number of FastORs over Eta for full- and 2/3-size SMs +constexpr int NchannelsOverPhi = 12; ///< number of FastORs over Phi for full- and 2/3-size SMs +constexpr int NchannelsOverEta_long = 24; ///< number of FastORs over Eta for 1/3-size SMs +constexpr int NchannelsOverPhi_long = 4; ///< number of FastORs over Phi for 1/3-size SMs +} // namespace TRUparam + } // namespace emcal } // namespace o2 #endif diff --git a/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/EMCALChannelData.h b/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/EMCALChannelData.h deleted file mode 100644 index 3c014d37e6f9e..0000000000000 --- a/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/EMCALChannelData.h +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// \file EMCALChannelData.h -/// \brief - -/// \class EMCALChannelCalibrator -/// \brief Class to store the data format for calibraton of the EMCal -/// \author Hannah Bossi, Yale University -/// \ingroup DetectorEMCAL -/// \since Feb 11, 2021 - -#ifndef ALICEO2_EMCALCHANNELDATA_H -#define ALICEO2_EMCALCHANNELDATA_H - -#include "Rtypes.h" - -namespace o2 -{ -namespace dataformats -{ -class EMCALChannelData -{ - public: - EMCALChannelData(int cellID, int timestamp, int flags = 0, int events) : mEMCALCellID(cellID), mTimestamp(timestamp), mFlags(flags){}; - EMCALChannelData() = default; - ~EMCALChannelData() = default; - - void setEMCALCellID(int index) { mEMCALCellID = index; } - int getEMCALCellID() const { return mEMCALCellID; } - - void setTimestamp(int ts) { mTimestamp = ts; } - int getTimestamp() const { return mTimestamp; } - - void setFlags(int flags) { mFlags = flags; } - float getFlags() const { return mFlags; } - - private: - int mEMCALCellID; ///< EMCal Cell ID - int mTimestamp; ///< timestamp in seconds - unsigned char mFlags; ///< bit mask with quality flags (to be defined) - - ClassDefNV(EMCALChannelData, 1); -}; -} // namespace dataformats -} // namespace o2 -#endif diff --git a/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/ErrorTypeFEE.h b/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/ErrorTypeFEE.h index 0be95db5c60f5..cc3540cd388d3 100644 --- a/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/ErrorTypeFEE.h +++ b/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/ErrorTypeFEE.h @@ -57,6 +57,9 @@ class ErrorTypeFEE FIT_ERROR, ///< Raw fit failed GEOMETRY_ERROR, ///< Decoded position outside EMCAL GAIN_ERROR, ///< Error due to gain type + LINK_ERROR, ///< Error due to missing DDL links + TRU_ERROR, ///< Errors from TRU data + STU_ERROR, ///< Error from STU data UNDEFINED ///< Error source undefined }; /// \brief Constructor @@ -97,6 +100,18 @@ class ErrorTypeFEE /// \param gainError Error code of the gain type error void setGainErrorType(int gainError) { setError(ErrorSource_t::GAIN_ERROR, gainError); } + /// \brief Set the error type as link error and store the error code + /// \param linkError Error code of the link error + void setLinkErrorTYpe(int linkError) { setError(ErrorSource_t::LINK_ERROR, linkError); } + + /// \brief Set the error as STU decoder error and store the error code + /// \param gainError Error code of the STU decoder error + void setSTUDecoderErrorType(int stuerror) { setError(ErrorSource_t::STU_ERROR, stuerror); } + + /// \brief Set the error as TRU decoder error and store the error code + /// \param gainError Error code of the TRU decoder error + void setTRUDecoderErrorType(int truerror) { setError(ErrorSource_t::TRU_ERROR, truerror); } + /// \brief Set the error type of the object /// \param errorsource Error type of the object void setErrorType(ErrorSource_t errorsource) { mErrorSource = errorsource; } @@ -157,6 +172,18 @@ class ErrorTypeFEE /// \return Error code (-1 in case the object is not a gain type error) int getGainTypeErrorType() const { return getRawErrorForType(ErrorSource_t::GAIN_ERROR); } + /// \brief Get the error code of the obect in case the object is a link error + /// \return Error code (-1 in case the object is not a gain type error) + int getLinkErrorType() const { return getRawErrorForType(ErrorSource_t::LINK_ERROR); } + + /// \brief Get the error code of the obect in case the object is a STU decoder error + /// \return Error code (-1 in case the object is not a STU decoder error) + int getSTUDecoderErrorType() const { return getRawErrorForType(ErrorSource_t::STU_ERROR); } + + /// \brief Get the error code of the obect in case the object is a TRU decoder error + /// \return Error code (-1 in case the object is not a STU decoder error) + int getTRUDecoderErrorType() const { return getRawErrorForType(ErrorSource_t::TRU_ERROR); } + /// \brief Get subspecification of the error /// \return Subspecification of the error int getSubspecification() const { return mSubspecification; } @@ -171,6 +198,20 @@ class ErrorTypeFEE /// Helper function, called in the output stream operator for the ErrorTypeFEE void PrintStream(std::ostream& stream) const; + /// \brief Get the number of error types + /// \return Number of error types (including undefined) + static constexpr int getNumberOfErrorTypes() { return 10; } + + /// \brief Get the name of the error type + /// \param errorTypeID ID of the error type + /// \return Name of the error type + static const char* getErrorTypeName(unsigned int errorTypeID); + + /// \brief Get the title of the error type + /// \param errorTypeID ID of the error type + /// \return Title of the error type + static const char* getErrorTypeTitle(unsigned int errorTypeID); + private: /// \brief Helper function getting the error code under condition that the error is of a certain type /// \return Error code (-1 in case the error handle by the object is not of the given type) diff --git a/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/EventData.h b/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/EventData.h index ae4fe9f350e76..90c56e7a354d9 100644 --- a/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/EventData.h +++ b/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/EventData.h @@ -12,9 +12,11 @@ #define ALICEO2_EMCAL_EVENTDATA_H_ #include #include +#include #include "CommonDataFormat/InteractionRecord.h" #include "DataFormatsEMCAL/Cell.h" #include "DataFormatsEMCAL/Cluster.h" +#include "DataFormatsEMCAL/MCLabel.h" namespace o2 { @@ -34,11 +36,12 @@ namespace emcal /// objects are not filled when creating the event structure. template struct EventData { - InteractionRecord mInteractionRecord; ///< Interaction record for the trigger corresponding to this event - gsl::span mClusters; ///< EMCAL clusters - gsl::span mCells; ///< EMCAL cells / digits - gsl::span mCellIndices; ///< Cell indices in cluster - uint64_t mTriggerBits; ///< Trigger bits for the event + InteractionRecord mInteractionRecord; ///< Interaction record for the trigger corresponding to this event + gsl::span mClusters; ///< EMCAL clusters + gsl::span mCells; ///< EMCAL cells / digits + gsl::span mCellIndices; ///< Cell indices in cluster + std::vector> mMCCellLabels; ///< span of MC labels for each cell + uint64_t mTriggerBits; ///< Trigger bits for the event /// \brief Reset event structure with empty interaction record and ranges void reset() @@ -47,6 +50,7 @@ struct EventData { mClusters = gsl::span(); mCells = gsl::span(); mCellIndices = gsl::span(); + mMCCellLabels = std::vector>(); mTriggerBits = 0; } @@ -57,4 +61,4 @@ struct EventData { } // namespace o2 -#endif // ALICEO2_EMCAL_EVENTDATA_H_ \ No newline at end of file +#endif // ALICEO2_EMCAL_EVENTDATA_H_ diff --git a/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/EventHandler.h b/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/EventHandler.h index 5bdd06c916b73..326a256d7a802 100644 --- a/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/EventHandler.h +++ b/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/EventHandler.h @@ -15,13 +15,16 @@ #include #include #include +#include #include "Rtypes.h" #include "fmt/format.h" #include "DataFormatsEMCAL/Cell.h" #include "DataFormatsEMCAL/Cluster.h" #include "DataFormatsEMCAL/Digit.h" #include "DataFormatsEMCAL/EventData.h" +#include "DataFormatsEMCAL/MCLabel.h" #include "DataFormatsEMCAL/TriggerRecord.h" +#include "SimulationDataFormat/MCTruthContainer.h" namespace o2 { @@ -363,6 +366,13 @@ class EventHandler /// \throw NotInitializedException in case the event handler is not initialized for cell const CellRange getCellsForEvent(int eventID) const; + /// \brief Get vector of MC labels belonging to the given event + /// \param eventID ID of the event + /// \return vector of MC labels for the event + /// \throw RangeException in case the required event ID exceeds the maximum number of events + /// \throw NotInitializedException in case the event handler is not initialized for cell + std::vector> getCellMCLabelForEvent(int eventID) const; + /// \brief Get range of cluster cell indices belonging to the given event /// \param eventID ID of the event /// \return Cluster cell index range for the event @@ -404,6 +414,13 @@ class EventHandler mTriggerRecordsCells = triggers; } + /// \brief Setting the pointer for the MCTruthContainer for cells + /// \param mclabels Pointer to the MCTruthContainer for cells in timeframe + void setCellMCTruthContainer(const o2::dataformats::MCTruthContainer* mclabels) + { + mCellLabels = mclabels; + } + /// \brief Reset containers with empty ranges void reset(); @@ -433,9 +450,10 @@ class EventHandler TriggerRange mTriggerRecordsCellIndices; ///< trigger record for cluster cell index type TriggerRange mTriggerRecordsCells; ///< Trigger record for cell type - ClusterRange mClusters; /// container for clusters in timeframe - CellIndexRange mClusterCellIndices; /// container for cell indices in timeframe - CellRange mCells; /// container for cells in timeframe + ClusterRange mClusters; /// container for clusters in timeframe + CellIndexRange mClusterCellIndices; /// container for cell indices in timeframe + CellRange mCells; /// container for cells in timeframe + const o2::dataformats::MCTruthContainer* mCellLabels = nullptr; /// pointer to the MCTruthContainer for cells in timeframe ClassDefNV(EventHandler, 1); }; @@ -443,4 +461,4 @@ class EventHandler } // namespace emcal } // namespace o2 -#endif // ALICEO2_EMCAL_EVENTHANDLER_H__ \ No newline at end of file +#endif // ALICEO2_EMCAL_EVENTHANDLER_H__ diff --git a/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/TriggerRecord.h b/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/TriggerRecord.h index a7e1f72ca1baf..1529eff104400 100644 --- a/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/TriggerRecord.h +++ b/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/TriggerRecord.h @@ -61,8 +61,9 @@ class TriggerRecord /// \enum TriggerBitsCoded_t /// \brief Position of trigger classes in compressed format enum TriggerBitsCoded_t { - PHYSTRIGGER, ///< Physics trigger - CALIBTRIGGER ///< Calib trigger + PHYSTRIGGER, ///< Physics trigger + CALIBTRIGGER, ///< Calib trigger + REJECTINCOMPLETE ///< Rejected as incomplete }; BCData mBCData; /// Bunch crossing data of the trigger DataRange mDataRange; /// Index of the triggering event (event index and first entry in the container) diff --git a/DataFormats/Detectors/EMCAL/src/AnalysisCluster.cxx b/DataFormats/Detectors/EMCAL/src/AnalysisCluster.cxx index 0bab205e815c3..05006b2618fd5 100644 --- a/DataFormats/Detectors/EMCAL/src/AnalysisCluster.cxx +++ b/DataFormats/Detectors/EMCAL/src/AnalysisCluster.cxx @@ -11,7 +11,7 @@ /// \file AnalysisCluster.cxx -#include "FairLogger.h" +#include #include #include #include diff --git a/DataFormats/Detectors/EMCAL/src/Cell.cxx b/DataFormats/Detectors/EMCAL/src/Cell.cxx index 4132d99c2a224..261384d53ca2a 100644 --- a/DataFormats/Detectors/EMCAL/src/Cell.cxx +++ b/DataFormats/Detectors/EMCAL/src/Cell.cxx @@ -17,58 +17,334 @@ using namespace o2::emcal; +namespace TimeEncoding +{ const float TIME_SHIFT = 600., TIME_RANGE = 1500., - TIME_RESOLUTION = TIME_RANGE / 2047., - ENERGY_TRUNCATION = 250., - ENERGY_RESOLUTION = ENERGY_TRUNCATION / 16383.; + TIME_RESOLUTION = TIME_RANGE / 2047.; + +} +namespace EnergyEncoding +{ +namespace v0 +{ +const float + ENERGY_TRUNCATION = 250., + ENERGY_RESOLUTION = ENERGY_TRUNCATION / 16383.; +} + +namespace v1 +{ +const float + ENERGY_BITS = static_cast(0x3FFF), + HGLGTRANSITION = o2::emcal::constants::EMCAL_HGLGTRANSITION * o2::emcal::constants::EMCAL_ADCENERGY, + ENERGY_TRUNCATION = 250., + ENERGY_RESOLUTION_LG = (ENERGY_TRUNCATION - HGLGTRANSITION) / ENERGY_BITS, + ENERGY_RESOLUTION_HG = HGLGTRANSITION / ENERGY_BITS, + ENERGY_RESOLUTION_TRU = ENERGY_TRUNCATION / ENERGY_BITS, + ENERGY_RESOLUTION_LEDMON = ENERGY_TRUNCATION / ENERGY_BITS; +} + +namespace v2 +{ +const float + ENERGY_BITS = static_cast(0x3FFF), + SAFETYMARGIN = 0.2, + HGLGTRANSITION = o2::emcal::constants::OVERFLOWCUT * o2::emcal::constants::EMCAL_ADCENERGY, + OFFSET_LG = HGLGTRANSITION - SAFETYMARGIN, + ENERGY_TRUNCATION = 250., + ENERGY_RESOLUTION_LG = (ENERGY_TRUNCATION - OFFSET_LG) / ENERGY_BITS, + ENERGY_RESOLUTION_HG = HGLGTRANSITION / ENERGY_BITS, + ENERGY_RESOLUTION_TRU = ENERGY_TRUNCATION / ENERGY_BITS, + ENERGY_RESOLUTION_LEDMON = ENERGY_TRUNCATION / ENERGY_BITS; + +} +} // namespace EnergyEncoding + +namespace DecodingV0 +{ +struct __attribute__((packed)) CellDataPacked { + uint16_t mTowerID : 15; ///< bits 0-14 Tower ID + uint16_t mTime : 11; ///< bits 15-25: Time (signed, can become negative after calibration) + uint16_t mEnergy : 14; ///< bits 26-39: Energy + uint16_t mCellStatus : 2; ///< bits 40-41: Cell status + uint16_t mZerod : 6; ///< bits 42-47: Zerod +}; +} // namespace DecodingV0 + +Cell::Cell(short tower, float energy, float timestamp, ChannelType_t ctype) : mTowerID(tower), mEnergy(energy), mTimestamp(timestamp), mChannelType(ctype) +{ +} + +Cell::Cell(uint16_t towerBits, uint16_t energyBits, uint16_t timestampBits, uint16_t channelBits, EncoderVersion version) +{ + initialiseFromEncoded(towerBits, timestampBits, energyBits, channelBits, version); +} + +uint16_t Cell::getTowerIDEncoded() const +{ + return mTowerID; +} + +uint16_t Cell::getTimeStampEncoded() const +{ + return encodeTime(mTimestamp); +} -Cell::Cell() +uint16_t Cell::getEnergyEncoded(EncoderVersion version) const { - memset(mCellWords, 0, sizeof(uint16_t) * 3); + uint16_t energyBits = 0; + switch (version) { + case EncoderVersion::EncodingV0: + energyBits = encodeEnergyV0(mEnergy); + break; + + case EncoderVersion::EncodingV1: + energyBits = encodeEnergyV1(mEnergy, mChannelType); + break; + + case EncoderVersion::EncodingV2: + energyBits = encodeEnergyV2(mEnergy, mChannelType); + break; + } + return energyBits; } -Cell::Cell(short tower, float energy, float time, ChannelType_t ctype) +uint16_t Cell::getCellTypeEncoded() const { - memset(mCellWords, 0, sizeof(uint16_t) * 3); - setTower(tower); - setTimeStamp(time); - setEnergy(energy); - setType(ctype); + return static_cast(mChannelType); } -void Cell::setTimeStamp(float timestamp) +void Cell::setEnergyEncoded(uint16_t energyBits, uint16_t channelTypeBits, EncoderVersion version) { - // truncate: - const float TIME_MIN = -1. * TIME_SHIFT, - TIME_MAX = TIME_RANGE - TIME_SHIFT; - if (timestamp < TIME_MIN) { - timestamp = TIME_MIN; - } else if (timestamp > TIME_MAX) { - timestamp = TIME_MAX; + switch (version) { + case EncoderVersion::EncodingV0: + mEnergy = decodeEnergyV0(energyBits); + break; + case EncoderVersion::EncodingV1: + mEnergy = decodeEnergyV1(energyBits, static_cast(channelTypeBits)); + break; + case EncoderVersion::EncodingV2: + mEnergy = decodeEnergyV2(energyBits, static_cast(channelTypeBits)); + break; } - getDataRepresentation()->mTime = static_cast(std::round((timestamp + TIME_SHIFT) / TIME_RESOLUTION)); } -float Cell::getTimeStamp() const +void Cell::setTimestampEncoded(uint16_t timestampBits) +{ + mTimestamp = decodeTime(timestampBits); +} + +void Cell::setTowerIDEncoded(uint16_t towerIDBits) { - return (static_cast(getDataRepresentation()->mTime) * TIME_RESOLUTION) - TIME_SHIFT; + mTowerID = towerIDBits; } -void Cell::setEnergy(float energy) +void Cell::setChannelTypeEncoded(uint16_t channelTypeBits) +{ + mChannelType = static_cast(channelTypeBits); +} + +void Cell::initializeFromPackedBitfieldV0(const char* bitfield) +{ + auto bitrepresentation = reinterpret_cast(bitfield); + mEnergy = decodeEnergyV0(bitrepresentation->mEnergy); + mTimestamp = decodeTime(bitrepresentation->mTime); + mTowerID = bitrepresentation->mTowerID; + mChannelType = static_cast(bitrepresentation->mCellStatus); +} + +float Cell::getEnergyFromPackedBitfieldV0(const char* bitfield) +{ + return decodeEnergyV0(reinterpret_cast(bitfield)->mEnergy); +} + +float Cell::getTimeFromPackedBitfieldV0(const char* bitfield) +{ + return decodeTime(reinterpret_cast(bitfield)->mTime); +} + +ChannelType_t Cell::getCellTypeFromPackedBitfieldV0(const char* bitfield) +{ + return static_cast(reinterpret_cast(bitfield)->mCellStatus); +} + +short Cell::getTowerFromPackedBitfieldV0(const char* bitfield) +{ + return reinterpret_cast(bitfield)->mTowerID; +} + +void Cell::truncate(EncoderVersion version) +{ + setEnergyEncoded(getEnergyEncoded(version), getCellTypeEncoded(), version); + setTimestampEncoded(getTimeStampEncoded()); +} + +uint16_t Cell::encodeTime(float timestamp) +{ + // truncate + auto timestampTruncated = timestamp; + const float TIME_MIN = -1. * TimeEncoding::TIME_SHIFT, + TIME_MAX = TimeEncoding::TIME_RANGE - TimeEncoding::TIME_SHIFT; + if (timestampTruncated < TIME_MIN) { + timestampTruncated = TIME_MIN; + } else if (timestampTruncated > TIME_MAX) { + timestampTruncated = TIME_MAX; + } + return static_cast(std::round((timestampTruncated + TimeEncoding::TIME_SHIFT) / TimeEncoding::TIME_RESOLUTION)); +} + +uint16_t Cell::encodeEnergyV0(float energy) +{ + auto truncatedEnergy = energy; + if (truncatedEnergy < 0.) { + truncatedEnergy = 0.; + } else if (truncatedEnergy > EnergyEncoding::v0::ENERGY_TRUNCATION) { + truncatedEnergy = EnergyEncoding::v0::ENERGY_TRUNCATION; + } + return static_cast(std::round(truncatedEnergy / EnergyEncoding::v0::ENERGY_RESOLUTION)); +} + +uint16_t Cell::encodeEnergyV1(float energy, ChannelType_t celltype) { double truncatedEnergy = energy; if (truncatedEnergy < 0.) { truncatedEnergy = 0.; - } else if (truncatedEnergy > ENERGY_TRUNCATION) { - truncatedEnergy = ENERGY_TRUNCATION; + } else if (truncatedEnergy > EnergyEncoding::v1::ENERGY_TRUNCATION) { + truncatedEnergy = EnergyEncoding::v1::ENERGY_TRUNCATION; } - getDataRepresentation()->mEnergy = static_cast(std::round(truncatedEnergy / ENERGY_RESOLUTION)); + float resolutionApplied = 0., energyOffset = 0.; + switch (celltype) { + case ChannelType_t::HIGH_GAIN: { + resolutionApplied = EnergyEncoding::v1::ENERGY_RESOLUTION_HG; + break; + } + case ChannelType_t::LOW_GAIN: { + resolutionApplied = EnergyEncoding::v1::ENERGY_RESOLUTION_LG; + energyOffset = EnergyEncoding::v1::HGLGTRANSITION; + break; + } + case ChannelType_t::TRU: { + resolutionApplied = EnergyEncoding::v1::ENERGY_RESOLUTION_TRU; + break; + } + case ChannelType_t::LEDMON: { + resolutionApplied = EnergyEncoding::v1::ENERGY_RESOLUTION_LEDMON; + break; + } + } + return static_cast(std::round((truncatedEnergy - energyOffset) / resolutionApplied)); +}; + +uint16_t Cell::encodeEnergyV2(float energy, ChannelType_t celltype) +{ + double truncatedEnergy = energy; + if (truncatedEnergy < 0.) { + truncatedEnergy = 0.; + } else if (truncatedEnergy > EnergyEncoding::v2::ENERGY_TRUNCATION) { + truncatedEnergy = EnergyEncoding::v2::ENERGY_TRUNCATION; + } + float resolutionApplied = 0., energyOffset = 0.; + switch (celltype) { + case ChannelType_t::HIGH_GAIN: { + resolutionApplied = EnergyEncoding::v2::ENERGY_RESOLUTION_HG; + break; + } + case ChannelType_t::LOW_GAIN: { + resolutionApplied = EnergyEncoding::v2::ENERGY_RESOLUTION_LG; + energyOffset = EnergyEncoding::v2::OFFSET_LG; + break; + } + case ChannelType_t::TRU: { + resolutionApplied = EnergyEncoding::v2::ENERGY_RESOLUTION_TRU; + break; + } + case ChannelType_t::LEDMON: { + resolutionApplied = EnergyEncoding::v2::ENERGY_RESOLUTION_LEDMON; + break; + } + } + return static_cast(std::round((truncatedEnergy - energyOffset) / resolutionApplied)); +}; + +uint16_t Cell::V0toV1(uint16_t energyBits, ChannelType_t celltype) +{ + auto decodedEnergy = decodeEnergyV0(energyBits); + return encodeEnergyV1(decodedEnergy, celltype); +} + +uint16_t Cell::V0toV2(uint16_t energyBits, ChannelType_t celltype) +{ + auto decodedEnergy = decodeEnergyV0(energyBits); + return encodeEnergyV2(decodedEnergy, celltype); +} + +uint16_t Cell::V1toV2(uint16_t energyBits, ChannelType_t celltype) +{ + auto decodedEnergy = decodeEnergyV1(energyBits, celltype); + return encodeEnergyV2(decodedEnergy, celltype); } -float Cell::getEnergy() const +float Cell::decodeTime(uint16_t timestampBits) { - return static_cast(getDataRepresentation()->mEnergy) * ENERGY_RESOLUTION; + return (static_cast(timestampBits) * TimeEncoding::TIME_RESOLUTION) - TimeEncoding::TIME_SHIFT; +} + +float Cell::decodeEnergyV0(uint16_t energyBits) +{ + return static_cast(energyBits) * EnergyEncoding::v0::ENERGY_RESOLUTION; +} + +float Cell::decodeEnergyV1(uint16_t energyBits, ChannelType_t celltype) +{ + float resolutionApplied = 0., + energyOffset = 0.; + switch (celltype) { + case ChannelType_t::HIGH_GAIN: { + resolutionApplied = EnergyEncoding::v1::ENERGY_RESOLUTION_HG; + break; + } + case ChannelType_t::LOW_GAIN: { + resolutionApplied = EnergyEncoding::v1::ENERGY_RESOLUTION_LG; + energyOffset = EnergyEncoding::v1::HGLGTRANSITION; + break; + } + case ChannelType_t::TRU: { + resolutionApplied = EnergyEncoding::v1::ENERGY_RESOLUTION_TRU; + break; + } + case ChannelType_t::LEDMON: { + resolutionApplied = EnergyEncoding::v1::ENERGY_RESOLUTION_LEDMON; + break; + } + } + return (static_cast(energyBits) * resolutionApplied) + energyOffset; +} + +float Cell::decodeEnergyV2(uint16_t energyBits, ChannelType_t celltype) +{ + float resolutionApplied = 0., + energyOffset = 0.; + switch (celltype) { + case ChannelType_t::HIGH_GAIN: { + resolutionApplied = EnergyEncoding::v2::ENERGY_RESOLUTION_HG; + break; + } + case ChannelType_t::LOW_GAIN: { + resolutionApplied = EnergyEncoding::v2::ENERGY_RESOLUTION_LG; + energyOffset = EnergyEncoding::v2::OFFSET_LG; + break; + } + case ChannelType_t::TRU: { + resolutionApplied = EnergyEncoding::v2::ENERGY_RESOLUTION_TRU; + break; + } + case ChannelType_t::LEDMON: { + resolutionApplied = EnergyEncoding::v2::ENERGY_RESOLUTION_LEDMON; + break; + } + } + return (static_cast(energyBits) * resolutionApplied) + energyOffset; } void Cell::PrintStream(std::ostream& stream) const @@ -76,7 +352,7 @@ void Cell::PrintStream(std::ostream& stream) const stream << "EMCAL Cell: Type " << getType() << ", Energy " << getEnergy() << ", Time " << getTimeStamp() << ", Tower " << getTower(); } -std::ostream& operator<<(std::ostream& stream, const Cell& c) +std::ostream& o2::emcal::operator<<(std::ostream& stream, const Cell& c) { c.PrintStream(stream); return stream; diff --git a/DataFormats/Detectors/EMCAL/src/CellLabel.cxx b/DataFormats/Detectors/EMCAL/src/CellLabel.cxx new file mode 100644 index 0000000000000..70a1a642c5449 --- /dev/null +++ b/DataFormats/Detectors/EMCAL/src/CellLabel.cxx @@ -0,0 +1,50 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file CellLabel.cxx + +#include "DataFormatsEMCAL/CellLabel.h" +#include "fairlogger/Logger.h" +#include +#include +#include +#include +#include + +using namespace o2::emcal; + +CellLabel::CellLabel(std::vector labels, std::vector amplitudeFractions) : mLabels(std::move(labels)), mAmplitudeFraction(std::move(amplitudeFractions)) +{ + if (labels.size() != amplitudeFractions.size()) { + LOG(error) << "Size of labels " << labels.size() << " does not match size of amplitude fraction " << amplitudeFractions.size() << " !"; + } +} + +CellLabel::CellLabel(gsl::span labels, gsl::span amplitudeFractions) : mLabels(labels.begin(), labels.end()), mAmplitudeFraction(amplitudeFractions.begin(), amplitudeFractions.end()) +{ + if (labels.size() != amplitudeFractions.size()) { + LOG(error) << "Size of labels " << labels.size() << " does not match size of amplitude fraction " << amplitudeFractions.size() << " !"; + } +} + +int32_t CellLabel::GetLeadingMCLabel() const +{ + size_t maxIndex = 0; + float maxFraction = mAmplitudeFraction[0]; + + for (size_t i = 1; i < mAmplitudeFraction.size(); ++i) { + if (mAmplitudeFraction[i] > maxFraction) { + maxFraction = mAmplitudeFraction[i]; + maxIndex = i; + } + } + return mLabels[maxIndex]; +} diff --git a/DataFormats/Detectors/EMCAL/src/ClusterLabel.cxx b/DataFormats/Detectors/EMCAL/src/ClusterLabel.cxx new file mode 100644 index 0000000000000..9ad1f9be5459f --- /dev/null +++ b/DataFormats/Detectors/EMCAL/src/ClusterLabel.cxx @@ -0,0 +1,75 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file ClusterLabel.cxx + +#include "DataFormatsEMCAL/ClusterLabel.h" + +using namespace o2::emcal; + +//_______________________________________________________________________ +void ClusterLabel::clear() +{ + mClusterLabels.clear(); +} + +//_______________________________________________________________________ +void ClusterLabel::addValue(int label, float energyFraction) +{ + auto it = std::find_if(mClusterLabels.begin(), mClusterLabels.end(), + [label](const labelWithE& lWE) { return lWE.label == label; }); + + if (it != mClusterLabels.end()) { + // label already exists, accumulate energy fraction + it->energyFraction += energyFraction; + } else { + // label does not exist, add new energy fraction + mClusterLabels.emplace_back(label, energyFraction); + } +} + +//_______________________________________________________________________ +void ClusterLabel::normalize(float factor) +{ + for (auto& clusterlabel : mClusterLabels) { + clusterlabel.energyFraction = clusterlabel.energyFraction / factor; + } +} + +//_______________________________________________________________________ +std::vector ClusterLabel::getLabels() +{ + std::vector vLabels; + vLabels.reserve(mClusterLabels.size()); + for (auto& clusterlabel : mClusterLabels) { + vLabels.push_back(clusterlabel.label); + } + return vLabels; +} + +//_______________________________________________________________________ +std::vector ClusterLabel::getEnergyFractions() +{ + std::vector vEnergyFractions; + vEnergyFractions.reserve(mClusterLabels.size()); + for (auto& clusterlabel : mClusterLabels) { + vEnergyFractions.push_back(clusterlabel.energyFraction); + } + return vEnergyFractions; +} + +//_______________________________________________________________________ +void ClusterLabel::orderLabels() +{ + // Sort the pairs based on values in descending order + std::sort(mClusterLabels.begin(), mClusterLabels.end(), + [](const labelWithE& a, const labelWithE& b) { return a.energyFraction > b.energyFraction; }); +} diff --git a/DataFormats/Detectors/EMCAL/src/CompressedTriggerData.cxx b/DataFormats/Detectors/EMCAL/src/CompressedTriggerData.cxx new file mode 100644 index 0000000000000..e60b58c958d03 --- /dev/null +++ b/DataFormats/Detectors/EMCAL/src/CompressedTriggerData.cxx @@ -0,0 +1,31 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include +#include "DataFormatsEMCAL/CompressedTriggerData.h" + +std::ostream& o2::emcal::operator<<(std::ostream& stream, const o2::emcal::CompressedTRU& tru) +{ + stream << "TRU " << tru.mTRUIndex << ": Fired " << (tru.mFired ? "yes" : "no") << ", time " << (tru.mFired ? std::to_string(static_cast(tru.mTriggerTime)) : "Undefined") << ", number of patches " << tru.mNumberOfPatches; + return stream; +} + +std::ostream& o2::emcal::operator<<(std::ostream& stream, const o2::emcal::CompressedTriggerPatch& patch) +{ + stream << "Patch " << patch.mPatchIndexInTRU << " in TRU " << patch.mTRUIndex << ": Time " << patch.mTime << ", ADC " << patch.mADC; + return stream; +} + +std::ostream& o2::emcal::operator<<(std::ostream& stream, const o2::emcal::CompressedL0TimeSum& timesum) +{ + stream << "FastOR " << timesum.mIndex << ": " << timesum.mTimesum << " ADC counts"; + return stream; +} \ No newline at end of file diff --git a/DataFormats/Detectors/EMCAL/src/DataFormatsEMCALLinkDef.h b/DataFormats/Detectors/EMCAL/src/DataFormatsEMCALLinkDef.h index 464b611d8b6f2..a648dd9fbbdc1 100644 --- a/DataFormats/Detectors/EMCAL/src/DataFormatsEMCALLinkDef.h +++ b/DataFormats/Detectors/EMCAL/src/DataFormatsEMCALLinkDef.h @@ -24,6 +24,11 @@ #pragma link C++ class o2::emcal::MCLabel + ; #pragma link C++ class o2::emcal::ErrorTypeFEE + ; +#pragma read \ + sourceClass = "o2::emcal::Cell" targetClass = "o2::emcal::Cell" source = "UShort_t mCellWords[3]" version = "[1]" include = "iostream" target = "mEnergy,mTimestamp,mTowerID,mChannelType" code = "{const char * oldwords = reinterpret_cast(onfile.mCellWords); mEnergy=o2::emcal::Cell::getEnergyFromPackedBitfieldV0(oldwords); mTimestamp=o2::emcal::Cell::getTimeFromPackedBitfieldV0(oldwords); mTowerID=o2::emcal::Cell::getTowerFromPackedBitfieldV0(oldwords); mChannelType=o2::emcal::Cell::getCellTypeFromPackedBitfieldV0(oldwords);}" +#pragma read \ + sourceClass = "o2::emcal::Cell" targetClass = "o2::emcal::Cell" source = "char mCellWords[6]" version = "[2]" include = "iostream" target = "mEnergy,mTimestamp,mTowerID,mChannelType" code = "{mEnergy=o2::emcal::Cell::getEnergyFromPackedBitfieldV0(onfile.mCellWords); mTimestamp=o2::emcal::Cell::getTimeFromPackedBitfieldV0(onfile.mCellWords); mTowerID=o2::emcal::Cell::getTowerFromPackedBitfieldV0(onfile.mCellWords); mChannelType=o2::emcal::Cell::getCellTypeFromPackedBitfieldV0(onfile.mCellWords);}" + #pragma link C++ class std::vector < o2::emcal::TriggerRecord> + ; #pragma link C++ class std::vector < o2::emcal::Cell> + ; #pragma link C++ class std::vector < o2::emcal::Digit> + ; @@ -37,8 +42,6 @@ // For channel type in digits and cells #pragma link C++ enum o2::emcal::ChannelType_t + ; -#pragma link C++ class std::vector < o2::emcal::Cluster> + ; - #pragma link C++ class o2::emcal::EventData < o2::emcal::Cell> + ; #pragma link C++ class o2::emcal::EventData < o2::emcal::Digit> + ; #pragma link C++ class o2::emcal::EventHandler < o2::emcal::Cell> + ; diff --git a/DataFormats/Detectors/EMCAL/src/Digit.cxx b/DataFormats/Detectors/EMCAL/src/Digit.cxx index 45ba800790e8f..a88cc652cdd6d 100644 --- a/DataFormats/Detectors/EMCAL/src/Digit.cxx +++ b/DataFormats/Detectors/EMCAL/src/Digit.cxx @@ -86,8 +86,8 @@ Int_t Digit::getAmplitudeADC(ChannelType_t ctype) const case ChannelType_t::TRU: { int ampADC = std::floor(mAmplitudeGeV / constants::EMCAL_TRU_ADCENERGY); // truncate energy in case dynamic range is saturated - if (ampADC >= constants::EMCAL_TRU_ADCENERGY) { - return constants::EMCAL_TRU_ADCENERGY; + if (ampADC >= constants::MAX_RANGE_ADC) { + return constants::MAX_RANGE_ADC; } return ampADC; }; diff --git a/DataFormats/Detectors/EMCAL/src/EMCALChannelData.cxx b/DataFormats/Detectors/EMCAL/src/EMCALChannelData.cxx deleted file mode 100644 index 8affa29259f7a..0000000000000 --- a/DataFormats/Detectors/EMCAL/src/EMCALChannelData.cxx +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// \file EMCALChannelData.cxx -/// \brief Class to store the data format for calibraton of the EMCal - -#include "DataFormatsEMCAL/EMCALChannelData.h" - -using namespace o2::dataformats; - -ClassImp(o2::dataformats::EMCALChannelData; diff --git a/DataFormats/Detectors/EMCAL/src/ErrorTypeFEE.cxx b/DataFormats/Detectors/EMCAL/src/ErrorTypeFEE.cxx index 53c0a0dc181cd..21d985b2a832a 100644 --- a/DataFormats/Detectors/EMCAL/src/ErrorTypeFEE.cxx +++ b/DataFormats/Detectors/EMCAL/src/ErrorTypeFEE.cxx @@ -34,9 +34,18 @@ void ErrorTypeFEE::PrintStream(std::ostream& stream) const case ErrorSource_t::GEOMETRY_ERROR: typestring = "geometry error"; break; - case ErrorTypeFEE::GAIN_ERROR: + case ErrorSource_t::GAIN_ERROR: typestring = "gain type error"; break; + case ErrorSource_t::TRU_ERROR: + typestring = "STU decoder error"; + break; + case ErrorSource_t::STU_ERROR: + typestring = "STU decoder error"; + break; + case ErrorSource_t::LINK_ERROR: + typestring = "Link missing error"; + break; case ErrorSource_t::UNDEFINED: typestring = "unknown error"; break; @@ -53,7 +62,63 @@ void ErrorTypeFEE::PrintStream(std::ostream& stream) const } } -std::ostream& operator<<(std::ostream& stream, const ErrorTypeFEE& error) +const char* ErrorTypeFEE::getErrorTypeName(unsigned int errorTypeID) +{ + switch (errorTypeID) { + case ErrorSource_t::PAGE_ERROR: + return "Page"; + case ErrorSource_t::ALTRO_ERROR: + return "MajorAltro"; + case ErrorSource_t::MINOR_ALTRO_ERROR: + return "MinorAltro"; + case ErrorSource_t::FIT_ERROR: + return "Fit"; + case ErrorSource_t::GEOMETRY_ERROR: + return "Geometry"; + case ErrorSource_t::GAIN_ERROR: + return "GainType"; + case ErrorSource_t::TRU_ERROR: + return "TRUDecoding"; + case ErrorSource_t::STU_ERROR: + return "STUDecoding"; + case ErrorSource_t::LINK_ERROR: + return "LinkMissing"; + case ErrorSource_t::UNDEFINED: + return "Undefined"; + default: + return ""; + }; +} + +const char* ErrorTypeFEE::getErrorTypeTitle(unsigned int errorTypeID) +{ + switch (errorTypeID) { + case ErrorSource_t::PAGE_ERROR: + return "Page"; + case ErrorSource_t::ALTRO_ERROR: + return "Major ALTRO"; + case ErrorSource_t::MINOR_ALTRO_ERROR: + return "Minor ALTRO"; + case ErrorSource_t::FIT_ERROR: + return "Fit"; + case ErrorSource_t::GEOMETRY_ERROR: + return "Geometry"; + case ErrorSource_t::GAIN_ERROR: + return "Gain"; + case ErrorSource_t::TRU_ERROR: + return "TRU Decoding"; + case ErrorSource_t::STU_ERROR: + return "STU Decoding"; + case ErrorSource_t::LINK_ERROR: + return "Link missing"; + case ErrorSource_t::UNDEFINED: + return "Unknown"; + default: + return ""; + }; +} + +std::ostream& o2::emcal::operator<<(std::ostream& stream, const ErrorTypeFEE& error) { error.PrintStream(stream); return stream; diff --git a/DataFormats/Detectors/EMCAL/src/EventHandler.cxx b/DataFormats/Detectors/EMCAL/src/EventHandler.cxx index de4e880d5bf05..79483f18c3635 100644 --- a/DataFormats/Detectors/EMCAL/src/EventHandler.cxx +++ b/DataFormats/Detectors/EMCAL/src/EventHandler.cxx @@ -138,6 +138,23 @@ const typename EventHandler::CellRange EventHandler +std::vector> EventHandler::getCellMCLabelForEvent(int eventID) const +{ + if (mCellLabels && mTriggerRecordsCells.size()) { + if (eventID >= mTriggerRecordsCells.size()) { + throw RangeException(eventID, mTriggerRecordsCells.size()); + } + auto& trgrecord = mTriggerRecordsCells[eventID]; + std::vector> eventlabels(trgrecord.getNumberOfObjects()); + for (int index = 0; index < trgrecord.getNumberOfObjects(); index++) { + eventlabels[index] = mCellLabels->getLabels(trgrecord.getFirstEntry() + index); + } + return eventlabels; + } + throw NotInitializedException(); +} + template const typename EventHandler::CellIndexRange EventHandler::getClusterCellIndicesForEvent(int eventID) const { @@ -160,6 +177,7 @@ void EventHandler::reset() mClusters = ClusterRange(); mClusterCellIndices = CellIndexRange(); mCells = CellRange(); + mCellLabels = nullptr; } template @@ -177,6 +195,9 @@ EventData EventHandler::buildEvent(int eventID) co if (hasCells()) { outputEvent.mCells = getCellsForEvent(eventID); } + if (mCellLabels) { + outputEvent.mMCCellLabels = getCellMCLabelForEvent(eventID); + } return outputEvent; } @@ -243,4 +264,4 @@ typename EventHandler::EventIterator EventHandler: } template class o2::emcal::EventHandler; -template class o2::emcal::EventHandler; \ No newline at end of file +template class o2::emcal::EventHandler; diff --git a/DataFormats/Detectors/EMCAL/src/TriggerRecord.cxx b/DataFormats/Detectors/EMCAL/src/TriggerRecord.cxx index f380b944f807d..7ca716dbb4ea2 100644 --- a/DataFormats/Detectors/EMCAL/src/TriggerRecord.cxx +++ b/DataFormats/Detectors/EMCAL/src/TriggerRecord.cxx @@ -13,6 +13,7 @@ #include #include "DataFormatsEMCAL/TriggerRecord.h" #include "CommonConstants/Triggers.h" +#include "DataFormatsEMCAL/Constants.h" namespace o2 { @@ -29,6 +30,9 @@ uint16_t TriggerRecord::getTriggerBitsCompressed() const if (mTriggerBits & o2::trigger::Cal) { result |= 1 << TriggerBitsCoded_t::CALIBTRIGGER; } + if (mTriggerBits & o2::emcal::triggerbits::Inc) { + result |= 1 << TriggerBitsCoded_t::REJECTINCOMPLETE; + } return result; } @@ -41,6 +45,9 @@ void TriggerRecord::setTriggerBitsCompressed(uint16_t triggerbits) if (triggerbits & (1 << TriggerBitsCoded_t::CALIBTRIGGER)) { mTriggerBits |= o2::trigger::Cal; } + if (triggerbits & (1 << TriggerBitsCoded_t::REJECTINCOMPLETE)) { + mTriggerBits |= o2::emcal::triggerbits::Inc; + } } void TriggerRecord::printStream(std::ostream& stream) const diff --git a/DataFormats/Detectors/FIT/FDD/CMakeLists.txt b/DataFormats/Detectors/FIT/FDD/CMakeLists.txt index fff07003f2700..140ba1165bff8 100644 --- a/DataFormats/Detectors/FIT/FDD/CMakeLists.txt +++ b/DataFormats/Detectors/FIT/FDD/CMakeLists.txt @@ -11,11 +11,14 @@ o2_add_library(DataFormatsFDD SOURCES src/RawEventData.cxx + src/RecPoint.cxx src/CTF.cxx + src/LookUpTable.cxx PUBLIC_LINK_LIBRARIES O2::FDDBase O2::DataFormatsFIT O2::SimulationDataFormat O2::CommonDataFormat + O2::DetectorsCommonDataFormats ) o2_target_root_dictionary(DataFormatsFDD @@ -27,3 +30,4 @@ o2_target_root_dictionary(DataFormatsFDD include/DataFormatsFDD/RawEventData.h include/DataFormatsFDD/LookUpTable.h include/DataFormatsFDD/CTF.h) + diff --git a/DataFormats/Detectors/FIT/FDD/include/DataFormatsFDD/CTF.h b/DataFormats/Detectors/FIT/FDD/include/DataFormatsFDD/CTF.h index 3c1df8d89ee4d..594f924e16598 100644 --- a/DataFormats/Detectors/FIT/FDD/include/DataFormatsFDD/CTF.h +++ b/DataFormats/Detectors/FIT/FDD/include/DataFormatsFDD/CTF.h @@ -41,8 +41,8 @@ struct CompressedDigits { // BC data std::vector trigger; // trigger bits - std::vector bcInc; // increment in BC if the same orbit, otherwise abs bc - std::vector orbitInc; // increment in orbit + std::vector bcInc; // increment in BC if the same orbit, otherwise abs bc + std::vector orbitInc; // increment in orbit std::vector nChan; // number of fired channels // channel data diff --git a/DataFormats/Detectors/FIT/FDD/include/DataFormatsFDD/ChannelData.h b/DataFormats/Detectors/FIT/FDD/include/DataFormatsFDD/ChannelData.h index 61bfbbd84c55a..7b46587bc0dc9 100644 --- a/DataFormats/Detectors/FIT/FDD/include/DataFormatsFDD/ChannelData.h +++ b/DataFormats/Detectors/FIT/FDD/include/DataFormatsFDD/ChannelData.h @@ -29,19 +29,23 @@ struct ChannelData { static constexpr char sChannelNameDPL[] = "DIGITSCH"; static constexpr char sDigitName[] = "ChannelData"; static constexpr char sDigitBranchName[] = "FDDDigitCh"; - uint8_t mPMNumber = -1; // PhotoMultiplier number (0 to 16) - int16_t mTime = -1024; // Time of Flight - int16_t mChargeADC = -1024; // ADC sample - uint8_t mFEEBits = 0; //Bit information from FEE - /* enum Flags { Integrator = 0x1 << 0, - DoubleEvent = 0x1 << 1, - Event1TimeLost = 0x1 << 2, - Event2TimeLost = 0x1 << 3, - AdcInGate = 0x1 << 4, - TimeTooLate = 0x1 << 5, - AmpTooHigh = 0x1 << 6, - EventInTrigger = 0x1 << 7, - TimeLost = 0x1 << 8 };*/ + static constexpr uint8_t DUMMY_CHANNEL_ID = 0xff; + static constexpr uint8_t DUMMY_CHAIN_QTC = 0; + static constexpr int16_t DUMMY_CFD_TIME = -5000; + static constexpr int16_t DUMMY_QTC_AMPL = -5000; + uint8_t mPMNumber = DUMMY_CHANNEL_ID; // PhotoMultiplier number (0 to 16) + int16_t mTime = DUMMY_CFD_TIME; // Time of Flight + int16_t mChargeADC = DUMMY_QTC_AMPL; // ADC sample + uint8_t mFEEBits = DUMMY_CHAIN_QTC; // Bit information from FEE + /* enum Flags { Integrator = 0x1 << 0, +DoubleEvent = 0x1 << 1, +Event1TimeLost = 0x1 << 2, +Event2TimeLost = 0x1 << 3, +AdcInGate = 0x1 << 4, +TimeTooLate = 0x1 << 5, +AmpTooHigh = 0x1 << 6, +EventInTrigger = 0x1 << 7, +TimeLost = 0x1 << 8 };*/ enum EEventDataBit { kNumberADC, kIsDoubleEvent, kIsTimeInfoNOTvalid, @@ -55,6 +59,9 @@ struct ChannelData { ChannelData() = default; ChannelData(uint8_t channel, int time, int adc, uint8_t bits) : mPMNumber(channel), mTime(time), mChargeADC(adc), mFEEBits(bits) {} uint8_t getChannelID() const { return mPMNumber; } + static void setFlag(EEventDataBit bitFlag, uint8_t& mFEEBits) { mFEEBits |= (1 << bitFlag); } + static void clearFlag(EEventDataBit bitFlag, uint8_t& mFEEBits) { mFEEBits &= ~(1 << bitFlag); } + bool getFlag(EEventDataBit bitFlag) const { return bool(mFEEBits & (1 << bitFlag)); } void print() const; bool operator==(ChannelData const& other) const { @@ -64,7 +71,7 @@ struct ChannelData { { LOG(info) << "ChId: " << static_cast(mPMNumber) << " | FEE bits:" << static_cast(mFEEBits) << " | Time: " << mTime << " | Charge: " << mChargeADC; } - ClassDefNV(ChannelData, 3); + ClassDefNV(ChannelData, 4); }; } // namespace fdd } // namespace o2 diff --git a/DataFormats/Detectors/FIT/FDD/include/DataFormatsFDD/LookUpTable.h b/DataFormats/Detectors/FIT/FDD/include/DataFormatsFDD/LookUpTable.h index 015eaa05390c2..ceac8e26fec7f 100644 --- a/DataFormats/Detectors/FIT/FDD/include/DataFormatsFDD/LookUpTable.h +++ b/DataFormats/Detectors/FIT/FDD/include/DataFormatsFDD/LookUpTable.h @@ -9,11 +9,6 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. // -//file RawEventData.h class for RAW data format -// Alla.Maevskaya -// simple look-up table just to feed digits 2 raw procedure. -//Will be really set after module/electronics connections -// #ifndef ALICEO2_FDD_LOOKUPTABLE_H_ #define ALICEO2_FDD_LOOKUPTABLE_H_ //////////////////////////////////////////////// @@ -21,219 +16,12 @@ ////////////////////////////////////////////// #include "DataFormatsFIT/LookUpTable.h" -#include -#include -#include -#include // std::setfill, std::setw - for stream formating -#include -#include "FDDBase/Constants.h" -#include "CommonUtils/NameConf.h" - +#include "DetectorsCommonDataFormats/DetID.h" namespace o2 { namespace fdd { - -struct Topo { - int modLink = 0; // Number of Processing Module, associated with GBT link ID - int modCh = 0; // Channel within the Processing Module in range from 0-11 - ClassDefNV(Topo, 1); -}; - -inline bool operator<(Topo const& a, Topo const& b) -{ - return (a.modLink < b.modLink || (a.modLink == b.modLink && a.modCh < b.modCh)); -} - -class LookUpTable -{ - public: - /// - /// Default constructor. - /// It must be kept public for root persistency purposes, - /// but should never be called by the outside world - LookUpTable() = default; - ~LookUpTable() = default; - - explicit LookUpTable(bool fillLinearly) - : mTopoVector(Nmodules * NChPerMod, {0, 0}), - mInvTopo(mTopoVector.size(), 0) - { - if (fillLinearly) { - LOG(info) << "Mapping of global channel and (PM, PM channel) pair"; - for (int link = 0; link < Nmodules; ++link) { - for (int ch = 0; ch < NChPerMod; ++ch) { - mTopoVector[link * NChPerMod + ch] = o2::fdd::Topo{link, ch}; - } - } - } else { - // TODO: If needed: implement more realistic splitting: 1 ring -> 1 PM instead of linear - LOG(warning) << "Don't use it - not implemented yet."; - } - - // Fill inverted LUT - matters only if LUT is not linear - for (size_t channel = 0; channel < mTopoVector.size(); ++channel) { - mInvTopo[getIdx(mTopoVector[channel].modLink, mTopoVector[channel].modCh)] = channel; - } - } - - int getChannel(int link, int mcp, int ep = 0) const { return mInvTopo[getIdx(link, mcp)]; } - int getLink(int channel) const { return mTopoVector[channel].modLink; } - int getModChannel(int channel) const { return mTopoVector[channel].modCh; } - int getTcmLink() const { return Nmodules; } - bool isTCM(int link, int ep) const { return link == 2 && ep == 0; } - Topo getTopoPM(int globalChannelID) const { return mTopoVector[globalChannelID]; } - Topo getTopoTCM() const { return Topo{getTcmLink(), 0}; } - std::size_t getNchannels() const { return mTopoVector.size(); } //get number of global PM channels - void printFullMap() const - { - LOG(info) << "o2::fdd::LookUpTable::printFullMap(): mTopoVector: [globalCh link modCh]"; - for (size_t channel = 0; channel < mTopoVector.size(); ++channel) { - LOG(info) << " " << channel << " " << mTopoVector[channel].modLink << " " << mTopoVector[channel].modCh; - } - LOG(info) << "o2::fdd::LookUpTable::printFullMap(): mInvTopo: [idx globalCh link modCh]"; - for (size_t idx = 0; idx < mInvTopo.size(); ++idx) { - LOG(info) << " " << idx << " " << mInvTopo[idx] << " " << getLinkFromIdx(mInvTopo[idx]) << " " << getModChannelFromIdx(mInvTopo[idx]); - } - } - - static o2::fdd::LookUpTable linear() - { - return o2::fdd::LookUpTable{1}; - } - - private: - std::vector mTopoVector; // iterator of each vector element gives the global channel number - std::vector mInvTopo; // each element is an iterator of mTopoVector - - static int getIdx(int link, int modCh) - { - assert(modCh < NChPerMod); - return link * NChPerMod + modCh; - } - static int getLinkFromIdx(int idx) { return idx / NChPerMod; } - static int getModChannelFromIdx(int idx) { return idx % NChPerMod; } - - ClassDefNV(LookUpTable, 1); -}; - -namespace deprecated -{ -//Singleton for LookUpTable -class SingleLUT : public LookUpTable -{ - private: - SingleLUT() : LookUpTable(LookUpTable::linear()) {} - SingleLUT(const SingleLUT&) = delete; - SingleLUT& operator=(SingleLUT&) = delete; - - public: - typedef Topo Topo_t; - static constexpr char sDetectorName[] = "FDD"; - static SingleLUT& Instance() - { - static SingleLUT instanceLUT; - return instanceLUT; - } - //Temporary - //Making topo for FEE recognizing(Local channelID is supressed) - static Topo_t makeGlobalTopo(const Topo_t& topo) - { - return Topo_t{topo.modLink, 0}; - } - static int getLocalChannelID(const Topo_t& topo) - { - return topo.modCh; - } - //Prepare full map for FEE metadata - template - auto makeMapFEEmetadata() -> std::map - { - std::map mapResult; - const uint16_t cruID = 0; //constant - const uint32_t endPointID = 0; //constant - uint64_t feeID = 0; //increments - //PM - for (int iCh = 0; iCh < Instance().getNchannels(); iCh++) { - auto pairInserted = mapResult.insert({makeGlobalTopo(Instance().getTopoPM(iCh)), RDHtype{}}); - if (pairInserted.second) { - auto& rdhObj = pairInserted.first->second; - const auto& topoObj = pairInserted.first->first; - if constexpr (std::is_same::value) { - rdhObj.linkID = topoObj.modLink; - rdhObj.endPointID = endPointID; - rdhObj.feeId = feeID; - rdhObj.cruID = cruID; - } else //Using RDHUtils - { - RDHhelper::setLinkID(&rdhObj, topoObj.modLink); - RDHhelper::setEndPointID(&rdhObj, endPointID); - RDHhelper::setFEEID(&rdhObj, feeID); - RDHhelper::setCRUID(&rdhObj, cruID); - } - feeID++; - } - } - //TCM - { - auto pairInserted = mapResult.insert({makeGlobalTopo(Instance().getTopoTCM()), RDHtype{}}); - if (pairInserted.second) { - auto& rdhObj = pairInserted.first->second; - const auto& topoObj = pairInserted.first->first; - if constexpr (std::is_same::value) { - rdhObj.linkID = topoObj.modLink; - rdhObj.endPointID = endPointID; - rdhObj.feeId = feeID; - rdhObj.cruID = cruID; - } else //Using RDHUtils - { - RDHhelper::setLinkID(&rdhObj, topoObj.modLink); - RDHhelper::setEndPointID(&rdhObj, endPointID); - RDHhelper::setFEEID(&rdhObj, feeID); - RDHhelper::setCRUID(&rdhObj, cruID); - } - } else { - LOG(info) << "WARNING! CHECK LUT! TCM METADATA IS INCORRECT!"; - } - } - assert(mapResult.size() > 0); - return mapResult; - } -}; -} // namespace deprecated -namespace new_lut -{ -//Singleton for LookUpTable -template -class SingleLUT : public LUT -{ - private: - SingleLUT(const std::string& ccdbPath, const std::string& ccdbPathToLUT) : LUT(ccdbPath, ccdbPathToLUT) {} - SingleLUT(const std::string& pathToFile) : LUT(pathToFile) {} - SingleLUT(const SingleLUT&) = delete; - SingleLUT& operator=(SingleLUT&) = delete; - - public: - static constexpr char sDetectorName[] = "FDD"; - static constexpr char sDefaultLUTpath[] = "FDD/Config/LookupTable"; - inline static std::string sCurrentCCDBpath = ""; - inline static std::string sCurrentLUTpath = sDefaultLUTpath; - //Before instance() call, setup url and path - static void setCCDBurl(const std::string& url) { sCurrentCCDBpath = url; } - static void setLUTpath(const std::string& path) { sCurrentLUTpath = path; } - static SingleLUT& Instance() - { - if (sCurrentCCDBpath == "") { - sCurrentCCDBpath = o2::base::NameConf::getCCDBServer(); - } - static SingleLUT instanceLUT(sCurrentCCDBpath, sCurrentLUTpath); - return instanceLUT; - } -}; -} //namespace new_lut - -using SingleLUT = new_lut::SingleLUT>; - +using SingleLUT = o2::fit::SingleLUT>; } // namespace fdd } // namespace o2 #endif diff --git a/DataFormats/Detectors/FIT/FDD/include/DataFormatsFDD/RawEventData.h b/DataFormats/Detectors/FIT/FDD/include/DataFormatsFDD/RawEventData.h index 8c9be56a94c68..9e0e7b4370aee 100644 --- a/DataFormats/Detectors/FIT/FDD/include/DataFormatsFDD/RawEventData.h +++ b/DataFormats/Detectors/FIT/FDD/include/DataFormatsFDD/RawEventData.h @@ -29,92 +29,10 @@ namespace o2 { namespace fdd { - using EventHeader = o2::fit::EventHeader; using EventData = o2::fit::EventData; using TCMdata = o2::fit::TCMdata; using TCMdataExtended = o2::fit::TCMdataExtended; -class RawEventData -{ - public: - RawEventData() = default; - EventHeader* getEventHeaderPtr() { return &mEventHeader; } - EventData* getEventDataPtr() { return mEventData; } - void print() const; - void printHexEventHeader() const; - void printHexEventData(uint64_t i) const; - enum EEventDataBit { kNumberADC, - kIsDoubleEvent, - kIs1TimeLostEvent, - kIs2TimeLostEvent, - kIsADCinGate, - kIsTimeInfoLate, - kIsAmpHigh, - kIsEventInTVDC, - kIsTimeInfoLost }; - const static int gStartDescriptor = 0x0000000f; - static const size_t sPayloadSizeSecondWord = 11; - static const size_t sPayloadSizeFirstWord = 5; - static const size_t sPayloadSize = 16; - int size() const - { - return 1 + mEventHeader.nGBTWords; // EventHeader + EventData size - } - - std::vector to_vector(bool tcm) - { - constexpr int CRUWordSize = 16; - - std::vector result(size() * CRUWordSize); - char* out = result.data(); - if (!tcm) { - std::memcpy(out, &mEventHeader, sPayloadSize); - out += sPayloadSize; - LOG(debug) << " Write PM header: nWords: " << (int)mEventHeader.nGBTWords - << " orbit: " << int(mEventHeader.orbit) - << " BC: " << int(mEventHeader.bc) - << " size: " << result.size(); - printHexEventHeader(); - out += CRUWordSize - sPayloadSize; // Padding enabled - - for (uint64_t i = 0; i < mEventHeader.nGBTWords; ++i) { - std::memcpy(out, &mEventData[2 * i], sPayloadSizeFirstWord); - out += sPayloadSizeFirstWord; - LOG(debug) << " 1st word: Ch: " << std::setw(2) << mEventData[2 * i].channelID - << " charge: " << std::setw(4) << mEventData[2 * i].charge - << " time: " << std::setw(4) << mEventData[2 * i].time; - std::memcpy(out, &mEventData[2 * i + 1], sPayloadSizeSecondWord); - out += sPayloadSizeSecondWord; - LOG(debug) << " 2nd word: Ch: " << std::setw(2) << mEventData[2 * i + 1].channelID - << " charge: " << std::setw(4) << mEventData[2 * i + 1].charge - << " time: " << std::setw(4) << mEventData[2 * i + 1].time; - - out += CRUWordSize - sPayloadSizeSecondWord - sPayloadSizeFirstWord; - printHexEventData(i); - } - } else { - // TCM data - std::memcpy(out, &mEventHeader, sPayloadSize); - out += sPayloadSize; - LOG(debug) << " Write TCM header: nWords: " << (int)mEventHeader.nGBTWords - << " orbit: " << int(mEventHeader.orbit) - << " BC: " << int(mEventHeader.bc) - << " size: " << result.size(); - std::memcpy(out, &mTCMdata, sizeof(TCMdata)); - out += sizeof(TCMdata); - // TODO: No TCM payload printing until the meaning of trigger bits and other flags is clarified - } - return result; - } - - public: - EventHeader mEventHeader; //! - EventData mEventData[NChPerMod]; //! - TCMdata mTCMdata; //! - - ClassDefNV(RawEventData, 1); -}; - } // namespace fdd } // namespace o2 #endif diff --git a/DataFormats/Detectors/FIT/FDD/include/DataFormatsFDD/RecPoint.h b/DataFormats/Detectors/FIT/FDD/include/DataFormatsFDD/RecPoint.h index ddd5344c9cea9..f784d99145728 100644 --- a/DataFormats/Detectors/FIT/FDD/include/DataFormatsFDD/RecPoint.h +++ b/DataFormats/Detectors/FIT/FDD/include/DataFormatsFDD/RecPoint.h @@ -9,8 +9,9 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. -/// \file RecPoint.h +/// \file RecPoint.h /// \brief Definition of the FDD RecPoint class + #ifndef ALICEO2_FDD_RECPOINT_H #define ALICEO2_FDD_RECPOINT_H @@ -42,6 +43,7 @@ struct ChannelDataFloat { } void print() const; + bool operator==(const ChannelDataFloat&) const = default; ClassDefNV(ChannelDataFloat, 1); }; @@ -77,6 +79,11 @@ class RecPoint return mRef.getEntries() ? gsl::span(tfdata).subspan(mRef.getFirstEntry(), mRef.getEntries()) : gsl::span(); } short static constexpr sDummyCollissionTime = 32767; + int getFirstEntry() const { return mRef.getFirstEntry(); } + int getEntriesInCurrentBC() const { return mRef.getEntries(); } + + void print() const; + bool operator==(const RecPoint&) const = default; private: o2::dataformats::RangeReference mRef; diff --git a/DataFormats/Detectors/FIT/FDD/src/DataFormatsFDDLinkDef.h b/DataFormats/Detectors/FIT/FDD/src/DataFormatsFDDLinkDef.h index fb43b97bd3f2c..daf1fb4027dc9 100644 --- a/DataFormats/Detectors/FIT/FDD/src/DataFormatsFDDLinkDef.h +++ b/DataFormats/Detectors/FIT/FDD/src/DataFormatsFDDLinkDef.h @@ -33,12 +33,6 @@ #pragma link C++ class o2::fdd::ChannelDataFloat + ; #pragma link C++ class vector < o2::fdd::ChannelDataFloat> + ; -#pragma link C++ class o2::fdd::RawEventData + ; -#pragma link C++ class o2::fdd::EventHeader + ; -#pragma link C++ class o2::fdd::EventData + ; -#pragma link C++ class o2::fdd::TCMdata + ; -#pragma link C++ class o2::fdd::Topo + ; - #pragma link C++ class o2::fdd::CTFHeader + ; #pragma link C++ class o2::fdd::CTF + ; #pragma link C++ class o2::ctf::EncodedBlocks < o2::fdd::CTFHeader, 8, uint32_t> + ; diff --git a/DataFormats/Detectors/FIT/FDD/src/LookUpTable.cxx b/DataFormats/Detectors/FIT/FDD/src/LookUpTable.cxx new file mode 100644 index 0000000000000..7965b73d6aa95 --- /dev/null +++ b/DataFormats/Detectors/FIT/FDD/src/LookUpTable.cxx @@ -0,0 +1,15 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include "DataFormatsFDD/LookUpTable.h" + +using namespace o2::fdd; +template class o2::fit::SingleLUT>; diff --git a/DataFormats/Detectors/FIT/FDD/src/RawEventData.cxx b/DataFormats/Detectors/FIT/FDD/src/RawEventData.cxx index a8b1af4ae4653..c533f56839567 100644 --- a/DataFormats/Detectors/FIT/FDD/src/RawEventData.cxx +++ b/DataFormats/Detectors/FIT/FDD/src/RawEventData.cxx @@ -14,30 +14,3 @@ #include using namespace o2::fdd; -void RawEventData::printHexEventHeader() const -{ - std::stringstream ssheader; - ssheader << std::setfill('0') << std::setw(16) << std::hex << mEventHeader.word[0] << " " << std::setw(16) << mEventHeader.word[1] << "\n "; - ssheader << std::setw(3) << (0x00000fff & mEventHeader.bc) << " " - << std::setw(8) << (0xffffffff & mEventHeader.orbit) << " " - << std::setw(5) << (0x000fffff & mEventHeader.reservedField1) << " " - << std::setw(2) << (0x000000ff & mEventHeader.reservedField2) << " " - << std::setw(1) << (0x0000000f & mEventHeader.nGBTWords) << " " - << std::setw(1) << (0x0000000f & mEventHeader.startDescriptor) << " " - << std::setw(12) << (0xffffffffffff & mEventHeader.reservedField3); - LOG(debug) << ssheader.str(); -} - -void RawEventData::printHexEventData(uint64_t i) const -{ - std::stringstream ssdata; - ssdata << "D0:0x "; - ssdata << std::setfill('0') << std::hex << std::setw(16) << mEventData[2 * i].word << "\n "; - ssdata << std::setw(3) << (0x0fff & mEventData[2 * i].time) << " " - << std::setw(8) << (0x1fff & mEventData[2 * i].charge) << "\n "; - ssdata << "D1:0x "; - ssdata << std::setfill('0') << std::hex << std::setw(16) << mEventData[2 * i + 1].word << "\n "; - ssdata << std::setw(3) << (0x0fff & mEventData[2 * i + 1].time) << " " - << std::setw(8) << (0x1fff & mEventData[2 * i + 1].charge); - LOG(debug) << " | " << ssdata.str(); -} diff --git a/DataFormats/Detectors/FIT/FDD/src/RecPoint.cxx b/DataFormats/Detectors/FIT/FDD/src/RecPoint.cxx new file mode 100644 index 0000000000000..854a09088a2f4 --- /dev/null +++ b/DataFormats/Detectors/FIT/FDD/src/RecPoint.cxx @@ -0,0 +1,33 @@ +// Copyright 2019-2024 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file RecPoint.cxx +/// \brief Implementation of the FDD RecPoint class +/// \author Andreas Molander andreas.molander@cern.ch + +#include "DataFormatsFDD/RecPoint.h" +#include "Framework/Logger.h" + +using namespace o2::fdd; + +void ChannelDataFloat::print() const +{ + LOG(info) << "ChannelDataFloat data:"; + LOG(info) << "Channel ID: " << mPMNumber << ", Time (ps): " << mTime << ", Charge (ADC): " << mChargeADC << ", QTC chain: " << adcId; +} + +void RecPoint::print() const +{ + LOG(info) << "RecPoint data:"; + LOG(info) << "Collision times: A: " << getCollisionTimeA() << ", C: " << getCollisionTimeC(); + LOG(info) << "Ref first: " << mRef.getFirstEntry() << ", Ref entries: " << mRef.getEntries(); + LOG(info) << "Triggers: " << mTriggers.print(); +} diff --git a/DataFormats/Detectors/FIT/FT0/CMakeLists.txt b/DataFormats/Detectors/FIT/FT0/CMakeLists.txt index 7062e1bf754b5..f7d6a111f4348 100644 --- a/DataFormats/Detectors/FIT/FT0/CMakeLists.txt +++ b/DataFormats/Detectors/FIT/FT0/CMakeLists.txt @@ -13,21 +13,27 @@ o2_add_library(DataFormatsFT0 SOURCES src/ChannelData.cxx src/Digit.cxx src/DigitFilterParam.cxx + src/CalibParam.cxx src/RecPoints.cxx src/RawEventData.cxx src/CTF.cxx + src/LookUpTable.cxx + src/SlewingCoef.cxx + PUBLIC_LINK_LIBRARIES O2::FT0Base O2::DataFormatsFIT O2::SimulationDataFormat O2::CommonDataFormat O2::Headers O2::CCDB + O2::DetectorsCommonDataFormats ) o2_target_root_dictionary(DataFormatsFT0 HEADERS include/DataFormatsFT0/ChannelData.h include/DataFormatsFT0/Digit.h include/DataFormatsFT0/DigitFilterParam.h + include/DataFormatsFT0/CalibParam.h include/DataFormatsFT0/MCLabel.h include/DataFormatsFT0/HitType.h include/DataFormatsFT0/RecPoints.h @@ -39,4 +45,7 @@ o2_target_root_dictionary(DataFormatsFT0 include/DataFormatsFT0/FT0ChannelTimeCalibrationObject.h include/DataFormatsFT0/GlobalOffsetsInfoObject.h include/DataFormatsFT0/GlobalOffsetsCalibrationObject.h + include/DataFormatsFT0/SpectraInfoObject.h + include/DataFormatsFT0/SlewingCoef.h + include/DataFormatsFT0/EventsPerBc.h ) diff --git a/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/CTF.h b/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/CTF.h index fdc073edfd5c1..3ddea05158577 100644 --- a/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/CTF.h +++ b/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/CTF.h @@ -41,8 +41,8 @@ struct CompressedDigits { // trigger data std::vector trigger; // trigger bits - std::vector bcInc; // increment in BC if the same orbit, otherwise abs bc - std::vector orbitInc; // increment in orbit + std::vector bcInc; // increment in BC if the same orbit, otherwise abs bc + std::vector orbitInc; // increment in orbit std::vector nChan; // number of fired channels std::vector eventStatus; // special flags about event conditions: pile-up, not use for collision time, not use for event plane, etc. diff --git a/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/CalibParam.h b/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/CalibParam.h new file mode 100644 index 0000000000000..f5dab5051bfb0 --- /dev/null +++ b/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/CalibParam.h @@ -0,0 +1,49 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef ALICEO2_CALIB_PARAM_H_ +#define ALICEO2_CALIB_PARAM_H_ + +#include "CommonUtils/ConfigurableParamHelper.h" +#include "FT0Base/Geometry.h" + +#include + +namespace o2 +{ +namespace ft0 +{ + +struct CalibParam : o2::conf::ConfigurableParamHelper { + static constexpr auto Nchannels = o2::ft0::Geometry::Nchannels; + // Logic for obtaining bad channels and for making decision concerning slot finalization + std::size_t mMinEntriesThreshold = 500; // Min number of entries + std::size_t mMaxEntriesThreshold = 1000; // Max number of entries + uint8_t mNExtraSlots = 0; // Number of extra slots + // Fitting ranges + double mMinFitRange = -200.; // Min fit range + double mMaxFitRange = 200.; // Max fit range + // Conditions for checking fit quality, otherwise hist mean will be taken as offset + double mMaxDiffMean = 5; // Max differnce between mean and fit result + double mMinRMS = 3; // Min RMS + double mMaxSigma = 32; // Max fit sigma + int mRebinFactorPerChID[Nchannels] = {0}; //[Nchannels] + // + bool mUseDynamicRange = true; // use dynamic ranges [mean-RMS*mRangeInRMS,mean+RMS*mRangeInRMS] for fitting + double mRangeInRMS = 1.5; // Range for RMS in dynamic case + + O2ParamDef(CalibParam, "FT0CalibParam"); +}; + +} // end namespace ft0 +} // end namespace o2 + +#endif /* FT0_CALIB_PARAM_H_ */ \ No newline at end of file diff --git a/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/ChannelData.h b/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/ChannelData.h index 96e99117110d5..9b3d6ec805604 100644 --- a/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/ChannelData.h +++ b/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/ChannelData.h @@ -10,11 +10,11 @@ // or submit itself to any jurisdiction. /// \file ChannelData.h -/// \brief Class to describe fired and stored channels for the BC and to refer to channel data +/// \brief Class to describe fired and stored channels for the BC and to refer to channel data /// \author Alla.Maevskaya@cern.ch -#ifndef _FT0_CHANNELDATA_H_ -#define _FT0_CHANNELDATA_H_ +#ifndef O2_FT0_CHANNELDATA_H_ +#define O2_FT0_CHANNELDATA_H_ #include #include @@ -30,9 +30,9 @@ struct ChannelData { static constexpr uint8_t DUMMY_CHAIN_QTC = 0xff; static constexpr int16_t DUMMY_CFD_TIME = -5000; static constexpr int16_t DUMMY_QTC_AMPL = -5000; - uint8_t ChId = DUMMY_CHANNEL_ID; //channel Id - uint8_t ChainQTC = DUMMY_CHAIN_QTC; //QTC chain - int16_t CFDTime = DUMMY_CFD_TIME; //time in #CFD channels, 0 at the LHC clk center + uint8_t ChId = DUMMY_CHANNEL_ID; // channel Id + uint8_t ChainQTC = DUMMY_CHAIN_QTC; // QTC chain + int16_t CFDTime = DUMMY_CFD_TIME; // time in #CFD channels, 0 at the LHC clk center int16_t QTCAmpl = DUMMY_QTC_AMPL; // Amplitude #channels enum EEventDataBit { kNumberADC, kIsDoubleEvent, @@ -60,7 +60,19 @@ struct ChannelData { { ChainQTC |= (value << bitFlag); } + static void setFlag(EEventDataBit bitFlag, uint8_t& chainQTC) { chainQTC |= (1 << bitFlag); } + static void clearFlag(EEventDataBit bitFlag, uint8_t& chainQTC) { chainQTC &= ~(1 << bitFlag); } bool getFlag(EEventDataBit bitFlag) const { return bool(ChainQTC & (1 << bitFlag)); } + bool areAllFlagsGood() const + { + return (!getFlag(ChannelData::kIsDoubleEvent) && + !getFlag(ChannelData::kIsTimeInfoNOTvalid) && + getFlag(ChannelData::kIsCFDinADCgate) && + !getFlag(ChannelData::kIsTimeInfoLate) && + !getFlag(ChannelData::kIsAmpHigh) && + getFlag(ChannelData::kIsEventInTVDC) && + !getFlag(ChannelData::kIsTimeInfoLost)); + } void print() const; void printLog() const; [[nodiscard]] uint8_t getChannelID() const { return ChId; } diff --git a/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/Digit.h b/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/Digit.h index 2b7758eeac1f5..c4051fd5b1a7d 100644 --- a/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/Digit.h +++ b/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/Digit.h @@ -50,6 +50,7 @@ struct DetTrigInput { (isSCnt << Triggers::bitSCen)) { } + bool isVertex() const { return mInputs.test(Triggers::bitVertex); } ClassDefNV(DetTrigInput, 1); }; diff --git a/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/DigitFilterParam.h b/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/DigitFilterParam.h index 19f1d43bd35e6..a00abc013c27d 100644 --- a/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/DigitFilterParam.h +++ b/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/DigitFilterParam.h @@ -27,11 +27,77 @@ struct DigitFilterParam : o2::conf::ConfigurableParamHelper { uint8_t mPMbitsGood = (1 << ChannelData::EEventDataBit::kIsCFDinADCgate) | (1 << ChannelData::EEventDataBit::kIsEventInTVDC); uint8_t mPMbitsBad = (1 << ChannelData::EEventDataBit::kIsDoubleEvent) | (1 << ChannelData::EEventDataBit::kIsTimeInfoNOTvalid) | (1 << ChannelData::EEventDataBit::kIsTimeInfoLate) | (1 << ChannelData::EEventDataBit::kIsAmpHigh) | (1 << ChannelData::EEventDataBit::kIsTimeInfoLost); uint8_t mPMbitsToCheck = mPMbitsGood | mPMbitsBad; - uint8_t mTrgBitsGood = (1 << Triggers::bitVertex) | (1 << Triggers::bitDataIsValid); - uint8_t mTrgBitsBad = (1 << Triggers::bitOutputsAreBlocked); - uint8_t mTrgBitsToCheck = mTrgBitsGood | mTrgBitsBad; + uint64_t mTrgBitsGood = Triggers::word(Triggers::bitVertex, Triggers::bitDataIsValid); + uint64_t mTrgBitsBad = Triggers::word(Triggers::bitOutputsAreBlocked); + uint64_t mTrgBitsToCheck = mTrgBitsGood | mTrgBitsBad; O2ParamDef(DigitFilterParam, "FT0DigitFilterParam"); }; + +struct ChannelFilterParam : o2::conf::ConfigurableParamHelper { + int16_t mAmpUpper = 4200; + int16_t mAmpLower = -4200; + int16_t mTimeUpper = 2050; + int16_t mTimeLower = -2050; + + uint8_t mPMbitsGood = 0; + uint8_t mPMbitsBad = 0; // no checking for bad bits + uint8_t mPMbitsToCheck = mPMbitsGood | mPMbitsBad; + + uint64_t mTrgBitsGood = 0; + uint64_t mTrgBitsBad = 0; // Laser haven't been used in 2022, no check for bad bits + uint64_t mTrgBitsToCheck = mTrgBitsGood | mTrgBitsBad; + + bool checkPMbits(uint8_t pmBits) const + { + return (pmBits & mPMbitsToCheck) == mPMbitsGood; + } + bool checkTCMbits(uint8_t tcmBits) const + { + return (tcmBits & mTrgBitsToCheck) == mTrgBitsGood; + } + bool checkTimeWindow(int16_t time) const + { + return time >= mTimeLower && time <= mTimeUpper; + } + bool checkAmpWindow(int16_t amp) const + { + return amp >= mAmpLower && amp <= mAmpUpper; + } + bool checkAll(const o2::ft0::ChannelData& channelData) const + { + return checkPMbits(channelData.ChainQTC) && checkAmpWindow(channelData.QTCAmpl) && checkTimeWindow(channelData.CFDTime); + } + O2ParamDef(ChannelFilterParam, "FT0ChannelFilterParam"); +}; + +struct TimeFilterParam : o2::conf::ConfigurableParamHelper { + int16_t mAmpUpper = 4200; + int16_t mAmpLower = 10; + int16_t mTimeUpper = 153; + int16_t mTimeLower = -153; + + uint8_t mPMbitsGood = 0; // No need in checking good PM bits + uint8_t mPMbitsBad = (1 << ChannelData::EEventDataBit::kIsTimeInfoNOTvalid) | (1 << ChannelData::EEventDataBit::kIsTimeInfoLost); // Check only two bad PM bits + uint8_t mPMbitsToCheck = mPMbitsGood | mPMbitsBad; + bool checkPMbits(uint8_t pmBits) const + { + return (pmBits & mPMbitsToCheck) == mPMbitsGood; + } + bool checkTimeWindow(int16_t time) const + { + return time >= mTimeLower && time <= mTimeUpper; + } + bool checkAmpWindow(int16_t amp) const + { + return amp >= mAmpLower && amp <= mAmpUpper; + } + bool checkAll(const o2::ft0::ChannelData& channelData) const + { + return checkPMbits(channelData.ChainQTC) && checkAmpWindow(channelData.QTCAmpl) && checkTimeWindow(channelData.CFDTime); + } + O2ParamDef(TimeFilterParam, "FT0TimeFilterParam"); +}; + } // namespace o2::ft0 #endif \ No newline at end of file diff --git a/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/EventsPerBc.h b/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/EventsPerBc.h new file mode 100644 index 0000000000000..9fcd1318914bd --- /dev/null +++ b/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/EventsPerBc.h @@ -0,0 +1,25 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef _FT0_EVENTS_PER_BC_CALIB_OBJECT +#define _FT0_EVENTS_PER_BC_CALIB_OBJECT + +#include "CommonConstants/LHCConstants.h" +#include + +namespace o2::ft0 +{ +struct EventsPerBc { + std::array histogram; + ClassDefNV(EventsPerBc, 1); +}; +} // namespace o2::ft0 +#endif \ No newline at end of file diff --git a/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/HitType.h b/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/HitType.h index 6654561ee33a0..dfb4a7e6d724f 100644 --- a/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/HitType.h +++ b/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/HitType.h @@ -16,7 +16,6 @@ #define ALICEO2_FIT_HITTYPE_H_ #include "SimulationDataFormat/BaseHits.h" -#include "SimulationDataFormat/Stack.h" #include "CommonUtils/ShmAllocator.h" namespace o2 diff --git a/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/LookUpTable.h b/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/LookUpTable.h index 70aba053698a5..1d31b079c2ac0 100644 --- a/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/LookUpTable.h +++ b/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/LookUpTable.h @@ -9,73 +9,18 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. // -// file RawEventData.h class for RAW data format -// Alla.Maevskaya -// simple look-up table just to feed digits 2 raw procedure. -// Will be really set after module/electronics connections -// #ifndef ALICEO2_FT0_LOOKUPTABLE_H_ #define ALICEO2_FT0_LOOKUPTABLE_H_ //////////////////////////////////////////////// // Look Up Table FT0 ////////////////////////////////////////////// - -#include "CCDB/BasicCCDBManager.h" -#include "FT0Base/Constants.h" #include "DataFormatsFIT/LookUpTable.h" -#include "CommonUtils/NameConf.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - +#include "DetectorsCommonDataFormats/DetID.h" namespace o2 { namespace ft0 { - -namespace new_lut -{ -//Singleton for LookUpTable -template -class SingleLUT : public LUT -{ - private: - SingleLUT(const std::string& ccdbPath, const std::string& ccdbPathToLUT) : LUT(ccdbPath, ccdbPathToLUT) {} - SingleLUT(const std::string& pathToFile) : LUT(pathToFile) {} - SingleLUT(const SingleLUT&) = delete; - SingleLUT& operator=(SingleLUT&) = delete; - - public: - static constexpr char sDetectorName[] = "FT0"; - static constexpr char sDefaultLUTpath[] = "FT0/Config/LookupTable"; - inline static std::string sCurrentCCDBpath = ""; - inline static std::string sCurrentLUTpath = sDefaultLUTpath; - //Before instance() call, setup url and path - static void setCCDBurl(const std::string& url) { sCurrentCCDBpath = url; } - static void setLUTpath(const std::string& path) { sCurrentLUTpath = path; } - static SingleLUT& Instance() - { - if (sCurrentCCDBpath == "") { - sCurrentCCDBpath = o2::base::NameConf::getCCDBServer(); - } - static SingleLUT instanceLUT(sCurrentCCDBpath, sCurrentLUTpath); - return instanceLUT; - } -}; -} //namespace new_lut - -using SingleLUT = new_lut::SingleLUT>; - +using SingleLUT = o2::fit::SingleLUT>; } // namespace ft0 } // namespace o2 #endif diff --git a/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/RawEventData.h b/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/RawEventData.h index 4bc2051bf1bf5..8bb19f63acd91 100644 --- a/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/RawEventData.h +++ b/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/RawEventData.h @@ -9,8 +9,8 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. // -//file RawEventData.h class for RAW data format -//Alla.Maevskaya@cern.ch +// file RawEventData.h class for RAW data format +// Alla.Maevskaya@cern.ch // with Artur.Furs // #ifndef ALICEO2_FT0_RAWEVENTDATA_H_ @@ -32,136 +32,11 @@ namespace o2 { namespace ft0 { -//constexpr int Nchannels_FT0 = o2::ft0::Geometry::Nchannels; -static constexpr int Nchannels_FT0 = o2::ft0::Constants::sNCHANNELS_PM; using EventHeader = o2::fit::EventHeader; using EventData = o2::fit::EventData; using TCMdata = o2::fit::TCMdata; using TCMdataExtended = o2::fit::TCMdataExtended; -class RawEventData -{ - public: - RawEventData() = default; - void print() const; - const static int gStartDescriptor = 0x0000000f; - static const size_t sPayloadSizeSecondWord = 11; - static const size_t sPayloadSizeFirstWord = 5; - static constexpr size_t sPayloadSize = 16; - int size() const - { - return 1 + mEventHeader.nGBTWords; // EventHeader + EventData size - } - - std::vector to_vector(bool tcm) - { - constexpr int CRUWordSize = 16; - const char padding[CRUWordSize] = {0}; - - std::vector result(size() * CRUWordSize); - char* out = result.data(); - if (!tcm) { - std::memcpy(out, &mEventHeader, sPayloadSize); - out += sPayloadSize; - LOG(debug) << "write header words " << (int)mEventHeader.nGBTWords << " orbit " << int(mEventHeader.orbit) << " bc " << int(mEventHeader.bc) << " out " << result.size(); - if (mIsPadded) { - out += CRUWordSize - sPayloadSize; - } - for (int i = 0; i < mEventHeader.nGBTWords; ++i) { - std::memcpy(out, &mEventData[2 * i], sPayloadSizeFirstWord); - LOG(debug) << " 1st word " << mEventData[2 * i].channelID << " charge " << mEventData[2 * i].charge << " time " << mEventData[2 * i].time << " out " << result.size(); - out += sPayloadSizeFirstWord; - std::memcpy(out, &mEventData[2 * i + 1], sPayloadSizeSecondWord); - out += sPayloadSizeSecondWord; - LOG(debug) << " 2nd word " << mEventData[2 * i + 1].channelID << " charge " << mEventData[2 * i + 1].charge << " time " << mEventData[2 * i + 1].time << " out " << result.size(); - if (mIsPadded) { - out += CRUWordSize - sPayloadSizeSecondWord - sPayloadSizeFirstWord; - } - } - } else { - // TCM data - std::memcpy(out, &mEventHeader, sPayloadSize); - out += sPayloadSize; - LOG(debug) << "write TCM header words " << (int)mEventHeader.nGBTWords << " orbit " << int(mEventHeader.orbit) << " bc " << int(mEventHeader.bc) << " out " << result.size(); - std::memcpy(out, &mTCMdata, sizeof(TCMdata)); - out += sizeof(TCMdata); - LOG(debug) << "write TCM words " << sizeof(mTCMdata) << " orbit " << int(mEventHeader.orbit) << " bc " << int(mEventHeader.bc) << " out " << result.size() << " sum time A " << mTCMdata.timeA; - } - - return result; - } - void setIsPadded(bool isPadding128) - { - mIsPadded = isPadding128; - } - - public: - EventHeader mEventHeader; //! - EventData mEventData[o2::ft0::Constants::sNCHANNELS_PER_PM]; //! - TCMdata mTCMdata; //! - bool mIsPadded = true; - ///////////////////////////////////////////////// - ClassDefNV(RawEventData, 2); -}; -std::ostream& operator<<(std::ostream& stream, const RawEventData& data); - -class DataPageWriter -{ - std::vector mBuffer; - int mNpacketsInBuffer = 0; - std::vector> mPages; - std::vector mNpackets; - static constexpr int MAX_Page_size = 8192; - - public: - o2::header::RAWDataHeader mRDH; - void flush(std::ostream& str) - { - writePage(); - mRDH.stop = 0; - for (int page = 0; page < int(mPages.size()); ++page) { - mRDH.memorySize = mPages[page].size() + mRDH.headerSize; - mRDH.offsetToNext = mRDH.memorySize; - mRDH.packetCounter = mNpackets[page]; - str.write(reinterpret_cast(&mRDH), sizeof(mRDH)); - str.write(mPages[page].data(), mPages[page].size()); - mRDH.pageCnt++; - LOG(info) << " header " << mRDH.linkID << " end " << mRDH.endPointID; - } - if (!mPages.empty()) { - mRDH.memorySize = mRDH.headerSize; - mRDH.offsetToNext = mRDH.memorySize; - mRDH.stop = 1; - mRDH.pageCnt++; - str.write(reinterpret_cast(&mRDH), sizeof(mRDH)); - mPages.clear(); - mNpackets.clear(); - } - } - - void writePage() - { - if (mBuffer.size() == 0) { - return; - } - mPages.emplace_back(std::move(mBuffer)); - LOG(debug) << " writePage " << mBuffer.size(); - mNpackets.push_back(mNpacketsInBuffer); - mNpacketsInBuffer = 0; - mBuffer.clear(); - } - void write(std::vector const& new_data) - { - if (mBuffer.size() + new_data.size() + mRDH.headerSize > MAX_Page_size) { - LOG(debug) << " write rest " << mBuffer.size() << " " << new_data.size() << " " << mRDH.headerSize; - writePage(); - } - LOG(debug) << " write vector " << new_data.size() << " buffer " << mBuffer.size() << " RDH " << mRDH.headerSize << " new data " << new_data.data(); - mBuffer.insert(mBuffer.end(), new_data.begin(), new_data.end()); - mNpacketsInBuffer++; - LOG(debug) << " write vector end mBuffer.size " << mBuffer.size() << " mNpacketsInBuffer " << mNpacketsInBuffer << " newdtata " << new_data.size(); - } -}; } // namespace ft0 } // namespace o2 #endif diff --git a/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/RecPoints.h b/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/RecPoints.h index 1178cc20a4da0..0503e4f39948f 100644 --- a/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/RecPoints.h +++ b/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/RecPoints.h @@ -24,7 +24,9 @@ #include "Rtypes.h" #include #include - +#include +#include +#include namespace o2 { namespace ft0 @@ -32,10 +34,10 @@ namespace ft0 struct ChannelDataFloat { - int ChId = -1; //channel Id - int ChainQTC = -1; //QTC chain - float CFDTime = -20000; //time in ps, 0 at the LHC clk center - float QTCAmpl = -20000; // Amplitude mV + int ChId = -1; // channel Id + int ChainQTC = -1; // QTC chain + float CFDTime = -20000; // time in ps, 0 at the LHC clk center + float QTCAmpl = -20000; // Amplitude mV ChannelDataFloat() = default; ChannelDataFloat(int iPmt, float time, float charge, int chainQTC) @@ -47,6 +49,7 @@ struct ChannelDataFloat { } void print() const; + bool operator==(const ChannelDataFloat&) const = default; ClassDefNV(ChannelDataFloat, 1); }; @@ -55,10 +58,39 @@ class RecPoints { public: - enum : int { TimeMean, - TimeA, - TimeC, - Vertex }; + enum ETimeType { kTimeMean, + kTimeA, + kTimeC, + kTimeVertex }; + + // Enum for trigger nits specified in rec-points and AOD data + enum ETriggerBits { kOrA = 0, // OrA time-trigger signal + kOrC = 1, // OrC time-trigger signal + kSemiCentral = 2, // Semi-central amplitude-trigger signal + kCentral = 3, // Central amplitude-trigger signal + kVertex = 4, // Vertex time-trigger signal + kIsActiveSideA = 5, // Side-A has at least one channel active + kIsActiveSideC = 6, // Side-C has at least one channel active + kIsFlangeEvent = 7 // Flange event at Side-C, at least one channel has time which corresponds to -82 cm area + }; + static const inline std::map sMapTriggerBits = { + {ETriggerBits::kOrA, "OrA"}, + {ETriggerBits::kOrC, "OrC"}, + {ETriggerBits::kSemiCentral, "Semicentral"}, + {ETriggerBits::kCentral, "Central"}, + {ETriggerBits::kVertex, "Vertex"}, + {ETriggerBits::kIsActiveSideA, "IsActiveSideA"}, + {ETriggerBits::kIsActiveSideC, "IsActiveSideC"}, + {ETriggerBits::kIsFlangeEvent, "IsFlangeEvent"}}; + + enum ETechnicalBits { kLaser = 0, // indicates the laser was triggered in this BC + kOutputsAreBlocked = 1, // indicates that laser-induced pulses should arrive from detector to FEE in this BC (and trigger outputs are blocked) + kDataIsValid = 2, // data is valid for processing + }; + static const inline std::map sMapTechnicalBits = { + {ETechnicalBits::kLaser, "Laser"}, + {ETechnicalBits::kOutputsAreBlocked, "OutputsAreBlocked"}, + {ETechnicalBits::kDataIsValid, "DataIsValid"}}; o2::dataformats::RangeReference ref; o2::InteractionRecord mIntRecord; // Interaction record (orbit, bc) @@ -72,38 +104,63 @@ class RecPoints mIntRecord = iRec; mTriggers = chTrig; } - ~RecPoints() = default; + RecPoints(int chDataFirstEntryPos, + int chDataNEntries, + const o2::InteractionRecord& ir, + const std::array& arrTimes, + const o2::fit::Triggers& digitTriggers, + uint8_t extraTriggerWord) : mIntRecord(ir), mCollisionTime(arrTimes), mTriggers(digitTriggers) + { + ref.setFirstEntry(chDataFirstEntryPos); + ref.setEntries(chDataNEntries); + initRecPointTriggers(digitTriggers, extraTriggerWord); + } - void print() const; + ~RecPoints() = default; short getCollisionTime(int side) const { return mCollisionTime[side]; } - short getCollisionTimeMean() const { return getCollisionTime(TimeMean); } - short getCollisionTimeA() const { return getCollisionTime(TimeA); } - short getCollisionTimeC() const { return getCollisionTime(TimeC); } + short getCollisionTimeMean() const { return getCollisionTime(kTimeMean); } + short getCollisionTimeA() const { return getCollisionTime(kTimeA); } + short getCollisionTimeC() const { return getCollisionTime(kTimeC); } bool isValidTime(int side) const { return getCollisionTime(side) < o2::InteractionRecord::DummyTime; } void setCollisionTime(short time, int side) { mCollisionTime[side] = time; } - short getVertex() const { return getCollisionTime(Vertex); } - void setVertex(short vertex) { mCollisionTime[Vertex] = vertex; } + short getVertex() const { return getCollisionTime(kTimeVertex); } + void setVertex(short vertex) { mCollisionTime[kTimeVertex] = vertex; } o2::fit::Triggers getTrigger() const { return mTriggers; } void setTriggers(o2::fit::Triggers trig) { mTriggers = trig; } + uint8_t getTechnicalWord() const { return mTechnicalWord; } + static constexpr uint8_t makeExtraTrgWord(bool isActiveA = true, bool isActiveC = true, bool isFlangeEvent = true) + { + return (static_cast(isActiveA) << kIsActiveSideA) | + (static_cast(isActiveC) << kIsActiveSideC) | + (static_cast(isFlangeEvent) << kIsFlangeEvent); + } o2::InteractionRecord getInteractionRecord() const { return mIntRecord; }; - - // void SetMgrEventTime(Double_t time) { mTimeStamp = time; } - gsl::span getBunchChannelData(const gsl::span tfdata) const; short static constexpr sDummyCollissionTime = 32767; + void print() const; + bool operator==(const RecPoints&) const = default; + private: + void initRecPointTriggers(const o2::fit::Triggers& digitTriggers, uint8_t extraTrgWord = 0) + { + const auto digitTriggerWord = digitTriggers.getTriggersignals(); + const auto trgAndTechWordPair = o2::fit::Triggers::parseDigitTriggerWord(digitTriggerWord, true); + mTriggers.setTriggers(trgAndTechWordPair.first | extraTrgWord); + mTechnicalWord = trgAndTechWordPair.second; + } + std::array mCollisionTime = {sDummyCollissionTime, sDummyCollissionTime, sDummyCollissionTime, sDummyCollissionTime}; o2::fit::Triggers mTriggers; // pattern of triggers in this BC - - ClassDefNV(RecPoints, 3); + uint8_t mTechnicalWord{0}; // field for keeping ETechnicalBits + ClassDefNV(RecPoints, 4); }; } // namespace ft0 } // namespace o2 diff --git a/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/SlewingCoef.h b/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/SlewingCoef.h new file mode 100644 index 0000000000000..704f4e3fd2324 --- /dev/null +++ b/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/SlewingCoef.h @@ -0,0 +1,47 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef ALICEO2_FT0_SLEWINGCOEF_H_ +#define ALICEO2_FT0_SLEWINGCOEF_H_ +//////////////////////////////////////////////// +// Slewing coefficients for FT0 +////////////////////////////////////////////// +#include "TGraph.h" +#include "Rtypes.h" + +#include +#include +#include + +namespace o2 +{ +namespace ft0 +{ + +struct SlewingCoef { + constexpr static int sNCHANNELS = 208; + constexpr static int sNAdc = 2; + using VecPoints_t = std::vector; // Set of points + using VecPlot_t = std::pair; // Plot as pair of two set of points + using VecSlewingCoefs_t = std::array, sNAdc>; // 0 - adc0, 1 - adc1 + typedef std::array, sNAdc> SlewingPlots_t; + VecSlewingCoefs_t mSlewingCoefs{}; + SlewingPlots_t makeSlewingPlots() const; + constexpr static const char* getObjectPath() + { + return "FT0/Calib/SlewingCoef"; + } + ClassDefNV(SlewingCoef, 1) +}; +} // namespace ft0 +} // namespace o2 + +#endif diff --git a/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/SpectraInfoObject.h b/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/SpectraInfoObject.h new file mode 100644 index 0000000000000..ac9d4ec10680f --- /dev/null +++ b/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/SpectraInfoObject.h @@ -0,0 +1,53 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef O2_FT0_SPECTRAINFOOBJECT_H +#define O2_FT0_SPECTRAINFOOBJECT_H + +#include +#include "Rtypes.h" +#include "FT0Base/Geometry.h" + +namespace o2::ft0 +{ + +struct SpectraInfoObject { + float mGausMean{}; // Peak of Gausian function + float mGausRMS{}; // RMS of Gausian function + float mGausConstant{}; // Constant of Gausian function + float mFitChi2{}; // Chi2 of Gausian fitting + float mStatMean{}; // Spectra mean + float mStatRMS{}; // Spectra RMS + float mStat{}; // Statistic + uint32_t mStatusBits{}; // Status bits for extra info + ClassDefNV(SpectraInfoObject, 1); +}; + +struct TimeSpectraInfoObject { + std::array mTime; + SpectraInfoObject mTimeA; + SpectraInfoObject mTimeC; + SpectraInfoObject mSumTimeAC; + SpectraInfoObject mDiffTimeCA; + static constexpr const char* getObjectPath() { return "FT0/Calib/TimeSpectraInfo"; } + ClassDefNV(TimeSpectraInfoObject, 1); +}; + +struct AmpSpectraInfoObject { + std::array mAmpADC0; + std::array mAmpADC1; + static constexpr const char* getObjectPath() { return "FT0/Calib/AmpSpectraInfo"; } + ClassDefNV(AmpSpectraInfoObject, 1); +}; + +} // namespace o2::ft0 + +#endif // O2_FT0_TIMESPECTRAINFOOBJECT_H diff --git a/Detectors/FIT/FT0/calibration/src/CalibParam.cxx b/DataFormats/Detectors/FIT/FT0/src/CalibParam.cxx similarity index 93% rename from Detectors/FIT/FT0/calibration/src/CalibParam.cxx rename to DataFormats/Detectors/FIT/FT0/src/CalibParam.cxx index b56f3f1418dd4..3c496824ac8ee 100644 --- a/Detectors/FIT/FT0/calibration/src/CalibParam.cxx +++ b/DataFormats/Detectors/FIT/FT0/src/CalibParam.cxx @@ -9,7 +9,7 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. -#include "FT0Calibration/CalibParam.h" +#include "DataFormatsFT0/CalibParam.h" using namespace o2::ft0; O2ParamImpl(CalibParam); \ No newline at end of file diff --git a/DataFormats/Detectors/FIT/FT0/src/DataFormatsFT0LinkDef.h b/DataFormats/Detectors/FIT/FT0/src/DataFormatsFT0LinkDef.h index ae8f8d0ebdae0..7f8c17a0cd191 100644 --- a/DataFormats/Detectors/FIT/FT0/src/DataFormatsFT0LinkDef.h +++ b/DataFormats/Detectors/FIT/FT0/src/DataFormatsFT0LinkDef.h @@ -17,6 +17,9 @@ #pragma link C++ class o2::ft0::Digit + ; #pragma link C++ class o2::ft0::DigitFilterParam + ; +#pragma link C++ class o2::ft0::ChannelFilterParam + ; +#pragma link C++ class o2::ft0::TimeFilterParam + ; +#pragma link C++ class o2::ft0::CalibParam + ; #pragma link C++ class o2::ft0::ChannelData + ; #pragma link C++ class o2::ft0::DetTrigInput + ; #pragma link C++ class o2::ft0::TriggersExt + ; @@ -37,7 +40,6 @@ #pragma link C++ class o2::ft0::HitType + ; #pragma link C++ class vector < o2::ft0::HitType> + ; -#pragma link C++ class o2::ft0::RawEventData + ; #pragma link C++ class o2::ft0::CTFHeader + ; #pragma link C++ class o2::ft0::CompressedDigits + ; #pragma link C++ class o2::ft0::CTF + ; @@ -45,7 +47,15 @@ #pragma link C++ class o2::ft0::FT0CalibrationInfoObject + ; #pragma link C++ class o2::ft0::FT0ChannelTimeCalibrationObject + ; +#pragma link C++ class o2::ft0::SpectraInfoObject + ; +#pragma link C++ class o2::ft0::TimeSpectraInfoObject + ; +#pragma link C++ class o2::ft0::AmpSpectraInfoObject + ; #pragma link C++ class o2::ft0::GlobalOffsetsCalibrationObject + ; #pragma link C++ class o2::ft0::RecoCalibInfoObject + ; #pragma link C++ class o2::ft0::GlobalOffsetsInfoObject + ; +#pragma link C++ class std::pair < std::vector < double>, std::vector < double>> + ; +#pragma link C++ class o2::ft0::SlewingCoef + ; + +#pragma link C++ class o2::ft0::EventsPerBc + ; + #endif diff --git a/DataFormats/Detectors/FIT/FT0/src/DigitFilterParam.cxx b/DataFormats/Detectors/FIT/FT0/src/DigitFilterParam.cxx index a71c140d61be9..b2d87e40f32bc 100644 --- a/DataFormats/Detectors/FIT/FT0/src/DigitFilterParam.cxx +++ b/DataFormats/Detectors/FIT/FT0/src/DigitFilterParam.cxx @@ -12,4 +12,6 @@ #include "DataFormatsFT0/DigitFilterParam.h" using namespace o2::ft0; -O2ParamImpl(DigitFilterParam); \ No newline at end of file +O2ParamImpl(DigitFilterParam); +O2ParamImpl(ChannelFilterParam); +O2ParamImpl(TimeFilterParam); \ No newline at end of file diff --git a/DataFormats/Detectors/FIT/FT0/src/LookUpTable.cxx b/DataFormats/Detectors/FIT/FT0/src/LookUpTable.cxx new file mode 100644 index 0000000000000..aa29f589acdec --- /dev/null +++ b/DataFormats/Detectors/FIT/FT0/src/LookUpTable.cxx @@ -0,0 +1,15 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include "DataFormatsFT0/LookUpTable.h" + +using namespace o2::ft0; +template class o2::fit::SingleLUT>; diff --git a/DataFormats/Detectors/FIT/FT0/src/RecPoints.cxx b/DataFormats/Detectors/FIT/FT0/src/RecPoints.cxx index f580d0dd1ea8c..afd244f977f71 100644 --- a/DataFormats/Detectors/FIT/FT0/src/RecPoints.cxx +++ b/DataFormats/Detectors/FIT/FT0/src/RecPoints.cxx @@ -21,14 +21,22 @@ using namespace o2::ft0; +void ChannelDataFloat::print() const +{ + printf(" ChID% d | CFDtime=%f | QTCampl=%f QTC chain %d\n", ChId, CFDTime, QTCAmpl, ChainQTC); +} + gsl::span RecPoints::getBunchChannelData(const gsl::span tfdata) const { // extract the span of channel data for this bunch from the whole TF data return ref.getEntries() ? gsl::span(tfdata).subspan(ref.getFirstEntry(), ref.getEntries()) : gsl::span(); } -void ChannelDataFloat::print() const +void RecPoints::print() const { - - printf(" ChID% d | CFDtime=%f | QTCampl=%f QTC chain %d\n", ChId, CFDTime, QTCAmpl, ChainQTC); + LOG(info) << "RecPoint data:"; + LOG(info) << "Collision times: mean: " << getCollisionTimeMean() << ", A: " << getCollisionTimeA() << ", C: " << getCollisionTimeC(); + LOG(info) << "Vertex: " << getVertex(); + LOG(info) << "Ref first: " << ref.getFirstEntry() << ", Ref entries: " << ref.getEntries(); + LOG(info) << "Triggers: " << mTriggers.print(); } diff --git a/DataFormats/Detectors/FIT/FT0/src/SlewingCoef.cxx b/DataFormats/Detectors/FIT/FT0/src/SlewingCoef.cxx new file mode 100644 index 0000000000000..e1d8bda5ac4ea --- /dev/null +++ b/DataFormats/Detectors/FIT/FT0/src/SlewingCoef.cxx @@ -0,0 +1,32 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include "DataFormatsFT0/SlewingCoef.h" +#include +using namespace o2::ft0; + +SlewingCoef::SlewingPlots_t SlewingCoef::makeSlewingPlots() const +{ + typename o2::ft0::SlewingCoef::SlewingPlots_t plots{}; + for (int iAdc = 0; iAdc < sNAdc; iAdc++) { + const auto& slewingCoefs = mSlewingCoefs[iAdc]; + auto& plotsAdc = plots[iAdc]; + for (int iCh = 0; iCh < sNCHANNELS; iCh++) { + const auto& points_x = slewingCoefs[iCh].first; + const auto& points_y = slewingCoefs[iCh].second; + assert(points_x.size() == points_y.size()); + const int nPoints = points_x.size(); + auto& plot = plotsAdc[iCh]; + plot = TGraph(nPoints, points_x.data(), points_y.data()); + } + } + return plots; +} diff --git a/DataFormats/Detectors/FIT/FV0/CMakeLists.txt b/DataFormats/Detectors/FIT/FV0/CMakeLists.txt index 9eea4dd092d7a..35bc653a8234e 100644 --- a/DataFormats/Detectors/FIT/FV0/CMakeLists.txt +++ b/DataFormats/Detectors/FIT/FV0/CMakeLists.txt @@ -16,11 +16,13 @@ o2_add_library(DataFormatsFV0 src/RecPoints.cxx src/RawEventData.cxx src/CTF.cxx + src/LookUpTable.cxx PUBLIC_LINK_LIBRARIES O2::FV0Base O2::DataFormatsFIT O2::SimulationDataFormat O2::CommonDataFormat Microsoft.GSL::GSL + O2::DetectorsCommonDataFormats ) o2_target_root_dictionary(DataFormatsFV0 @@ -33,5 +35,4 @@ o2_target_root_dictionary(DataFormatsFV0 include/DataFormatsFV0/RecPoints.h include/DataFormatsFV0/RawEventData.h include/DataFormatsFV0/LookUpTable.h - include/DataFormatsFV0/CTF.h -) + include/DataFormatsFV0/CTF.h) diff --git a/DataFormats/Detectors/FIT/FV0/include/DataFormatsFV0/CTF.h b/DataFormats/Detectors/FIT/FV0/include/DataFormatsFV0/CTF.h index e71e42ca2f4ad..275b682a15276 100644 --- a/DataFormats/Detectors/FIT/FV0/include/DataFormatsFV0/CTF.h +++ b/DataFormats/Detectors/FIT/FV0/include/DataFormatsFV0/CTF.h @@ -41,8 +41,8 @@ struct CompressedDigits { // trigger data std::vector trigger; // trigger bits - std::vector bcInc; // increment in BC if the same orbit, otherwise abs bc - std::vector orbitInc; // increment in orbit + std::vector bcInc; // increment in BC if the same orbit, otherwise abs bc + std::vector orbitInc; // increment in orbit std::vector nChan; // number of fired channels // channel data diff --git a/DataFormats/Detectors/FIT/FV0/include/DataFormatsFV0/ChannelData.h b/DataFormats/Detectors/FIT/FV0/include/DataFormatsFV0/ChannelData.h index f939a8f75108c..054b336510c4f 100644 --- a/DataFormats/Detectors/FIT/FV0/include/DataFormatsFV0/ChannelData.h +++ b/DataFormats/Detectors/FIT/FV0/include/DataFormatsFV0/ChannelData.h @@ -13,8 +13,8 @@ /// \brief Container class to store time and charge values of single FV0 channel /// \author maciej.slupecki@cern.ch -#ifndef _FV0_CHANNEL_DATA_H_ -#define _FV0_CHANNEL_DATA_H_ +#ifndef _FV0_CHANNELDATA_H_ +#define _FV0_CHANNELDATA_H_ #include #include @@ -22,7 +22,6 @@ namespace o2 { namespace fv0 { - struct ChannelData { static constexpr char sChannelNameDPL[] = "DIGITSCH"; static constexpr char sDigitName[] = "ChannelData"; @@ -64,6 +63,16 @@ struct ChannelData { static void setFlag(EEventDataBit bitFlag, uint8_t& chainQTC) { chainQTC |= (1 << bitFlag); } static void clearFlag(EEventDataBit bitFlag, uint8_t& chainQTC) { chainQTC &= ~(1 << bitFlag); } bool getFlag(EEventDataBit bitFlag) const { return bool(ChainQTC & (1 << bitFlag)); } + bool areAllFlagsGood() const + { + return (!getFlag(ChannelData::kIsDoubleEvent) && + !getFlag(ChannelData::kIsTimeInfoNOTvalid) && + getFlag(ChannelData::kIsCFDinADCgate) && + !getFlag(ChannelData::kIsTimeInfoLate) && + !getFlag(ChannelData::kIsAmpHigh) && + getFlag(ChannelData::kIsEventInTVDC) && + !getFlag(ChannelData::kIsTimeInfoLost)); + } void print() const; void printLog() const; [[nodiscard]] uint8_t getChannelID() const { return ChId; } @@ -78,5 +87,4 @@ struct ChannelData { }; } // namespace fv0 } // namespace o2 - #endif diff --git a/DataFormats/Detectors/FIT/FV0/include/DataFormatsFV0/FV0ChannelTimeCalibrationObject.h b/DataFormats/Detectors/FIT/FV0/include/DataFormatsFV0/FV0ChannelTimeCalibrationObject.h index 77f5b3cf5bbf1..d59dee5c12684 100644 --- a/DataFormats/Detectors/FIT/FV0/include/DataFormatsFV0/FV0ChannelTimeCalibrationObject.h +++ b/DataFormats/Detectors/FIT/FV0/include/DataFormatsFV0/FV0ChannelTimeCalibrationObject.h @@ -23,8 +23,10 @@ struct FV0ChannelTimeCalibrationObject { std::array mTimeOffsets{}; static constexpr const char* getObjectPath() { return "FV0/Calib/ChannelTimeOffset"; } + ClassDefNV(FV0ChannelTimeCalibrationObject, 1); }; + } // namespace o2::fv0 #endif // O2_FV0CHANNELTIMECALIBRATIONOBJECT_H diff --git a/DataFormats/Detectors/FIT/FV0/include/DataFormatsFV0/LookUpTable.h b/DataFormats/Detectors/FIT/FV0/include/DataFormatsFV0/LookUpTable.h index 67e7adad03bb4..4c77b0143d1dc 100644 --- a/DataFormats/Detectors/FIT/FV0/include/DataFormatsFV0/LookUpTable.h +++ b/DataFormats/Detectors/FIT/FV0/include/DataFormatsFV0/LookUpTable.h @@ -9,70 +9,18 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. // -//file RawEventData.h class for RAW data format -// Alla.Maevskaya -// simple look-up table just to feed digits 2 raw procedure. -//Will be really set after module/electronics connections -// #ifndef ALICEO2_FV0_LOOKUPTABLE_H_ #define ALICEO2_FV0_LOOKUPTABLE_H_ //////////////////////////////////////////////// // Look Up Table FV0 ////////////////////////////////////////////// - #include "DataFormatsFIT/LookUpTable.h" -#include "CommonUtils/NameConf.h" - +#include "DetectorsCommonDataFormats/DetID.h" namespace o2 { namespace fv0 { - -/*struct Topo { - int pmLink = 0; // Number of Processing Module, associated with GBT link ID - int pmCh = 0; // Channel within the Processing Module in range from 0-11 - ClassDefNV(Topo, 1); -}; - -inline bool operator<(Topo const& a, Topo const& b) -{ - return (a.pmLink < b.pmLink || (a.pmLink == b.pmLink && a.pmCh < b.pmCh)); -} -*/ -namespace new_lut - -{ -//Singleton for LookUpTable -template -class SingleLUT : public LUT -{ - private: - SingleLUT(const std::string& ccdbPath, const std::string& ccdbPathToLUT) : LUT(ccdbPath, ccdbPathToLUT) {} - SingleLUT(const std::string& pathToFile) : LUT(pathToFile) {} - SingleLUT(const SingleLUT&) = delete; - SingleLUT& operator=(SingleLUT&) = delete; - - public: - static constexpr char sDetectorName[] = "FV0"; - static constexpr char sDefaultLUTpath[] = "FV0/Config/LookupTable"; - inline static std::string sCurrentCCDBpath = ""; - inline static std::string sCurrentLUTpath = sDefaultLUTpath; - //Before instance() call, setup url and path - static void setCCDBurl(const std::string& url) { sCurrentCCDBpath = url; } - static void setLUTpath(const std::string& path) { sCurrentLUTpath = path; } - static SingleLUT& Instance() - { - if (sCurrentCCDBpath == "") { - sCurrentCCDBpath = o2::base::NameConf::getCCDBServer(); - } - static SingleLUT instanceLUT(sCurrentCCDBpath, sCurrentLUTpath); - return instanceLUT; - } -}; -} //namespace new_lut - -using SingleLUT = new_lut::SingleLUT>; - +using SingleLUT = o2::fit::SingleLUT>; } // namespace fv0 } // namespace o2 #endif diff --git a/DataFormats/Detectors/FIT/FV0/include/DataFormatsFV0/RawEventData.h b/DataFormats/Detectors/FIT/FV0/include/DataFormatsFV0/RawEventData.h index 30763250bc528..54b0d32a677c3 100644 --- a/DataFormats/Detectors/FIT/FV0/include/DataFormatsFV0/RawEventData.h +++ b/DataFormats/Detectors/FIT/FV0/include/DataFormatsFV0/RawEventData.h @@ -29,65 +29,6 @@ using EventHeader = o2::fit::EventHeader; using EventData = o2::fit::EventData; using TCMdata = o2::fit::TCMdata; using TCMdataExtended = o2::fit::TCMdataExtended; -class RawEventData -{ - public: - RawEventData() = default; - void print() const; - void printHexEventHeader() const; - void printHexEventData(uint64_t i) const; - const static int gStartDescriptor = 0x0000000f; - static const size_t sPayloadSizeSecondWord = 11; - static const size_t sPayloadSizeFirstWord = 5; - static constexpr size_t sPayloadSize = 16; - int size() const - { - return 1 + mEventHeader.nGBTWords; // EventHeader + EventData size - } - - std::vector to_vector(bool tcm) - { - constexpr int CRUWordSize = 16; - - std::vector result(size() * CRUWordSize); - char* out = result.data(); - if (!tcm) { - std::memcpy(out, &mEventHeader, sPayloadSize); - out += sPayloadSize; - LOG(debug) << "write PM header words " << (int)mEventHeader.nGBTWords << " orbit: " << int(mEventHeader.orbit) << " bc " << int(mEventHeader.bc) << " out " << result.size(); - printHexEventHeader(); - out += CRUWordSize - sPayloadSize; - - for (int i = 0; i < mEventHeader.nGBTWords; ++i) { - std::memcpy(out, &mEventData[2 * i], sPayloadSizeFirstWord); - LOG(debug) << " 1st word " << mEventData[2 * i].channelID << " charge " << mEventData[2 * i].charge << " time " << mEventData[2 * i].time << " out " << result.size(); - out += sPayloadSizeFirstWord; - std::memcpy(out, &mEventData[2 * i + 1], sPayloadSizeSecondWord); - out += sPayloadSizeSecondWord; - LOG(debug) << " 2nd word " << mEventData[2 * i + 1].channelID << " charge " << mEventData[2 * i + 1].charge << " time " << mEventData[2 * i + 1].time << " out " << result.size(); - out += CRUWordSize - sPayloadSizeSecondWord - sPayloadSizeFirstWord; - printHexEventData(i); - } - } else { - // TCM data - std::memcpy(out, &mEventHeader, sPayloadSize); - out += sPayloadSize; - LOG(debug) << "write TCM header words " << (int)mEventHeader.nGBTWords << " orbit " << int(mEventHeader.orbit) << " bc " << int(mEventHeader.bc) << " out " << result.size(); - std::memcpy(out, &mTCMdata, sizeof(TCMdata)); - out += sizeof(TCMdata); - LOG(debug) << "write TCM words " << sizeof(mTCMdata) << " orbit " << int(mEventHeader.orbit) << " bc " << int(mEventHeader.bc) << " out " << result.size() << " sum time A " << mTCMdata.timeA; - } - - return result; - } - - public: - EventHeader mEventHeader; //! - EventData mEventData[Constants::nChannelsPerPm]; //! - TCMdata mTCMdata; //! - - ClassDefNV(RawEventData, 1); -}; } // namespace fv0 } // namespace o2 #endif diff --git a/DataFormats/Detectors/FIT/FV0/include/DataFormatsFV0/RecPoints.h b/DataFormats/Detectors/FIT/FV0/include/DataFormatsFV0/RecPoints.h index c5e0a827de132..b3527fdd049d2 100644 --- a/DataFormats/Detectors/FIT/FV0/include/DataFormatsFV0/RecPoints.h +++ b/DataFormats/Detectors/FIT/FV0/include/DataFormatsFV0/RecPoints.h @@ -28,7 +28,7 @@ namespace fv0 struct ChannelDataFloat { int channel = -1; // channel Id - double time = -20000; // time in ps, 0 at the LHC clk center + double time = -20000; // time in ns, 0 at the LHC clk center double charge = -20000; // charge [channels] int adcId = -1; // QTC chain @@ -42,6 +42,7 @@ struct ChannelDataFloat { } void print() const; + bool operator==(const ChannelDataFloat&) const = default; ClassDefNV(ChannelDataFloat, 1); }; @@ -77,6 +78,9 @@ class RecPoints gsl::span getBunchChannelData(const gsl::span tfdata) const; short static constexpr sDummyCollissionTime = 32767; + void print() const; + bool operator==(const RecPoints&) const = default; + private: o2::dataformats::RangeReference mRef; o2::InteractionRecord mIntRecord; diff --git a/DataFormats/Detectors/FIT/FV0/src/DataFormatsFV0LinkDef.h b/DataFormats/Detectors/FIT/FV0/src/DataFormatsFV0LinkDef.h index ccc1516067a1e..ab23b4aa727bc 100644 --- a/DataFormats/Detectors/FIT/FV0/src/DataFormatsFV0LinkDef.h +++ b/DataFormats/Detectors/FIT/FV0/src/DataFormatsFV0LinkDef.h @@ -18,6 +18,8 @@ #pragma link C++ class o2::fv0::Hit + ; #pragma link C++ class vector < o2::fv0::Hit> + ; #pragma link C++ class o2::fv0::MCLabel + ; +#include "SimulationDataFormat/MCTruthContainer.h" +#pragma link C++ class o2::dataformats::MCTruthContainer < o2::fv0::MCLabel> + ; #pragma link C++ class o2::fv0::ChannelData + ; #pragma link C++ class o2::fv0::Digit + ; @@ -26,7 +28,6 @@ #pragma link C++ class std::vector < o2::fv0::DetTrigInput> + ; #pragma link C++ class std::vector < o2::fv0::Digit> + ; -#pragma link C++ class o2::fv0::RawEventData + ; #pragma link C++ class o2::fv0::CTFHeader + ; #pragma link C++ class o2::fv0::CTF + ; #pragma link C++ class o2::ctf::EncodedBlocks < o2::fv0::CTFHeader, 8, uint32_t> + ; diff --git a/DataFormats/Detectors/FIT/FV0/src/LookUpTable.cxx b/DataFormats/Detectors/FIT/FV0/src/LookUpTable.cxx new file mode 100644 index 0000000000000..7a1d5969abb79 --- /dev/null +++ b/DataFormats/Detectors/FIT/FV0/src/LookUpTable.cxx @@ -0,0 +1,15 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include "DataFormatsFV0/LookUpTable.h" + +using namespace o2::fv0; +template class o2::fit::SingleLUT>; diff --git a/DataFormats/Detectors/FIT/FV0/src/RawEventData.cxx b/DataFormats/Detectors/FIT/FV0/src/RawEventData.cxx index eb5522ec9f9f4..6a25842917d89 100644 --- a/DataFormats/Detectors/FIT/FV0/src/RawEventData.cxx +++ b/DataFormats/Detectors/FIT/FV0/src/RawEventData.cxx @@ -13,33 +13,4 @@ #include #include -using namespace o2::fv0; - -ClassImp(RawEventData); -void RawEventData::printHexEventHeader() const -{ - std::stringstream ssheader; - ssheader << std::setfill('0') << std::setw(16) << std::hex << mEventHeader.word[0] << " " << std::setw(16) << mEventHeader.word[1] << "\n "; - ssheader << std::setw(3) << (0x00000fff & mEventHeader.bc) << " " - << std::setw(8) << (0xffffffff & mEventHeader.orbit) << " " - << std::setw(5) << (0x000fffff & mEventHeader.reservedField1) << " " - << std::setw(2) << (0x000000ff & mEventHeader.reservedField2) << " " - << std::setw(1) << (0x0000000f & mEventHeader.nGBTWords) << " " - << std::setw(1) << (0x0000000f & mEventHeader.startDescriptor) << " " - << std::setw(12) << (0xffffffffffff & mEventHeader.reservedField3); - LOG(debug) << ssheader.str(); -} - -void RawEventData::printHexEventData(uint64_t i) const -{ - std::stringstream ssdata; - ssdata << "D0:0x "; - ssdata << std::setfill('0') << std::hex << std::setw(16) << mEventData[2 * i].word << "\n "; - ssdata << std::setw(3) << (0x0fff & mEventData[2 * i].time) << " " - << std::setw(8) << (0x1fff & mEventData[2 * i].charge) << "\n "; - ssdata << "D1:0x "; - ssdata << std::setfill('0') << std::hex << std::setw(16) << mEventData[2 * i + 1].word << "\n "; - ssdata << std::setw(3) << (0x0fff & mEventData[2 * i + 1].time) << " " - << std::setw(8) << (0x1fff & mEventData[2 * i + 1].charge); - LOG(debug) << " | " << ssdata.str(); -} +using namespace o2::fv0; \ No newline at end of file diff --git a/DataFormats/Detectors/FIT/FV0/src/RecPoints.cxx b/DataFormats/Detectors/FIT/FV0/src/RecPoints.cxx index 18f2effc281e8..ef1554acf5419 100644 --- a/DataFormats/Detectors/FIT/FV0/src/RecPoints.cxx +++ b/DataFormats/Detectors/FIT/FV0/src/RecPoints.cxx @@ -14,13 +14,22 @@ using namespace o2::fv0; -gsl::span RecPoints::getBunchChannelData(const gsl::span tfdata) const +void ChannelDataFloat::print() const { - // extract the span of channel data for this bunch from the whole TF data - return mRef.getEntries() ? gsl::span(tfdata).subspan(mRef.getFirstEntry(), mRef.getEntries()) : gsl::span(); + printf(" Channel=%d | time=%f | charge=%f | adcId=%d\n", channel, time, charge, adcId); } -void ChannelDataFloat::print() const +void RecPoints::print() const { - printf(" Channel=%d | time=%f | charge=%f | adcId=%d\n", channel, time, charge, adcId); + printf("RecPoint data:"); + printf("Collision times: first: %f, global mean: %f, selected mean: %f\n", getCollisionFirstTime(), getCollisionGlobalMeanTime(), getCollisionSelectedMeanTime()); + printf("Ref first: %d, Ref entries: %d\n", mRef.getFirstEntry(), mRef.getEntries()); + printf("Triggers: "); + mTriggers.print(); +} + +gsl::span RecPoints::getBunchChannelData(const gsl::span tfdata) const +{ + // extract the span of channel data for this bunch from the whole TF data + return mRef.getEntries() ? gsl::span(tfdata).subspan(mRef.getFirstEntry(), mRef.getEntries()) : gsl::span(); } diff --git a/DataFormats/Detectors/FIT/common/CMakeLists.txt b/DataFormats/Detectors/FIT/common/CMakeLists.txt index 1599d19ee6c9b..61dbcabc7f087 100644 --- a/DataFormats/Detectors/FIT/common/CMakeLists.txt +++ b/DataFormats/Detectors/FIT/common/CMakeLists.txt @@ -12,9 +12,15 @@ o2_add_library(DataFormatsFIT SOURCES src/RawEventData.cxx src/Triggers.cxx + src/RawDataMetric.cxx + src/LookUpTable.cxx PUBLIC_LINK_LIBRARIES O2::CommonDataFormat + O2::DetectorsCommonDataFormats O2::CCDB) o2_target_root_dictionary(DataFormatsFIT - HEADERS include/DataFormatsFIT/LookUpTable.h - include/DataFormatsFIT/Triggers.h) + HEADERS include/DataFormatsFIT/DCSDPValues.h + include/DataFormatsFIT/LookUpTable.h + include/DataFormatsFIT/Triggers.h + include/DataFormatsFIT/ChannelData.h + include/DataFormatsFIT/Digit.h) diff --git a/DataFormats/Detectors/FIT/common/include/DataFormatsFIT/ChannelData.h b/DataFormats/Detectors/FIT/common/include/DataFormatsFIT/ChannelData.h new file mode 100644 index 0000000000000..452307d05f200 --- /dev/null +++ b/DataFormats/Detectors/FIT/common/include/DataFormatsFIT/ChannelData.h @@ -0,0 +1,70 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file ChannelData.h +/// \brief Class represents enity with info per channel in given event +/// \author Artur Furs afurs@cern.ch + +#ifndef _FT0_CHANNELDATA_H_ +#define _FT0_CHANNELDATA_H_ + +#include +#include +#include +#include "DetectorsCommonDataFormats/DetID.h" + +namespace o2 +{ +namespace fit +{ + +template +struct ChannelData { + static constexpr o2::detectors::DetID sDetID = o2::detectors::DetID(DetID); + static constexpr uint8_t sDUMMY_CHANNEL_ID = 0xff; + static constexpr uint8_t sDUMMY_PM_WORD = 0xff; + static constexpr int16_t sDUMMY_TIME = -5000; + static constexpr int16_t sDUMMY_AMP = -5000; + uint8_t mChannelID = sDUMMY_CHANNEL_ID; // channel id + uint8_t mWordPM = sDUMMY_PM_WORD; // PM word, based on EBitsPM + int16_t mTime = sDUMMY_TIME; // time in TDC units + int16_t mAmp = sDUMMY_AMP; // amplitude in ADC units + enum EBitsPM { + kNumberADC, + kIsDoubleEvent, + kIsTimeInfoNotValid, + kIsCFDinADCgate, + kIsTimeInfoLate, + kIsAmpNotValid, + kIsVertexEvent, + kIsTimeInfoLost + }; + static const inline std::map sMapBitsPM = { + {EBitsPM::kNumberADC, "NumberADC"}, + {EBitsPM::kIsDoubleEvent, "IsDoubleEvent"}, + {EBitsPM::kIsTimeInfoNotValid, "IsTimeInfoNotValid"}, + {EBitsPM::kIsCFDinADCgate, "IsCFDinADCgate"}, + {EBitsPM::kIsTimeInfoLate, "IsTimeInfoLate"}, + {EBitsPM::kIsAmpNotValid, "IsAmpNotValid"}, + {EBitsPM::kIsVertexEvent, "IsVertexEvent"}, + {EBitsPM::kIsTimeInfoLost, "IsTimeInfoLost"}}; + ChannelData() = default; + ChannelData(uint8_t channelID, uint8_t wordPM, int16_t time, int16_t amp) : mChannelID(channelID), mWordPM(wordPM), mTime(time), mAmp(amp) + { + } + // void print() const; + bool operator<=>(ChannelData const& other) const = default; + bool operator==(ChannelData const& other) const = default; + ClassDefNV(ChannelData, 1); +}; +} // namespace fit +} // namespace o2 +#endif diff --git a/DataFormats/Detectors/FIT/common/include/DataFormatsFIT/DCSDPValues.h b/DataFormats/Detectors/FIT/common/include/DataFormatsFIT/DCSDPValues.h index f17c7b57b606d..97bbe982e6aac 100644 --- a/DataFormats/Detectors/FIT/common/include/DataFormatsFIT/DCSDPValues.h +++ b/DataFormats/Detectors/FIT/common/include/DataFormatsFIT/DCSDPValues.h @@ -14,22 +14,28 @@ #include #include "Framework/Logger.h" +#include "Framework/O2LongInt.h" namespace o2 { namespace fit { struct DCSDPValues { - std::vector> values; + std::vector> values; DCSDPValues() { - values = std::vector>(); + values = std::vector>(); } - void add(uint64_t timestamp, int value) + void add(uint64_t timestamp, int64_t value) { - values.push_back(std::pair(timestamp, value)); + values.push_back(std::pair(timestamp, value)); + } + + bool empty() + { + return values.empty(); } void makeEmpty() @@ -52,10 +58,10 @@ struct DCSDPValues { } } - ClassDefNV(DCSDPValues, 1); + ClassDefNV(DCSDPValues, 3); }; } // namespace fit } // namespace o2 -#endif \ No newline at end of file +#endif diff --git a/DataFormats/Detectors/FIT/common/include/DataFormatsFIT/DeadChannelMap.h b/DataFormats/Detectors/FIT/common/include/DataFormatsFIT/DeadChannelMap.h new file mode 100644 index 0000000000000..87bd1fee2165e --- /dev/null +++ b/DataFormats/Detectors/FIT/common/include/DataFormatsFIT/DeadChannelMap.h @@ -0,0 +1,53 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file DeadChannelMap.h +/// \brief Dead channel map for FIT +/// +/// \author Andreas Molander , University of Jyvaskyla, Finland + +#ifndef O2_FIT_DEADCHANNELMAP_H +#define O2_FIT_DEADCHANNELMAP_H + +#include +#include + +namespace o2 +{ +namespace fit +{ + +struct DeadChannelMap { + /// Dead channel map as 'channel id - state' pairs. true = alive, false = dead. + std::unordered_map map; + + void setChannelAlive(const uint8_t& chId, const bool isAlive) + { + map[chId] = isAlive; + } + + const bool isChannelAlive(const uint8_t& chId) const + { + return map.at(chId); + } + + void clear() + { + map.clear(); + } + + ClassDefNV(DeadChannelMap, 1); +}; + +} // namespace fit +} // namespace o2 + +#endif // O2_FIT_DEADCHANNELMAP_H \ No newline at end of file diff --git a/DataFormats/Detectors/FIT/common/include/DataFormatsFIT/Digit.h b/DataFormats/Detectors/FIT/common/include/DataFormatsFIT/Digit.h new file mode 100644 index 0000000000000..adaef4ec8349a --- /dev/null +++ b/DataFormats/Detectors/FIT/common/include/DataFormatsFIT/Digit.h @@ -0,0 +1,79 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +// \file Digit.h +/// \brief FIT event entity +/// \author Artur Furs afurs@cern.ch + +#ifndef O2_FIT_DIGIT_H_ +#define O2_FIT_DIGIT_H_ + +#include "CommonDataFormat/InteractionRecord.h" +#include "CommonDataFormat/RangeReference.h" +#include "DetectorsCommonDataFormats/DetID.h" +#include "DataFormatsFIT/ChannelData.h" +#include "DataFormatsFIT/Triggers.h" + +#include +#include + +namespace o2 +{ +namespace fit +{ +using Triggers = o2::fit::Triggers; + +template +struct DigitBase { + static constexpr o2::detectors::DetID sDetID = o2::detectors::DetID(DetID); + typedef ChannelData ChannelData_t; // related ChannelData entity + o2::InteractionRecord mIR{}; // Interaction record (orbit, bc) + o2::dataformats::RangeReference mReference{}; + DigitBase() = default; + DigitBase(int first, int nEntries, const o2::InteractionRecord& ir) : mIR(ir) + { + mReference.setFirstEntry(first); + mReference.setEntries(nEntries); + } + uint32_t getOrbit() const { return mIR.orbit; } + uint16_t getBC() const { return mIR.bc; } + const o2::InteractionRecord& getIntRecord() const { return mIR; }; + gsl::span getBunchChannelData(const gsl::span channelData) const + { + return mReference.getEntries() ? gsl::span(&channelData[mReference.getFirstEntry()], mReference.getEntries()) : gsl::span(); + } + ClassDefNV(DigitBase, 1); +}; + +template +struct Digit : public DigitBase { + uint8_t mTriggerWord{}; + Digit() = default; + Digit(int first, int nEntries, const o2::InteractionRecord& ir, uint8_t trgWord) : DigitBase(first, nEntries, ir), mTriggerWord(trgWord) + { + } + ClassDefNV(Digit, 1); +}; + +template +struct DigitExt : public DigitBase { + Triggers mTriggers{}; + DigitExt() = default; + DigitExt(int first, int nEntries, const o2::InteractionRecord& ir, const Triggers& triggers) : DigitBase(first, nEntries, ir), mTriggers(triggers) + { + } + ClassDefNV(DigitExt, 1); +}; + +} // namespace fit +} // namespace o2 + +#endif diff --git a/DataFormats/Detectors/FIT/common/include/DataFormatsFIT/LookUpTable.h b/DataFormats/Detectors/FIT/common/include/DataFormatsFIT/LookUpTable.h index 176c1669e8e26..aa4bb1fba8d41 100644 --- a/DataFormats/Detectors/FIT/common/include/DataFormatsFIT/LookUpTable.h +++ b/DataFormats/Detectors/FIT/common/include/DataFormatsFIT/LookUpTable.h @@ -15,7 +15,10 @@ // Look Up Table FIT ////////////////////////////////////////////// -#include "CCDB/BasicCCDBManager.h" +#include "DetectorsCommonDataFormats/DetID.h" +#include "CommonUtils/NameConf.h" +#include "Framework/Logger.h" +#define BOOST_BIND_GLOBAL_PLACEHOLDERS #include #include #include @@ -32,7 +35,7 @@ namespace o2 { namespace fit { -struct EntryCRU { //This is specific struct for CRU entry +struct EntryCRU { // This is specific struct for CRU entry int mLinkID; int mEndPointID; int mCRUID; @@ -102,7 +105,7 @@ struct HasherPM { }; struct ComparerPM { - //Always true due to perfect hasher + // Always true due to perfect hasher bool operator()(const EntryPM& entry1, const EntryPM& entry2) const { return ((entry1.mEntryCRU.mLinkID << 8) | (entry1.mLocalChannelID << 4) | (entry1.mEntryCRU.mEndPointID)) == ((entry2.mEntryCRU.mLinkID << 8) | (entry2.mLocalChannelID << 4) | (entry2.mEntryCRU.mEndPointID)); @@ -111,9 +114,9 @@ struct ComparerPM { struct EntryFEE { EntryCRU mEntryCRU; - std::string mChannelID; //ChannelID, string type because some entries containes N/A - std::string mLocalChannelID; //Local channelID, string type because some entries containes N/A - std::string mModuleType; //PM, PM-LCS, TCM + std::string mChannelID; // ChannelID, string type because some entries containes N/A + std::string mLocalChannelID; // Local channelID, string type because some entries containes N/A + std::string mModuleType; // PM, PM-LCS, TCM std::string mModuleName; std::string mBoardHV; std::string mChannelHV; @@ -156,13 +159,11 @@ enum class EModuleType : int { kUnknown, kTCM }; template , - typename MapEntryPM2ChannelID = std::unordered_map, - typename = typename std::enable_if_t::value>> + typename MapEntryPM2ChannelID = std::unordered_map> class LookupTableBase { public: - LookupTableBase(const std::string& pathToFile) { initFromFile(pathToFile); } - LookupTableBase(const std::string& urlCCDB, const std::string& pathToStorageInCCDB) { initCCDB(urlCCDB, pathToStorageInCCDB); } + typedef std::vector Table_t; typedef MapEntryPM2ChannelID MapEntryPM2ChannelID_t; typedef MapEntryCRU2ModuleType MapEntryCRU2ModuleType_t; typedef typename MapEntryPM2ChannelID_t::key_type EntryPM_t; @@ -170,14 +171,25 @@ class LookupTableBase typedef typename MapEntryPM2ChannelID_t::mapped_type ChannelID_t; typedef std::map MapChannelID2EntryPM_t; // for digit2raw typedef std::map MapModuleType2EntryCRU; // for digit2raw - typedef EntryPM_t Topo_t; //temporary for common interface - //Map of str module names -> enum types + typedef EntryPM_t Topo_t; // temporary for common interface + + LookupTableBase() = default; + LookupTableBase(const Table_t* vecEntryFEE) { initFromTable(vecEntryFEE); } + LookupTableBase(const std::string& pathToFile) { initFromFile(pathToFile); } + LookupTableBase(const std::string& urlCCDB, const std::string& pathToStorageInCCDB, long timestamp = -1) { initCCDB(urlCCDB, pathToStorageInCCDB, timestamp); } + // Map of str module names -> enum types const std::map mMapModuleTypeStr2Enum = {{"PM", EModuleType::kPM}, {"PM-LCS", EModuleType::kPM_LCS}, {"TCM", EModuleType::kTCM}}; - //Warning! To exclude double mapping do not use isTCM and isPM in the same time + // Warning! To exclude double mapping do not use isTCM and isPM in the same time bool isTCM(int linkID, int epID) const { return mEntryCRU_TCM.mLinkID == linkID && mEntryCRU_TCM.mEndPointID == epID; } + + bool isPM(int linkID, int epID) const + { + return isPM(EntryCRU_t{linkID, epID}); + } + bool isTCM(const EntryCRU_t& entryCRU) const { if (getModuleType(entryCRU) == EModuleType::kTCM) { @@ -231,11 +243,10 @@ class LookupTableBase prepareEntriesFEE(filepath); prepareLUT(); } - void initCCDB(const std::string& urlCCDB, const std::string& pathToStorageInCCDB) + void initCCDB(const std::string& urlCCDB, const std::string& pathToStorageInCCDB, long timestamp = -1); + void initFromTable(const Table_t* vecEntryFEE) { - auto& mgr = o2::ccdb::BasicCCDBManager::instance(); - mgr.setURL(urlCCDB); - mVecEntryFEE = *(mgr.get>(pathToStorageInCCDB)); + mVecEntryFEE = *vecEntryFEE; prepareLUT(); } ChannelID_t getGlobalChannelID(const EntryPM_t& entryPM, bool& isValid) const @@ -270,9 +281,9 @@ class LookupTableBase boost::property_tree::read_json(pathToConfigFile.c_str(), propertyTree); mVecEntryFEE = prepareEntriesFEE(propertyTree); } - std::vector prepareEntriesFEE(const boost::property_tree::ptree& propertyTree) + Table_t prepareEntriesFEE(const boost::property_tree::ptree& propertyTree) { - std::vector vecEntryFEE; + Table_t vecEntryFEE; for (const auto& pairEntry : propertyTree) { const auto& propertyTreeSingle = pairEntry.second; EntryFEE entryFEE{}; @@ -321,12 +332,12 @@ class LookupTableBase } */ } - const std::vector& getVecMetadataFEE() const { return mVecEntryFEE; } + const Table_t& getVecMetadataFEE() const { return mVecEntryFEE; } const MapEntryCRU2ModuleType_t& getMapEntryCRU2ModuleType() const { return mMapEntryCRU2ModuleType; } const MapEntryPM2ChannelID_t& getMapEntryPM2ChannelID() const { return mMapEntryPM2ChannelID; } const EntryCRU_t& getEntryCRU_TCM() const { return mEntryCRU_TCM; } - //Temporary - //Making topo for FEE recognizing(Local channelID is supressed) + // Temporary + // Making topo for FEE recognizing(Local channelID is supressed) static Topo_t makeGlobalTopo(const Topo_t& topo) { return Topo_t{topo.mEntryCRU, 0}; @@ -351,15 +362,15 @@ class LookupTableBase }); return Topo_t{findResult->first, 0}; } - //Prepare full map for FEE metadata(for digit2raw convertion) + // Prepare full map for FEE metadata(for digit2raw convertion) template auto makeMapFEEmetadata() -> std::map { std::map mapResult; - const uint16_t cruID = 0; //constant - uint64_t feeID = 0; //increments + const uint16_t cruID = 0; // constant + uint64_t feeID = 0; // increments const auto& mapEntryPM2ChannelID = getMapEntryPM2ChannelID(); - //Temporary for sorting FEEIDs without using them from LUT(for digit2raw convertion), and by using GlobalChannelID + // Temporary for sorting FEEIDs without using them from LUT(for digit2raw convertion), and by using GlobalChannelID std::map mapBuf; for (const auto& entry : mapEntryPM2ChannelID) { mapBuf.insert({entry.second, entry.first}); @@ -381,7 +392,7 @@ class LookupTableBase rdhObj.endPointID = topoObj.mEntryCRU.mEndPointID; rdhObj.feeId = feeID; rdhObj.cruID = cruID; - } else //Using RDHUtils + } else // Using RDHUtils { RDHhelper::setLinkID(&rdhObj, topoObj.mEntryCRU.mLinkID); RDHhelper::setEndPointID(&rdhObj, topoObj.mEntryCRU.mEndPointID); @@ -399,10 +410,71 @@ class LookupTableBase private: EntryCRU_t mEntryCRU_TCM; - std::vector mVecEntryFEE; + Table_t mVecEntryFEE; MapEntryCRU2ModuleType_t mMapEntryCRU2ModuleType; MapEntryPM2ChannelID_t mMapEntryPM2ChannelID; + typedef std::enable_if_t::value> CheckChannelIDtype; // should be integral }; + +// Singleton for LookUpTable, coomon for all three FIT detectors +template +class SingleLUT : public LUT +{ + private: + SingleLUT() = default; + SingleLUT(const std::string& ccdbPath, const std::string& ccdbPathToLUT) : LUT(ccdbPath, ccdbPathToLUT) {} + SingleLUT(const std::string& pathToFile) : LUT(pathToFile) {} + SingleLUT(const SingleLUT&) = delete; + SingleLUT& operator=(SingleLUT&) = delete; + constexpr static bool isValidDet() + { + return (DetID == o2::detectors::DetID::FDD) || (DetID == o2::detectors::DetID::FT0) || (DetID == o2::detectors::DetID::FV0); + } + + public: + typedef LUT LookupTable_t; + typedef typename LookupTable_t::Table_t Table_t; + + constexpr static const char* getObjectPath() + { + static_assert(isValidDet(), "Invalid detector type(o2::detectors::DetID::ID)! Should be one of the FIT detector!"); + if constexpr (DetID == o2::detectors::DetID::FDD) { + return "FDD/Config/LookupTable"; + } else if constexpr (DetID == o2::detectors::DetID::FT0) { + return "FT0/Config/LookupTable"; + } else if constexpr (DetID == o2::detectors::DetID::FV0) { + return "FV0/Config/LookupTable"; + } + return ""; + } + static constexpr o2::detectors::DetID sDetID = o2::detectors::DetID(DetID); + static constexpr const char* sDetectorName = o2::detectors::DetID::getName(DetID); + static constexpr const char* sDefaultLUTpath = getObjectPath(); + static constexpr const char sObjectName[] = "LookupTable"; + inline static std::string sCurrentCCDBpath = ""; + inline static std::string sCurrentLUTpath = sDefaultLUTpath; + // Before instance() call, setup url and path + static void setCCDBurl(const std::string& url) { sCurrentCCDBpath = url; } + static void setLUTpath(const std::string& path) { sCurrentLUTpath = path; } + bool mFirstUpdate{true}; // option in case if LUT should be updated during workflow initialization + static SingleLUT& Instance(const Table_t* table = nullptr, long timestamp = -1) + { + if (sCurrentCCDBpath == "") { + sCurrentCCDBpath = o2::base::NameConf::getCCDBServer(); + } + static SingleLUT instanceLUT; + if (table != nullptr) { + instanceLUT.initFromTable(table); + instanceLUT.mFirstUpdate = false; + } else if (instanceLUT.mFirstUpdate) { + instanceLUT.initCCDB(sCurrentCCDBpath, sCurrentLUTpath, timestamp); + instanceLUT.mFirstUpdate = false; + } + return instanceLUT; + } +}; + } // namespace fit } // namespace o2 + #endif diff --git a/DataFormats/Detectors/FIT/common/include/DataFormatsFIT/RawDataMetric.h b/DataFormats/Detectors/FIT/common/include/DataFormatsFIT/RawDataMetric.h new file mode 100644 index 0000000000000..79aba23841abc --- /dev/null +++ b/DataFormats/Detectors/FIT/common/include/DataFormatsFIT/RawDataMetric.h @@ -0,0 +1,83 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +// +// file RawDataMetric.h class Struct for collecting metrics during decoding +// with Artur.Furs@cern.ch +// +#ifndef ALICEO2_FIT_RAWDATAMETRIC_H_ +#define ALICEO2_FIT_RAWDATAMETRIC_H_ + +#include +#include +#include +#include + +namespace o2 +{ +namespace fit +{ +struct RawDataMetric { + RawDataMetric(uint8_t linkID, uint8_t EPID, uint16_t FEEID, bool isRegisteredFEE = true) : mLinkID(linkID), mEPID(EPID), mFEEID(FEEID), mIsRegisteredFEE(isRegisteredFEE) {} + ~RawDataMetric() = default; + + enum EStatusBits { + kIncompletePayload, // Incomplete payload + kWrongDescriptor, // Wrong descriptors in header + kWrongChannelMapping, // Wrong channel mapping + kEmptyDataBlock, // Only header in data block + kDecodedDataBlock // Decoded w/o any issue data block + }; + typedef uint8_t Status_t; + constexpr static uint8_t sNbits = 5; + inline bool checkBadDataBlock(Status_t metric) + { + bool result = checkStatusBit(metric, EStatusBits::kIncompletePayload); + if (!result) { // Incomplete payload has high priority among errors + result |= checkStatusBit(metric, EStatusBits::kEmptyDataBlock); + // Lets just check this for a while, w/o any decision for data block + // result |= checkStatusBit(metric,EStatusBits::kWrongDescriptor); + checkStatusBit(metric, EStatusBits::kWrongDescriptor); + } + return result; + } + inline void addStatusBit(EStatusBits statusBit, bool val = true) + { + mBitStats[statusBit] += static_cast(val); + } + + inline bool checkStatusBit(Status_t metric, EStatusBits statusBit) + { + const bool result = (metric & (1 << statusBit)) > 0; + mBitStats[statusBit] += static_cast(result); + return result; + } + + inline static bool isBitActive(Status_t metric, EStatusBits statusBit) + { + return (metric & (1 << statusBit)) > 0; + } + + inline static void setStatusBit(Status_t& metric, EStatusBits statusBit, bool val = true) + { + metric |= (static_cast(val) << statusBit); + } + void print() const; + static Status_t getAllBitsActivated(); + uint8_t mLinkID; + uint8_t mEPID; + uint16_t mFEEID; + bool mIsRegisteredFEE; + std::array mBitStats{}; + const static std::map sMapBitsToNames; +}; +} // namespace fit +} // namespace o2 +#endif diff --git a/DataFormats/Detectors/FIT/common/include/DataFormatsFIT/RawEventData.h b/DataFormats/Detectors/FIT/common/include/DataFormatsFIT/RawEventData.h index 63d81fcaf151f..ec04f770f8061 100644 --- a/DataFormats/Detectors/FIT/common/include/DataFormatsFIT/RawEventData.h +++ b/DataFormats/Detectors/FIT/common/include/DataFormatsFIT/RawEventData.h @@ -23,26 +23,20 @@ namespace o2 { namespace fit { - struct EventHeader { static constexpr size_t PayloadSize = 10; static constexpr size_t PayloadPerGBTword = 10; static constexpr size_t MinNelements = 1; static constexpr size_t MaxNelements = 1; - union { - uint64_t word[2] = {}; - struct { - uint64_t bc : 12; - uint64_t orbit : 32; - uint64_t phase : 3; - uint64_t errorPhase : 1; - uint64_t reservedField1 : 16; - uint64_t reservedField2 : 8; - uint64_t nGBTWords : 4; - uint64_t startDescriptor : 4; - uint64_t reservedField3 : 48; - }; - }; + static constexpr uint64_t sDescriptor = 0xf; + uint64_t bc : 12; + uint64_t orbit : 32; + uint64_t phase : 3; + uint64_t errorPhase : 1; + uint64_t reservedField1 : 16; + uint64_t reservedField2 : 8; + uint64_t nGBTWords : 4; + uint64_t startDescriptor : 4; InteractionRecord getIntRec() const { return InteractionRecord{(uint16_t)bc, (uint32_t)orbit}; } uint16_t getBC() const { return static_cast(bc); } uint32_t getOrbit() const { return static_cast(orbit); } @@ -51,57 +45,42 @@ struct EventHeader { bc = intRec.bc; orbit = intRec.orbit; } + inline bool isBadDescriptor() const { return startDescriptor != sDescriptor; } void print() const; -}; +} __attribute__((__packed__)); struct EventData { static constexpr size_t PayloadSize = 5; static constexpr size_t PayloadPerGBTword = 10; static constexpr size_t MinNelements = 0; static constexpr size_t MaxNelements = 12; - // - static constexpr int BitFlagPos = 25; // position of first bit flag(numberADC) - - union { - uint64_t word = {0}; - struct { - int64_t time : 12; - int64_t charge : 13; - uint64_t numberADC : 1, // 25 bit - isDoubleEvent : 1, - isTimeInfoNOTvalid : 1, - isCFDinADCgate : 1, - isTimeInfoLate : 1, - isAmpHigh : 1, - isEventInTVDC : 1, - isTimeInfoLost : 1, - reservedField : 3, - channelID : 4; - }; - }; + int16_t time : 12; + int16_t charge : 13; + uint16_t pmBits : 8; + uint8_t reservedField : 3; + uint8_t channelID : 4; void generateFlags() { - numberADC = std::rand() % 2; - isDoubleEvent = 0; - isTimeInfoNOTvalid = 0; - isCFDinADCgate = 1; - isTimeInfoLate = 0; - isAmpHigh = 0; - isEventInTVDC = 1; - isTimeInfoLost = 0; - } - uint8_t getFlagWord() const - { - return uint8_t(word >> BitFlagPos); + /* + numberADC = std::rand() % 2; + isDoubleEvent = 0; + isTimeInfoNOTvalid = 0; + isCFDinADCgate = 1; + isTimeInfoLate = 0; + isAmpHigh = 0; + isEventInTVDC = 1; + isTimeInfoLost = 0; + */ } void print() const; -}; +} __attribute__((__packed__)); struct TCMdata { static constexpr size_t PayloadSize = 10; static constexpr size_t PayloadPerGBTword = 10; - static constexpr size_t MinNelements = 1; + static constexpr size_t MinNelements = 0; static constexpr size_t MaxNelements = 1; + uint64_t orA : 1, // 0 bit (0 byte) orC : 1, // 1 bit sCen : 1, // 2 bit @@ -118,13 +97,11 @@ struct TCMdata { reservedField4 : 1, // 41 bit amplC : 17, // 42 bit. reservedField5 : 1, // 59 bit. - // in standard case(without __atribute__((packed)) macros, or packing by using union) - // here will be empty 4 bits, end next field("timeA") will start from 64 bit. - timeA : 9, // 60 bit - reservedField6 : 1, // 69 bit - timeC : 9, // 70 bit - reservedField7 : 1, // 79 bit - reservedField8 : 48; // 80 bit + timeA : 9, // 60 bit + reservedField6 : 1, // 69 bit + timeC : 9, // 70 bit + reservedField7 : 1, // 79 bit + reservedField8 : 48; // 80 bit void print() const; } __attribute__((__packed__)); @@ -134,11 +111,7 @@ struct TCMdataExtended { static constexpr size_t PayloadPerGBTword = 10; static constexpr size_t MinNelements = 0; static constexpr size_t MaxNelements = 20; - union { - uint32_t word[1] = {}; - uint32_t triggerWord; - }; - + uint32_t triggerWord; void print() const; }; diff --git a/DataFormats/Detectors/FIT/common/include/DataFormatsFIT/Triggers.h b/DataFormats/Detectors/FIT/common/include/DataFormatsFIT/Triggers.h index ec4955b7c1d53..a660d77820207 100644 --- a/DataFormats/Detectors/FIT/common/include/DataFormatsFIT/Triggers.h +++ b/DataFormats/Detectors/FIT/common/include/DataFormatsFIT/Triggers.h @@ -25,10 +25,15 @@ namespace o2 { namespace fit { - class Triggers { public: + template + constexpr static uint64_t word(TrgBits&&... trgBits) + { + return ((1ull << std::forward(trgBits)) | ...); + } + enum { bitA = 0, bitC = 1, // alias of bitAOut (FT0/FDD) bitAOut = 1, // alias of bitC (FV0) @@ -40,7 +45,8 @@ class Triggers bitAIn = 4, // alias of bitVertex (FV0) bitLaser = 5, // indicates the laser was triggered in this BC bitOutputsAreBlocked = 6, // indicates that laser-induced pulses should arrive from detector to FEE in this BC (and trigger outputs are blocked) - bitDataIsValid = 7 }; // data is valid for processing + bitDataIsValid = 7, // data is valid for processing + bitMinBias = 8 }; // extra calculated bit, vrt & (cern || semicent) static const int16_t DEFAULT_TIME = -5000; // for average of one side (A or C) static const int16_t DEFAULT_AMP = 0; static const int16_t DEFAULT_ZERO = 0; @@ -56,6 +62,21 @@ class Triggers timeA = atimeA; timeC = atimeC; } + inline static bool checkMinBiasFT0(uint64_t trgWord) + { + return static_cast(trgWord & word(bitVertex)) && static_cast(trgWord & word(bitSCen, bitCen)); + } + static uint64_t makeExtendedTrgWord(uint64_t trgWord) + { + return trgWord | (static_cast(checkMinBiasFT0(trgWord)) << bitMinBias); + } + static constexpr std::pair parseDigitTriggerWord(uint8_t digitWord, bool shiftTechBitsToBegin = false) + { + const uint8_t techWordMask = word(bitLaser, bitOutputsAreBlocked, bitDataIsValid); + const uint8_t shiftTechWordPos = shiftTechBitsToBegin ? bitLaser : 0; + return {(digitWord & (~techWordMask)), (digitWord & techWordMask) >> shiftTechWordPos}; + } + bool getOrA() const { return (triggersignals & (1 << bitA)) != 0; } bool getOrC() const { return (triggersignals & (1 << bitC)) != 0; } // only used by FT0/FDD (same bit as OrAOut in FV0) bool getOrAOut() const { return (triggersignals & (1 << bitAOut)) != 0; } // only used by FV0 (same bit as OrC in FT0/FDD) @@ -66,9 +87,10 @@ class Triggers bool getVertex() const { return (triggersignals & (1 << bitVertex)) != 0; } // only used by FT0/FDD (same bit as OrAIn in FV0) bool getOrAIn() const { return (triggersignals & (1 << bitAIn)) != 0; } // only used by FV0 (same bit as OrC in FT0/FDD) bool getLaser() const { return (triggersignals & (1 << bitLaser)) != 0; } - bool getLaserBit() const { return getLaser(); } // TODO: remove after QC is modified bool getOutputsAreBlocked() const { return (triggersignals & (1 << bitOutputsAreBlocked)) != 0; } bool getDataIsValid() const { return (triggersignals & (1 << bitDataIsValid)) != 0; } + bool getMinBiasFT0() const { return checkMinBiasFT0(static_cast(triggersignals)); } + uint64_t getExtendedTrgWordFT0() const { return makeExtendedTrgWord(static_cast(triggersignals)); } uint8_t getTriggersignals() const { return triggersignals; } uint8_t getNChanA() const { return nChanA; } @@ -88,6 +110,10 @@ class Triggers timeA = atimeA; timeC = atimeC; } + void setTriggers(uint8_t trgsig) + { + triggersignals = trgsig; + } void setTriggers(Bool_t isA, Bool_t isC, Bool_t isVrtx, Bool_t isCnt, Bool_t isSCnt, uint8_t chanA, uint8_t chanC, int32_t aamplA, int32_t aamplC, int16_t atimeA, int16_t atimeC, Bool_t isLaser, Bool_t isOutputsAreBlocked, Bool_t isDataValid) @@ -114,7 +140,7 @@ class Triggers void print(std::ostream&) const; void printLog() const; - public: // TODO: change to 'private' after modifying QC to use the setters/getters + private: uint8_t triggersignals = DEFAULT_ZERO; // FIT trigger signals uint8_t nChanA = DEFAULT_ZERO; // number of fired channels A side uint8_t nChanC = DEFAULT_ZERO; // number of fired channels A side diff --git a/DataFormats/Detectors/FIT/common/src/DataFormatsFITLinkDef.h b/DataFormats/Detectors/FIT/common/src/DataFormatsFITLinkDef.h index e5f33a0e6728e..e005315e2d6a0 100644 --- a/DataFormats/Detectors/FIT/common/src/DataFormatsFITLinkDef.h +++ b/DataFormats/Detectors/FIT/common/src/DataFormatsFITLinkDef.h @@ -23,4 +23,26 @@ #pragma link C++ class o2::fit::Triggers + ; #pragma link C++ class vector < o2::fit::Triggers> + ; +#pragma link C++ std::vector < std::pair < uint64_t, int>> + ; +#pragma link C++ struct o2::fit::DCSDPValues + ; +/* +#include "DetectorsCommonDataFormats/DetID.h" +#pragma link C++ struct o2::fit::ChannelData + ; +#pragma link C++ struct o2::fit::Digit + ; + +#pragma link C++ struct o2::fit::ChannelData + ; +#pragma link C++ struct o2::fit::Digit + ; + +#pragma link C++ struct o2::fit::ChannelData + ; +#pragma link C++ struct o2::fit::Digit + ; +*/ +// #pragma link C++ struct o2::fit::ChannelData; +// #pragma link C++ struct o2::fit::Digit; + +// TODO AM: Set this here when unused class warning is solved. +// #pragma link C++ class std::unordered_map < o2::dcs::DataPointIdentifier, o2::fit::DCSDPValues> + ; + +// Needed in O2/Detectors/FIT/macros/readFITDCSdata.C +#pragma link C++ class std::map < std::string, o2::fit::DCSDPValues> + ; + #endif diff --git a/DataFormats/Detectors/FIT/common/src/LookUpTable.cxx b/DataFormats/Detectors/FIT/common/src/LookUpTable.cxx new file mode 100644 index 0000000000000..73c0b1bf1bb9e --- /dev/null +++ b/DataFormats/Detectors/FIT/common/src/LookUpTable.cxx @@ -0,0 +1,26 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include "DataFormatsFIT/LookUpTable.h" +#include "CCDB/BasicCCDBManager.h" +#include +using namespace o2::fit; +template +void LookupTableBase::initCCDB(const std::string& urlCCDB, const std::string& pathToStorageInCCDB, long timestamp) +{ + + auto& mgr = o2::ccdb::BasicCCDBManager::instance(); + mgr.setURL(urlCCDB); + mVecEntryFEE = *(mgr.getForTimeStamp::Table_t>(pathToStorageInCCDB, timestamp)); + prepareLUT(); +} +template class o2::fit::LookupTableBase, + std::unordered_map>; diff --git a/DataFormats/Detectors/FIT/common/src/RawDataMetric.cxx b/DataFormats/Detectors/FIT/common/src/RawDataMetric.cxx new file mode 100644 index 0000000000000..2b686afc6cfcd --- /dev/null +++ b/DataFormats/Detectors/FIT/common/src/RawDataMetric.cxx @@ -0,0 +1,38 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include "DataFormatsFIT/RawDataMetric.h" +#include +using namespace o2::fit; +const std::map RawDataMetric::sMapBitsToNames = { + {RawDataMetric::EStatusBits::kIncompletePayload, "IncompletePayload"}, + {RawDataMetric::EStatusBits::kWrongDescriptor, "WrongDescriptor"}, + {RawDataMetric::EStatusBits::kWrongChannelMapping, "WrongChannelMapping"}, + {RawDataMetric::EStatusBits::kEmptyDataBlock, "EmptyDataBlock"}, + {RawDataMetric::EStatusBits::kDecodedDataBlock, "DecodedDataBlock"}}; +void RawDataMetric::print() const +{ + LOG(info) << "=============================================================="; + LOG(info) << "Raw data metric: linkID " << static_cast(mLinkID) << " mEPID " << static_cast(mEPID) << " FEEID " << static_cast(mFEEID); + LOG(info) << "Is registered FEE: " << mIsRegisteredFEE; + for (const auto& entry : sMapBitsToNames) { + LOG(info) << entry.second << ": " << mBitStats[entry.first]; + } + LOG(info) << "=============================================================="; +} +RawDataMetric::Status_t RawDataMetric::getAllBitsActivated() +{ + Status_t metricStatus{}; + for (const auto& entry : sMapBitsToNames) { + metricStatus |= (1 << entry.first); + } + return metricStatus; +} diff --git a/DataFormats/Detectors/FIT/common/src/RawEventData.cxx b/DataFormats/Detectors/FIT/common/src/RawEventData.cxx index 2c04202bab50b..724ebeabe8aaa 100644 --- a/DataFormats/Detectors/FIT/common/src/RawEventData.cxx +++ b/DataFormats/Detectors/FIT/common/src/RawEventData.cxx @@ -32,14 +32,6 @@ void EventData::print() const LOG(info) << "------------Channel " << channelID << "------------"; LOG(info) << "Charge: " << charge; LOG(info) << "Time: " << time; - LOG(info) << "numberADC: " << numberADC; - LOG(info) << "isDoubleEvent: " << isDoubleEvent; - LOG(info) << "isTimeInfoNOTvalid: " << isTimeInfoNOTvalid; - LOG(info) << "isCFDinADCgate: " << isCFDinADCgate; - LOG(info) << "isTimeInfoLate: " << isTimeInfoLate; - LOG(info) << "isAmpHigh: " << isAmpHigh; - LOG(info) << "isEventInTVDC: " << isEventInTVDC; - LOG(info) << "isTimeInfoLost: " << isTimeInfoLost; LOG(info) << "##########################################"; LOG(info) << std::dec; } diff --git a/DataFormats/Detectors/FOCAL/CMakeLists.txt b/DataFormats/Detectors/FOCAL/CMakeLists.txt new file mode 100644 index 0000000000000..f74058046d584 --- /dev/null +++ b/DataFormats/Detectors/FOCAL/CMakeLists.txt @@ -0,0 +1,27 @@ +# Copyright 2019-2020 CERN and copyright holders of ALICE O2. +# See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +# All rights not expressly granted are reserved. +# +# This software is distributed under the terms of the GNU General Public +# License v3 (GPL Version 3), copied verbatim in the file "COPYING". +# +# In applying this license CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization +# or submit itself to any jurisdiction. + +o2_add_library(DataFormatsFOCAL + SOURCES src/Event.cxx + src/ErrorHandling.cxx + src/PixelHit.cxx + src/PixelChip.cxx + src/PixelChipRecord.cxx + src/TriggerRecord.cxx + PUBLIC_LINK_LIBRARIES O2::CommonDataFormat + O2::Headers) + +o2_target_root_dictionary(DataFormatsFOCAL + HEADERS include/DataFormatsFOCAL/Event.h + include/DataFormatsFOCAL/PixelHit.h + include/DataFormatsFOCAL/PixelChip.h + include/DataFormatsFOCAL/PixelChipRecord.h + include/DataFormatsFOCAL/TriggerRecord.h) diff --git a/DataFormats/Detectors/FOCAL/include/DataFormatsFOCAL/Constants.h b/DataFormats/Detectors/FOCAL/include/DataFormatsFOCAL/Constants.h new file mode 100644 index 0000000000000..eb3913c1de9ab --- /dev/null +++ b/DataFormats/Detectors/FOCAL/include/DataFormatsFOCAL/Constants.h @@ -0,0 +1,24 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +#ifndef ALICEO2_FOCAL_CONSTANTS_H +#define ALICEO2_FOCAL_CONSTANTS_H + +namespace o2::focal::constants +{ +constexpr int PADLAYER_MODULE_NCHANNELS = 72; +constexpr int PADLAYER_MODULE_NHALVES = 2; +constexpr int PADS_NLAYERS = 20; +constexpr int PIXELS_NLAYERS = 2; +constexpr int PADLAYER_WINDOW_LENGTH = 20; + +} // namespace o2::focal::constants + +#endif // ALICEO2_FOCAL_CONSTANTS_H \ No newline at end of file diff --git a/DataFormats/Detectors/FOCAL/include/DataFormatsFOCAL/ErrorHandling.h b/DataFormats/Detectors/FOCAL/include/DataFormatsFOCAL/ErrorHandling.h new file mode 100644 index 0000000000000..645e9d0dbcd9a --- /dev/null +++ b/DataFormats/Detectors/FOCAL/include/DataFormatsFOCAL/ErrorHandling.h @@ -0,0 +1,49 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +#ifndef ALICEO2_FOCAL_ERRORHANDLING_H +#define ALICEO2_FOCAL_ERRORHANDLING_H +#include +#include +#include + +namespace o2::focal +{ +class IndexExceptionEvent : public std::exception +{ + public: + enum class IndexType_t { + PAD_LAYER, + PAD_NHALVES, + PAD_CHANNEL, + PIXEL_LAYER, + TRIGGER_WINDOW + }; + IndexExceptionEvent(unsigned int index, unsigned int maxindex, IndexType_t source); + ~IndexExceptionEvent() noexcept final = default; + + const char* what() const noexcept final; + + unsigned int getIndex() const noexcept { return mIndex; } + unsigned int getMaxIndex() const noexcept { return mMaxIndex; } + IndexType_t getSource() const noexcept { return mSource; } + + private: + unsigned int mIndex; + unsigned int mMaxIndex; + IndexType_t mSource; + std::string mMessageBuffer; +}; + +std::ostream& operator<<(std::ostream& in, const IndexExceptionEvent& error); + +} // namespace o2::focal + +#endif \ No newline at end of file diff --git a/DataFormats/Detectors/FOCAL/include/DataFormatsFOCAL/Event.h b/DataFormats/Detectors/FOCAL/include/DataFormatsFOCAL/Event.h new file mode 100644 index 0000000000000..382b1075cb4eb --- /dev/null +++ b/DataFormats/Detectors/FOCAL/include/DataFormatsFOCAL/Event.h @@ -0,0 +1,131 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +#ifndef ALICEO2_FOCAL_EVENT_H +#define ALICEO2_FOCAL_EVENT_H + +#include +#include +#include +#include "Rtypes.h" +#include "CommonDataFormat/InteractionRecord.h" +#include "DataFormatsFOCAL/Constants.h" +#include "DataFormatsFOCAL/PixelChip.h" +#include "DataFormatsFOCAL/PixelChipRecord.h" + +namespace o2::focal +{ + +class PadLayerEvent +{ + public: + struct Header { + uint8_t mHeader; + uint8_t mBC; + uint8_t mWADD; + uint8_t mFourbits; + uint8_t mTrailer; + }; + struct Channel { + uint16_t mADC; + uint16_t mTOA; + uint16_t mTOT; + }; + struct TriggerWindow { + uint32_t mHeader0; + uint32_t mHeader1; + std::array mTriggers; + }; + + void setHeader(unsigned int half, uint8_t header, uint8_t bc, uint8_t wadd, uint8_t fourbits, uint8_t trialer); + void setChannel(unsigned int channel, uint16_t adc, uint16_t toa, uint16_t tot); + void setCMN(unsigned int half, uint16_t adc, uint16_t toa, uint16_t tot); + void setCalib(unsigned int half, uint16_t adc, uint16_t toa, uint16_t tot); + void setTrigger(unsigned int window, uint32_t header0, uint32_t header1, const gsl::span triggers); + + const Header& getHeader(unsigned int half) const; + const Channel& getChannel(unsigned int channel) const; + const Channel& getCMN(unsigned int half) const; + const Channel& getCalib(unsigned int half) const; + const TriggerWindow& getTrigger(unsigned int window) const; + + std::array getADCs() const; + std::array getTOAs() const; + std::array getTOTs() const; + + void reset(); + + private: + void check_halfs(unsigned int half) const; + void check_channel(unsigned int channel) const; + + std::array mHeaders; + std::array mChannels; + std::array mCMN; + std::array mCalib; + std::array mTriggers; + ClassDefNV(PadLayerEvent, 1); +}; + +class PixelLayerEvent +{ + public: + PixelLayerEvent() = default; + ~PixelLayerEvent() = default; + + void addChip(const PixelChip& chip); + void addChip(int feeID, int laneID, int chipID, uint16_t statusCode, gsl::span hits); + const std::vector& getChips() const { return mChips; } + + void reset(); + + private: + std::vector mChips; + ClassDefNV(PixelLayerEvent, 1); +}; + +class Event +{ + public: + Event() = default; + ~Event() = default; + + PadLayerEvent& getPadLayer(unsigned int index); + const PadLayerEvent& getPadLayer(unsigned int index) const; + void setPadLayer(unsigned int layer, const PadLayerEvent& event); + + PixelLayerEvent& getPixelLayer(unsigned int index); + const PixelLayerEvent& getPixelLayer(unsigned int index) const; + void setPixelLayerEvent(unsigned int layer, const PixelLayerEvent& event); + + const InteractionRecord& getInteractionRecord() const { return mInteractionRecord; } + void setInteractionRecord(const InteractionRecord& ir) { mInteractionRecord = ir; } + + void reset(); + + void construct(const o2::InteractionRecord& interaction, gsl::span pads, gsl::span eventPixels, gsl::span pixelHits); + + bool isInitialized() const { return mInitialized; } + + private: + void check_pad_layers(unsigned int index) const; + void check_pixel_layers(unsigned int index) const; + + InteractionRecord mInteractionRecord; + std::array mPadLayers; + std::array mPixelLayers; + bool mInitialized = false; + + ClassDefNV(Event, 1); +}; + +} // namespace o2::focal + +#endif // ALICEO2_FOCAL_EVENT_H \ No newline at end of file diff --git a/DataFormats/Detectors/FOCAL/include/DataFormatsFOCAL/PixelChip.h b/DataFormats/Detectors/FOCAL/include/DataFormatsFOCAL/PixelChip.h new file mode 100644 index 0000000000000..804f87ff211c9 --- /dev/null +++ b/DataFormats/Detectors/FOCAL/include/DataFormatsFOCAL/PixelChip.h @@ -0,0 +1,113 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +#ifndef ALICEO2_FOCAL_PIXELCHIP_H +#define ALICEO2_FOCAL_PIXELCHIP_H + +#include +#include +#include +#include "Rtypes.h" +#include + +namespace o2::focal +{ + +struct PixelChip { + static constexpr uint16_t DATAFRAME = 0x1 << 0; + static constexpr uint16_t EMPTYFRAME = 0x1 << 1; + static constexpr uint16_t BUSY_ON = 0x1 << 2; + static constexpr uint16_t BUSY_OFF = 0x1 << 3; + + uint8_t mFeeID; + uint8_t mLaneID; + uint8_t mChipID; + uint16_t mStatusCode; + std::vector mHits; + + bool operator==(const PixelChip& other) const { return mChipID == other.mChipID && mLaneID == other.mLaneID && mFeeID == other.mFeeID; } + bool operator<(const PixelChip& other) const + { + if (mFeeID < other.mFeeID) { + return true; + } else if (mFeeID == other.mFeeID) { + if (mLaneID < other.mLaneID) { + return true; + } else if ((mLaneID == other.mLaneID) && (mChipID < other.mChipID)) { + return true; + } else { + return false; + } + } else { + return false; + } + } + + void setDataframe() + { + mStatusCode |= DATAFRAME; + } + + void setEmptyframe() + { + mStatusCode |= EMPTYFRAME; + } + + void setBusyOn() + { + mStatusCode |= BUSY_ON; + } + + void removeDataframe() + { + mStatusCode &= ~(DATAFRAME); + } + + void removeEmptyframe() + { + mStatusCode &= ~(EMPTYFRAME); + } + + void removeBusyOn() + { + mStatusCode &= ~(BUSY_ON); + } + + void removeBusyOff() + { + mStatusCode &= ~(BUSY_ON); + } + + bool isDataframe() const + { + return mStatusCode & DATAFRAME; + } + + bool isEmptyframe() const + { + return mStatusCode & EMPTYFRAME; + } + + bool isBusyOn() const + { + return mStatusCode & BUSY_ON; + } + + bool isBusyOff() const + { + return mStatusCode & BUSY_OFF; + } + + ClassDefNV(PixelChip, 1); +}; + +std::ostream& operator<<(std::ostream& stream, const PixelChip& chip); +} // namespace o2::focal +#endif // QC_MODULE_FOCAL_PIXELCHIP_H \ No newline at end of file diff --git a/DataFormats/Detectors/FOCAL/include/DataFormatsFOCAL/PixelChipRecord.h b/DataFormats/Detectors/FOCAL/include/DataFormatsFOCAL/PixelChipRecord.h new file mode 100644 index 0000000000000..6a16675dd4e1c --- /dev/null +++ b/DataFormats/Detectors/FOCAL/include/DataFormatsFOCAL/PixelChipRecord.h @@ -0,0 +1,63 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +#ifndef ALICEO2_FOCAL_PIXELCHIPRECORD_H +#define ALICEO2_FOCAL_PIXELCHIPRECORD_H + +#include +#include +#include "Rtypes.h" +#include "CommonDataFormat/RangeReference.h" + +namespace o2::focal +{ +class PixelChipRecord +{ + using DataRange = o2::dataformats::RangeReference; + + public: + PixelChipRecord() = default; + PixelChipRecord(int layerID, int feeID, int laneID, int chipID, uint32_t statusCode, int firsthit, int nhits) : mLayerID(layerID), mLaneID(laneID), mChipID(chipID), mFeeID(feeID), mStatusCode(statusCode), mHitIndexRange(firsthit, nhits) {} + ~PixelChipRecord() = default; + + void setLayerID(int layerID) { mLayerID = layerID; } + void setLaneID(int laneID) { mLaneID = laneID; } + void setChipID(int chipID) { mChipID = chipID; } + void setFeeID(int feeID) { mFeeID = feeID; } + void setStatusCode(uint16_t statusCode) { mStatusCode = statusCode; } + void setIndexFirstHit(int firsthit) { mHitIndexRange.setFirstEntry(firsthit); } + void setNumberOfHits(int nhits) { mHitIndexRange.setEntries(nhits); } + + int getLayerID() const { return mLayerID; } + int getLaneID() const { return mLaneID; } + int getChipID() const { return mChipID; } + int getFeeID() const { return mFeeID; } + uint16_t getStatusCode() const { return mStatusCode; } + int getFirstHit() const { return mHitIndexRange.getFirstEntry(); } + int getNumberOfHits() const { return mHitIndexRange.getEntries(); } + + void printStream(std::ostream& stream) const; + + private: + int mLayerID = -1; /// Layer index + int mLaneID = -1; /// Lane index + int mChipID = -1; /// Chip index + int mFeeID = -1; /// FEE ID + uint16_t mStatusCode = 0; /// status codes + DataRange mHitIndexRange; /// Index range of hits belonging to theh chip + + ClassDefNV(PixelChipRecord, 1); +}; + +std::ostream& operator<<(std::ostream& stream, const PixelChipRecord& trg); + +}; // namespace o2::focal + +#endif \ No newline at end of file diff --git a/DataFormats/Detectors/FOCAL/include/DataFormatsFOCAL/PixelHit.h b/DataFormats/Detectors/FOCAL/include/DataFormatsFOCAL/PixelHit.h new file mode 100644 index 0000000000000..f4d2984a69c5d --- /dev/null +++ b/DataFormats/Detectors/FOCAL/include/DataFormatsFOCAL/PixelHit.h @@ -0,0 +1,34 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +#ifndef ALICEO2_FOCAL_PIXELHIT_H +#define ALICEO2_FOCAL_PIXELHIT_H + +#include +#include +#include "Rtypes.h" + +namespace o2::focal +{ + +struct PixelHit { + uint16_t mColumn; + uint16_t mRow; + + bool operator==(const PixelHit& other) const { return mColumn == other.mColumn && mRow == other.mRow; } + bool operator<(const PixelHit& other) const; + + ClassDefNV(PixelHit, 1); +}; + +std::ostream& operator<<(std::ostream& stream, const PixelHit& hit); + +} // namespace o2::focal +#endif // ALICEO2_FOCAL_PIXELHIT_H \ No newline at end of file diff --git a/DataFormats/Detectors/FOCAL/include/DataFormatsFOCAL/TriggerRecord.h b/DataFormats/Detectors/FOCAL/include/DataFormatsFOCAL/TriggerRecord.h new file mode 100644 index 0000000000000..dbb10613b5654 --- /dev/null +++ b/DataFormats/Detectors/FOCAL/include/DataFormatsFOCAL/TriggerRecord.h @@ -0,0 +1,67 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +#ifndef ALICEO2_FOCAL_TRIGGERRECORD_H +#define ALICEO2_FOCAL_TRIGGERRECORD_H + +#include +#include +#include "Rtypes.h" +#include "CommonDataFormat/InteractionRecord.h" +#include "CommonDataFormat/RangeReference.h" + +namespace o2::focal +{ +class TriggerRecord +{ + using BCData = o2::InteractionRecord; + using DataRange = o2::dataformats::RangeReference; + + public: + TriggerRecord() = default; + TriggerRecord(const BCData& bunchcrossing, int firstpadentry, int npadentries, int firstchipentry, int nchipentries, int firsthitentry, int nhitentries) : mBCData(bunchcrossing), mPadDataRange(firstpadentry, npadentries), mPixelChipRange(firstchipentry, nchipentries), mPixelHitRange(firsthitentry, nhitentries) {} + ~TriggerRecord() = default; + + void setBCData(const BCData& data) { mBCData = data; } + void setPadDataRange(int firstentry, int nentries) { mPadDataRange.set(firstentry, nentries); } + void setPixelChipDataRange(int firstentry, int nentries) { mPixelChipRange.set(firstentry, nentries); } + void setPixelHitDataRange(int firstentry, int nentries) { mPixelHitRange.set(firstentry, nentries); } + void setIndexFirstPadObject(int firstentry) { mPadDataRange.setFirstEntry(firstentry); } + void setIndexFirstPixelChipObject(int firstentry) { mPixelChipRange.setFirstEntry(firstentry); } + void setIndexFirstPixelHitObject(int firstentry) { mPixelHitRange.setFirstEntry(firstentry); } + void setNumberOfPadObjects(int nentries) { mPadDataRange.setEntries(nentries); } + void setNumberOfPixelChipObjects(int nentries) { mPixelChipRange.setEntries(nentries); } + void setNumberOfPixelHitObjects(int nentries) { mPixelHitRange.setEntries(nentries); } + + const BCData& getBCData() const { return mBCData; } + BCData& getBCData() { return mBCData; } + int getNumberOfPadObjects() const { return mPadDataRange.getEntries(); } + int getNumberOfPixelChipObjects() const { return mPixelChipRange.getEntries(); } + int getNumberOfPixelHitObjects() const { return mPixelHitRange.getEntries(); } + int getFirstPadEntry() const { return mPadDataRange.getFirstEntry(); } + int getFirstPixelChipEntry() const { return mPixelChipRange.getFirstEntry(); } + int getFirstPixelHitEntry() const { return mPixelHitRange.getFirstEntry(); } + + void printStream(std::ostream& stream) const; + + private: + BCData mBCData; /// Bunch crossing data of the trigger + DataRange mPadDataRange; /// Index range of the pad data for the same trigger (number of pads and first entry in the container) + DataRange mPixelChipRange; /// Index range of the pixel chips in the same trigger (number of chips and first entry in the container) + DataRange mPixelHitRange; /// Index range of the pixel hits in the same trigger (number of hits and first entry in the container) + + ClassDefNV(TriggerRecord, 1); +}; + +std::ostream& operator<<(std::ostream& stream, const TriggerRecord& trg); + +}; // namespace o2::focal + +#endif \ No newline at end of file diff --git a/DataFormats/Detectors/FOCAL/src/DataFormatsFOCALLinkDef.h b/DataFormats/Detectors/FOCAL/src/DataFormatsFOCALLinkDef.h new file mode 100644 index 0000000000000..f4dda04362f38 --- /dev/null +++ b/DataFormats/Detectors/FOCAL/src/DataFormatsFOCALLinkDef.h @@ -0,0 +1,35 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifdef __CLING__ + +#pragma link off all globals; +#pragma link off all classes; +#pragma link off all functions; + +#pragma link C++ class o2::focal::Event + ; +#pragma link C++ class o2::focal::PadLayerEvent + ; +#pragma link C++ class o2::focal::PadLayerEvent::Header + ; +#pragma link C++ class o2::focal::PadLayerEvent::Channel + ; +#pragma link C++ class o2::focal::PadLayerEvent::TriggerWindow + ; +#pragma link C++ class o2::focal::PixelLayerEvent + ; +#pragma link C++ class o2::focal::PixelHit + ; +#pragma link C++ class o2::focal::PixelChip + ; +#pragma link C++ class o2::focal::PixelChipRecord + ; +#pragma link C++ class o2::focal::TriggerRecord + ; + +#pragma link C++ class std::vector < o2::focal::Event> + ; +#pragma link C++ class std::vector < o2::focal::PadLayerEvent> + ; +#pragma link C++ class std::vector < o2::focal::PixelHit> + ; +#pragma link C++ class std::vector < o2::focal::PixelChip> + ; +#pragma link C++ class std::vector < o2::focal::PixelChipRecord> + ; +#pragma link C++ class std::vector < o2::focal::TriggerRecord> + ; +#endif diff --git a/DataFormats/Detectors/FOCAL/src/ErrorHandling.cxx b/DataFormats/Detectors/FOCAL/src/ErrorHandling.cxx new file mode 100644 index 0000000000000..8e028ffe795f7 --- /dev/null +++ b/DataFormats/Detectors/FOCAL/src/ErrorHandling.cxx @@ -0,0 +1,50 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +#include +#include +#include "DataFormatsFOCAL/ErrorHandling.h" + +using namespace o2::focal; + +IndexExceptionEvent::IndexExceptionEvent(unsigned int index, unsigned int maxindex, IndexType_t source) +{ + std::string nameIndexType; + switch (source) { + case IndexType_t::PAD_CHANNEL: + nameIndexType = "channel"; + break; + case IndexType_t::PAD_LAYER: + nameIndexType = "pad layer"; + break; + case IndexType_t::PAD_NHALVES: + nameIndexType = "pad half stave"; + break; + case IndexType_t::PIXEL_LAYER: + nameIndexType = "pixel layer"; + break; + case IndexType_t::TRIGGER_WINDOW: + nameIndexType = "trigger window"; + default: + break; + } + mMessageBuffer = "Accessing invalid entry of type " + nameIndexType + ": " + std::to_string(index) + " ( max " + std::to_string(maxindex) + ")"; +} + +const char* IndexExceptionEvent::what() const noexcept +{ + return mMessageBuffer.data(); +} + +std::ostream& o2::focal::operator<<(std::ostream& stream, const o2::focal::IndexExceptionEvent& error) +{ + stream << error.what(); + return stream; +} \ No newline at end of file diff --git a/DataFormats/Detectors/FOCAL/src/Event.cxx b/DataFormats/Detectors/FOCAL/src/Event.cxx new file mode 100644 index 0000000000000..89bb59d820486 --- /dev/null +++ b/DataFormats/Detectors/FOCAL/src/Event.cxx @@ -0,0 +1,287 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +#include +#include +#include +#include + +using namespace o2::focal; + +PadLayerEvent& Event::getPadLayer(unsigned int index) +{ + check_pad_layers(index); + return mPadLayers[index]; +} + +const PadLayerEvent& Event::getPadLayer(unsigned int index) const +{ + check_pad_layers(index); + return mPadLayers[index]; +} + +PixelLayerEvent& Event::getPixelLayer(unsigned int index) +{ + check_pixel_layers(index); + return mPixelLayers[index]; +} +const PixelLayerEvent& Event::getPixelLayer(unsigned int index) const +{ + check_pixel_layers(index); + return mPixelLayers[index]; +} + +void Event::setPadLayer(unsigned int layer, const PadLayerEvent& event) +{ + check_pad_layers(layer); + mPadLayers[layer] = event; +} + +void Event::setPixelLayerEvent(unsigned int layer, const PixelLayerEvent& event) +{ + check_pixel_layers(layer); + mPixelLayers[layer] = event; +} + +void Event::reset() +{ + mInitialized = false; + for (auto& padlayer : mPadLayers) { + padlayer.reset(); + } + for (auto& pixellayer : mPixelLayers) { + pixellayer.reset(); + } +} + +void Event::construct(const o2::InteractionRecord& interaction, gsl::span pads, gsl::span eventPixels, gsl::span pixelHits) +{ + reset(); + mInteractionRecord = interaction; + int ilayer = 0; + for (auto& padlayer : pads) { + mPadLayers[ilayer] = padlayer; + ilayer++; + } + + int currentlast = 0; + for (auto& chip : eventPixels) { + if (chip.getLayerID() > 1) { + std::cerr << "Invalid layer ID chip " << chip.getChipID() << ": " << chip.getLayerID() << std::endl; + continue; + } + if (chip.getNumberOfHits()) { + if (chip.getFirstHit() >= pixelHits.size()) { + std::cerr << "First hit index " << chip.getFirstHit() << " exceeding hit contiainer " << pixelHits.size() << std::endl; + continue; + } + if (chip.getFirstHit() + chip.getNumberOfHits() - 1 >= pixelHits.size()) { + std::cerr << "First hit index " << chip.getFirstHit() + chip.getNumberOfHits() - 1 << " exceeding hit contiainer " << pixelHits.size() << std::endl; + continue; + } + mPixelLayers[chip.getLayerID()].addChip(chip.getFeeID(), chip.getLaneID(), chip.getChipID(), chip.getStatusCode(), pixelHits.subspan(chip.getFirstHit(), chip.getNumberOfHits())); + } + currentlast = chip.getFirstHit() + chip.getNumberOfHits(); + } + if (currentlast < pixelHits.size()) { + std::cerr << "Inconsistent number of hits / event : all chips -> " << currentlast << " hits, container size -> " << pixelHits.size() << std::endl; + } + mInitialized = true; +} + +void Event::check_pad_layers(unsigned int index) const +{ + if (index >= constants::PADS_NLAYERS) { + throw IndexExceptionEvent(index, constants::PADS_NLAYERS, IndexExceptionEvent::IndexType_t::PAD_LAYER); + } +} + +void Event::check_pixel_layers(unsigned int index) const +{ + if (index >= constants::PIXELS_NLAYERS) { + throw IndexExceptionEvent(index, constants::PIXELS_NLAYERS, IndexExceptionEvent::IndexType_t::PIXEL_LAYER); + } +} + +void PadLayerEvent::setHeader(unsigned int half, uint8_t header, uint8_t bc, uint8_t wadd, uint8_t fourbits, uint8_t trailer) +{ + check_halfs(half); + auto& asicheader = mHeaders[half]; + asicheader.mHeader = header; + asicheader.mBC = bc; + asicheader.mFourbits = fourbits; + asicheader.mWADD = wadd; + asicheader.mTrailer = trailer; +} + +void PadLayerEvent::setChannel(unsigned int channel, uint16_t adc, uint16_t toa, uint16_t tot) +{ + check_channel(channel); + auto& asicchannel = mChannels[channel]; + asicchannel.mADC = adc; + asicchannel.mTOA = toa; + asicchannel.mTOT = tot; +} + +void PadLayerEvent::setCMN(unsigned int channel, uint16_t adc, uint16_t toa, uint16_t tot) +{ + check_halfs(channel); + auto& cmn = mCMN[channel]; + cmn.mADC = adc; + cmn.mTOA = toa; + cmn.mTOT = tot; +} + +void PadLayerEvent::setCalib(unsigned int channel, uint16_t adc, uint16_t toa, uint16_t tot) +{ + check_halfs(channel); + auto& calib = mCalib[channel]; + calib.mADC = adc; + calib.mTOA = toa; + calib.mTOT = tot; +} + +void PadLayerEvent::setTrigger(unsigned int window, uint32_t header0, uint32_t header1, const gsl::span triggers) +{ + if (window >= constants::PADLAYER_WINDOW_LENGTH) { + throw IndexExceptionEvent(window, constants::PADLAYER_WINDOW_LENGTH, IndexExceptionEvent::IndexType_t::TRIGGER_WINDOW); + } + auto& currenttrigger = mTriggers[window]; + currenttrigger.mHeader0 = header0; + currenttrigger.mHeader1 = header1; + std::copy(triggers.begin(), triggers.end(), currenttrigger.mTriggers.begin()); +} + +const PadLayerEvent::Header& PadLayerEvent::getHeader(unsigned int half) const +{ + check_halfs(half); + return mHeaders[half]; +} + +const PadLayerEvent::Channel& PadLayerEvent::getChannel(unsigned int channel) const +{ + check_channel(channel); + return mChannels[channel]; +} +const PadLayerEvent::Channel& PadLayerEvent::getCMN(unsigned int half) const +{ + check_halfs(half); + return mCMN[half]; +} +const PadLayerEvent::Channel& PadLayerEvent::getCalib(unsigned int half) const +{ + check_halfs(half); + return mCalib[half]; +} + +const PadLayerEvent::TriggerWindow& PadLayerEvent::getTrigger(unsigned int window) const +{ + if (window >= constants::PADLAYER_WINDOW_LENGTH) { + throw IndexExceptionEvent(window, constants::PADLAYER_WINDOW_LENGTH, IndexExceptionEvent::IndexType_t::TRIGGER_WINDOW); + } + return mTriggers[window]; +} + +std::array PadLayerEvent::getADCs() const +{ + std::array adcs; + for (std::size_t ichan = 0; ichan < constants::PADLAYER_MODULE_NCHANNELS; ichan++) { + adcs[ichan] = mChannels[ichan].mADC; + } + return adcs; +} +std::array PadLayerEvent::getTOAs() const +{ + std::array toas; + for (std::size_t ichan = 0; ichan < constants::PADLAYER_MODULE_NCHANNELS; ichan++) { + toas[ichan] = mChannels[ichan].mTOA; + } + return toas; +} +std::array PadLayerEvent::getTOTs() const +{ + std::array tots; + for (std::size_t ichan = 0; ichan < constants::PADLAYER_MODULE_NCHANNELS; ichan++) { + tots[ichan] = mChannels[ichan].mTOT; + } + return tots; +} + +void PadLayerEvent::reset() +{ + for (auto& header : mHeaders) { + header.mBC = 0; + header.mHeader = 0; + header.mFourbits = 0; + header.mWADD = 0; + header.mTrailer = 0; + } + for (auto& chan : mChannels) { + chan.mADC = 0; + chan.mTOA = 0; + chan.mTOT = 0; + } + for (auto& calib : mCalib) { + calib.mADC = 0; + calib.mTOA = 0; + calib.mTOT = 0; + } + for (auto& cmn : mCMN) { + cmn.mADC = 0; + cmn.mTOA = 0; + cmn.mTOT = 0; + } + for (auto& trg : mTriggers) { + trg.mHeader0 = 0; + trg.mHeader1 = 0; + std::fill(trg.mTriggers.begin(), trg.mTriggers.end(), 0); + } +} + +void PadLayerEvent::check_halfs(unsigned int half) const +{ + if (half >= constants::PADLAYER_MODULE_NHALVES) { + throw IndexExceptionEvent(half, constants::PADLAYER_MODULE_NHALVES, IndexExceptionEvent::IndexType_t::PAD_NHALVES); + } +} + +void PadLayerEvent::check_channel(unsigned int channel) const +{ + if (channel >= constants::PADLAYER_MODULE_NCHANNELS) { + throw IndexExceptionEvent(channel, constants::PADLAYER_MODULE_NCHANNELS, IndexExceptionEvent::IndexType_t::PAD_CHANNEL); + } +} + +void PixelLayerEvent::addChip(const PixelChip& chip) +{ + auto found = std::find_if(mChips.begin(), mChips.end(), [&chip](const PixelChip& testchip) { return chip == testchip; }); + if (found != mChips.end()) { + std::copy(chip.mHits.begin(), chip.mHits.end(), std::back_inserter(found->mHits)); + } else { + mChips.push_back(chip); + } +} + +void PixelLayerEvent::addChip(int feeID, int laneID, int chipID, uint16_t statusCode, gsl::span hits) +{ + auto found = std::find_if(mChips.begin(), mChips.end(), [laneID, chipID, feeID](const PixelChip& testchip) { return chipID == testchip.mChipID && laneID == testchip.mLaneID && feeID == testchip.mFeeID; }); + if (found != mChips.end()) { + std::copy(hits.begin(), hits.end(), std::back_inserter(found->mHits)); + } else { + mChips.push_back({static_cast(feeID), static_cast(laneID), static_cast(chipID), statusCode}); + auto& currentchip = mChips.back(); + std::copy(hits.begin(), hits.end(), std::back_inserter(currentchip.mHits)); + } +} + +void PixelLayerEvent::reset() +{ + mChips.clear(); +} diff --git a/DataFormats/Detectors/FOCAL/src/PixelChip.cxx b/DataFormats/Detectors/FOCAL/src/PixelChip.cxx new file mode 100644 index 0000000000000..e4c05020fb0f8 --- /dev/null +++ b/DataFormats/Detectors/FOCAL/src/PixelChip.cxx @@ -0,0 +1,21 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include +#include + +using namespace o2::focal; + +std::ostream& o2::focal::operator<<(std::ostream& stream, const PixelChip& chip) +{ + stream << "Chip " << static_cast(chip.mChipID) << " (lane " << static_cast(chip.mLaneID) << ", fee " << chip.mFeeID << "), found " << chip.mHits.size() << " hits"; + return stream; +} \ No newline at end of file diff --git a/DataFormats/Detectors/FOCAL/src/PixelChipRecord.cxx b/DataFormats/Detectors/FOCAL/src/PixelChipRecord.cxx new file mode 100644 index 0000000000000..7229051e70c57 --- /dev/null +++ b/DataFormats/Detectors/FOCAL/src/PixelChipRecord.cxx @@ -0,0 +1,27 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include +#include +#include "DataFormatsFOCAL/PixelChipRecord.h" + +using namespace o2::focal; + +void PixelChipRecord::printStream(std::ostream& stream) const +{ + stream << "Chip " << mChipID << " in layer " << mLayerID << " (lane " << mLaneID << "), number of hits: " << getNumberOfHits() << " (first hit: " << getFirstHit() << ")"; +} + +std::ostream& o2::focal::operator<<(std::ostream& stream, const PixelChipRecord& chip) +{ + chip.printStream(stream); + return stream; +} \ No newline at end of file diff --git a/DataFormats/Detectors/FOCAL/src/PixelHit.cxx b/DataFormats/Detectors/FOCAL/src/PixelHit.cxx new file mode 100644 index 0000000000000..aa8aaded3a9b4 --- /dev/null +++ b/DataFormats/Detectors/FOCAL/src/PixelHit.cxx @@ -0,0 +1,33 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include +#include + +using namespace o2::focal; + +bool PixelHit::operator<(const PixelHit& other) const +{ + if (mColumn < other.mColumn) { + return true; + } else if (mColumn == other.mColumn) { + if (mRow < other.mRow) { + return true; + } + } + return false; +} + +std::ostream& o2::focal::operator<<(std::ostream& stream, const PixelHit& hit) +{ + stream << "Pixel hit col " << hit.mColumn << ", row " << hit.mRow; + return stream; +} diff --git a/DataFormats/Detectors/FOCAL/src/TriggerRecord.cxx b/DataFormats/Detectors/FOCAL/src/TriggerRecord.cxx new file mode 100644 index 0000000000000..a2065c8627e9e --- /dev/null +++ b/DataFormats/Detectors/FOCAL/src/TriggerRecord.cxx @@ -0,0 +1,31 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include +#include +#include "DataFormatsFOCAL/TriggerRecord.h" +#include "CommonConstants/Triggers.h" + +using namespace o2::focal; + +void TriggerRecord::printStream(std::ostream& stream) const +{ + stream << "Data for bc " << getBCData().bc << ", orbit " << getBCData().orbit + << ", [Pixels Chips] starting from entry " << getFirstPixelChipEntry() << " with " << getNumberOfPixelChipObjects() + << ", [Pixels Hits] starting from entry " << getFirstPixelHitEntry() << " with " << getNumberOfPixelHitObjects() + << " objects, [Pads] starting from entry " << getFirstPadEntry() << " with " << getNumberOfPadObjects(); +} + +std::ostream& o2::focal::operator<<(std::ostream& stream, const TriggerRecord& trg) +{ + trg.printStream(stream); + return stream; +} \ No newline at end of file diff --git a/DataFormats/Detectors/GlobalTracking/CMakeLists.txt b/DataFormats/Detectors/GlobalTracking/CMakeLists.txt index 35976a3d0e39a..b219de73f5b47 100644 --- a/DataFormats/Detectors/GlobalTracking/CMakeLists.txt +++ b/DataFormats/Detectors/GlobalTracking/CMakeLists.txt @@ -9,10 +9,13 @@ # granted to it by virtue of its status as an Intergovernmental Organization # or submit itself to any jurisdiction. +#add_compile_options(-O0 -g -fPIC -fno-omit-frame-pointer) + o2_add_library( DataFormatsGlobalTracking SOURCES src/RecoContainer.cxx src/FilteredRecoTF.cxx + src/TrackTuneParams.cxx PUBLIC_LINK_LIBRARIES O2::DataFormatsTPC O2::DataFormatsITSMFT @@ -33,11 +36,13 @@ o2_add_library( O2::DataFormatsCPV O2::DataFormatsPHOS O2::DataFormatsEMCAL - O2::GPUDataTypeHeaders + O2::GPUDataTypes + $<$:O2::ITS3Reconstruction> PRIVATE_LINK_LIBRARIES O2::Framework) o2_target_root_dictionary( DataFormatsGlobalTracking HEADERS include/DataFormatsGlobalTracking/FilteredRecoTF.h + include/DataFormatsGlobalTracking/TrackTuneParams.h ) diff --git a/DataFormats/Detectors/GlobalTracking/include/DataFormatsGlobalTracking/RecoContainer.h b/DataFormats/Detectors/GlobalTracking/include/DataFormatsGlobalTracking/RecoContainer.h index c15db02f9d8e4..31d531ef19265 100644 --- a/DataFormats/Detectors/GlobalTracking/include/DataFormatsGlobalTracking/RecoContainer.h +++ b/DataFormats/Detectors/GlobalTracking/include/DataFormatsGlobalTracking/RecoContainer.h @@ -19,12 +19,14 @@ #include "CommonDataFormat/InteractionRecord.h" #include "ReconstructionDataFormats/GlobalTrackAccessor.h" #include "CommonDataFormat/RangeReference.h" +#include "ReconstructionDataFormats/Decay3Body.h" #include "ReconstructionDataFormats/GlobalTrackID.h" #include "ReconstructionDataFormats/MatchingType.h" #include "CommonDataFormat/AbstractRefAccessor.h" #include "SimulationDataFormat/MCCompLabel.h" #include "SimulationDataFormat/MCTruthContainer.h" #include "SimulationDataFormat/ConstMCTruthContainer.h" +#include "DataFormatsCTP/LumiInfo.h" #include #include @@ -35,6 +37,7 @@ namespace o2::tpc class TrackTPC; using TPCClRefElem = uint32_t; struct ClusterNativeAccess; +struct TriggerInfoDLBZS; namespace internal { struct getWorkflowTPCInput_ret; @@ -161,7 +164,12 @@ class PrimaryVertex; class VtxTrackIndex; class VtxTrackRef; class V0; +class V0Index; class Cascade; +class CascadeIndex; +class Decay3Body; +class Decay3BodyIndex; +class StrangeTrack; class TrackCosmics; class GlobalFwdTrack; class MatchInfoFwd; @@ -195,7 +203,7 @@ struct DataRequest { bool isRequested(const std::string& t) const { return !t.empty() && requestMap.find(t) != requestMap.end(); } void requestTracks(o2::dataformats::GlobalTrackID::mask_t src, bool mc); - void requestClusters(o2::dataformats::GlobalTrackID::mask_t src, bool useMC); + void requestClusters(o2::dataformats::GlobalTrackID::mask_t src, bool useMC, o2::detectors::DetID::mask_t skipDetClusters = {}); void requestITSTracks(bool mc); void requestMFTTracks(bool mc); @@ -217,12 +225,14 @@ struct DataRequest { void requestITSClusters(bool mc); void requestMFTClusters(bool mc); void requestTPCClusters(bool mc); + void requestTPCOccMap(); + void requestTPCTriggers(); void requestTOFClusters(bool mc); void requestTRDTracklets(bool mc); void requestMCHClusters(bool mc); void requestMIDClusters(bool mc); void requestHMPClusters(bool mc); - // void requestHMPMatches(bool mc); // no input available at the moment + void requestHMPMatches(bool mc); // no input available at the moment void requestCTPDigits(bool mc); @@ -232,11 +242,16 @@ struct DataRequest { void requestCoscmicTracks(bool mc); - void requestPrimaryVertertices(bool mc); - void requestPrimaryVerterticesTMP(bool mc); - void requestSecondaryVertertices(bool mc); + void requestPrimaryVertices(bool mc); + void requestPrimaryVerticesTMP(bool mc); + void requestSecondaryVertices(bool mc); + void requestStrangeTracks(bool mc); void requestIRFramesITS(); + +#ifdef ENABLE_UPGRADES + void requestIT3Clusters(bool mc); +#endif }; // Helper class to requested data. @@ -276,12 +291,24 @@ struct RecoContainer { NPVTXSLOTS }; // slots to register secondary vertex data - enum SVTXSlots { V0S, // V0 objects - PVTX_V0REFS, // PV -> V0 references - CASCS, // Cascade objects - PVTX_CASCREFS, // PV -> Cascade reference + enum SVTXSlots { V0SIDX, // V0s indices + V0S, // V0 objects + PVTX_V0REFS, // PV -> V0 references + CASCSIDX, // Cascade indices + CASCS, // Cascade objects + PVTX_CASCREFS, // PV -> Cascade reference + DECAY3BODYIDX, // 3-body decay indices + DECAY3BODY, // 3-body decay objects + PVTX_3BODYREFS, // PV -> 3-body decay references NSVTXSLOTS }; + // slots to register strangeness tracking data + enum STRKSlots { + STRACK, + STRACK_MC, + NSTRKSLOTS + }; + // slots for cosmics enum CosmicsSlots { COSM_TRACKS, COSM_TRACKS_MC, @@ -290,15 +317,19 @@ struct RecoContainer { using AccSlots = o2::dataformats::AbstractRefAccessor; // int here is a dummy placeholder using PVertexAccessor = o2::dataformats::AbstractRefAccessor; using SVertexAccessor = o2::dataformats::AbstractRefAccessor; + using STrackAccessor = o2::dataformats::AbstractRefAccessor; using CosmicsAccessor = o2::dataformats::AbstractRefAccessor; using GTrackID = o2::dataformats::GlobalTrackID; using GlobalIDSet = std::array; + static constexpr float PS2MUS = 1e-6; + o2::InteractionRecord startIR; // TF start IR std::array commonPool; PVertexAccessor pvtxPool; // containers for primary vertex related objects SVertexAccessor svtxPool; // containers for secondary vertex related objects + STrackAccessor strkPool; // containers for strangeness tracking related objects CosmicsAccessor cosmPool; // containers for cosmics track data std::unique_ptr> mcITSClusters; @@ -311,8 +342,10 @@ struct RecoContainer { std::unique_ptr> mcMIDTrackClusters; std::unique_ptr> mcMIDClusters; std::unique_ptr> mcMIDTracks; + o2::ctp::LumiInfo mCTPLumi; gsl::span clusterShMapTPC; ///< externally set TPC clusters sharing map + gsl::span occupancyMapTPC; ///< externally set TPC clusters occupancy map std::unique_ptr inputsTPCclusters; // special struct for TPC clusters access std::unique_ptr inputsTRD; // special struct for TRD tracklets, trigger records @@ -320,7 +353,7 @@ struct RecoContainer { void collectData(o2::framework::ProcessingContext& pc, const DataRequest& request); void createTracks(std::function const& creator) const; template - void createTracksVariadic(T creator) const; + void createTracksVariadic(T creator, GTrackID::mask_t srcSel = GTrackID::getSourcesMask("all")) const; void fillTrackMCLabels(const gsl::span gids, std::vector& mcinfo) const; void addITSTracks(o2::framework::ProcessingContext& pc, bool mc); @@ -338,14 +371,15 @@ struct RecoContainer { void addTOFMatchesTPCTRD(o2::framework::ProcessingContext& pc, bool mc); void addTOFMatchesITSTPCTRD(o2::framework::ProcessingContext& pc, bool mc); - // void addHMPMatches(o2::framework::ProcessingContext& pc, bool mc); // no input available for the moment - + void addHMPMatches(o2::framework::ProcessingContext& pc, bool mc); void addMFTMCHMatches(o2::framework::ProcessingContext& pc, bool mc); void addMCHMIDMatches(o2::framework::ProcessingContext& pc, bool mc); void addITSClusters(o2::framework::ProcessingContext& pc, bool mc); void addMFTClusters(o2::framework::ProcessingContext& pc, bool mc); - void addTPCClusters(o2::framework::ProcessingContext& pc, bool mc, bool shmap); + void addTPCClusters(o2::framework::ProcessingContext& pc, bool mc, bool shmap, bool occmap); + void addTPCOccMap(o2::framework::ProcessingContext& pc); + void addTPCTriggers(o2::framework::ProcessingContext& pc); void addTOFClusters(o2::framework::ProcessingContext& pc, bool mc); void addHMPClusters(o2::framework::ProcessingContext& pc, bool mc); void addTRDTracklets(o2::framework::ProcessingContext& pc, bool mc); @@ -370,8 +404,14 @@ struct RecoContainer { void addPVerticesTMP(o2::framework::ProcessingContext& pc, bool mc); void addSVertices(o2::framework::ProcessingContext& pc, bool); + void addStrangeTracks(o2::framework::ProcessingContext& pc, bool mc); + void addIRFramesITS(o2::framework::ProcessingContext& pc); +#ifdef ENABLE_UPGRADES + void addIT3Clusters(o2::framework::ProcessingContext& pc, bool mc); +#endif + // custom getters // get contributors from single detectors: return array with sources set to all contributing GTrackIDs @@ -509,6 +549,7 @@ struct RecoContainer { auto getTPCTrackMCLabel(GTrackID id) const { return getObject(id, MCLABELS); } const o2::tpc::ClusterNativeAccess& getTPCClusters() const; const o2::dataformats::ConstMCTruthContainerView* getTPCClustersMCLabels() const; + auto getTPCTriggers() const { return getSpan(GTrackID::TPC, MATCHES); } // ITS-TPC const o2::dataformats::TrackTPCITS& getTPCITSTrack(GTrackID gid) const { return getTrack(gid); } @@ -594,13 +635,9 @@ struct RecoContainer { auto getITSTPCTRDTOFMatches() const { return getSpan(GTrackID::ITSTPCTRDTOF, MATCHES); } auto getITSTPCTRDTOFMatchesMCLabels() const { return getSpan(GTrackID::ITSTPCTRDTOF, MCLABELS); } - /* no input available for the moment // HMPID matches - auto getHMPMatchTriggers() const { return getSpan(GTrackID::HMP, TRACKREFS); } auto getHMPMatches() const { return getSpan(GTrackID::HMP, MATCHES); } - auto getHMPPhotsClusterCharges() const { return getSpan(GTrackID::HMP, PATTERNS); } auto getHMPMatchesMCLabels() const { return getSpan(GTrackID::HMP, MCLABELS); } - */ // TOF clusters auto getTOFClusters() const { return getSpan(GTrackID::TOF, CLUSTERS); } @@ -631,6 +668,7 @@ struct RecoContainer { // CTP auto getCTPDigits() const { return getSpan(GTrackID::CTP, CLUSTERS); } + const o2::ctp::LumiInfo& getCTPLumi() const { return mCTPLumi; } // CPV auto getCPVClusters() const { return getSpan(GTrackID::CPV, CLUSTERS); } @@ -658,13 +696,29 @@ struct RecoContainer { auto getPrimaryVertexMCLabels() const { return pvtxPool.getSpan(PVTX_MCTR); } // Secondary vertices + const o2::dataformats::V0Index& getV0Idx(int i) const { return svtxPool.get_as(V0SIDX, i); } const o2::dataformats::V0& getV0(int i) const { return svtxPool.get_as(V0S, i); } + const o2::dataformats::CascadeIndex& getCascadeIdx(int i) const { return svtxPool.get_as(CASCSIDX, i); } const o2::dataformats::Cascade& getCascade(int i) const { return svtxPool.get_as(CASCS, i); } + + auto getV0sIdx() const { return svtxPool.getSpan(V0SIDX); } auto getV0s() const { return svtxPool.getSpan(V0S); } auto getPV2V0Refs() { return svtxPool.getSpan>(PVTX_V0REFS); } + + auto getCascadesIdx() const { return svtxPool.getSpan(CASCSIDX); } auto getCascades() const { return svtxPool.getSpan(CASCS); } auto getPV2CascadesRefs() { return svtxPool.getSpan>(PVTX_CASCREFS); } + auto getDecays3BodyIdx() const { return svtxPool.getSpan(DECAY3BODYIDX); } + auto getDecays3Body() const { return svtxPool.getSpan(DECAY3BODY); } + auto getPV2Decays3BodyRefs() { return svtxPool.getSpan>(PVTX_3BODYREFS); } + + // Strangeness track + auto getStrangeTracks() const { return strkPool.getSpan(STRACK); } + auto getStrangeTracksMCLabels() const { return strkPool.getSpan(STRACK_MC); } + const o2::dataformats::StrangeTrack& getStrangeTrack(int i) const { return strkPool.get_as(STRACK, i); } + + // Cosmic tracks const o2::dataformats::TrackCosmics& getCosmicTrack(int i) const { return cosmPool.get_as(COSM_TRACKS, i); } auto getCosmicTrackMCLabel(int i) const { return cosmPool.get_as(COSM_TRACKS_MC, i); } auto getCosmicTracks() const { return cosmPool.getSpan(COSM_TRACKS); } @@ -672,6 +726,18 @@ struct RecoContainer { // IRFrames where ITS was reconstructed and tracks were seen (e.g. sync.w-flow mult. selection) auto getIRFramesITS() const { return getSpan(GTrackID::ITS, VARIA); } + + void getTrackTimeITSTPCTRDTOF(GTrackID gid, float& t, float& tErr) const; + void getTrackTimeTPCTRDTOF(GTrackID gid, float& t, float& tErr) const; + void getTrackTimeITSTPCTOF(GTrackID gid, float& t, float& tErr) const; + void getTrackTimeITSTPCTRD(GTrackID gid, float& t, float& tErr) const; + void getTrackTimeTPCTRD(GTrackID gid, float& t, float& tErr) const; + void getTrackTimeITSTPC(GTrackID gid, float& t, float& tErr) const; + void getTrackTimeTPCTOF(GTrackID gid, float& t, float& tErr) const; + void getTrackTimeITS(GTrackID gid, float& t, float& tErr) const; + void getTrackTimeTPC(GTrackID gid, float& t, float& tErr) const; + + void getTrackTime(GTrackID gid, float& t, float& tErr) const; }; } // namespace globaltracking diff --git a/DataFormats/Detectors/GlobalTracking/include/DataFormatsGlobalTracking/RecoContainerCreateTracksVariadic.h b/DataFormats/Detectors/GlobalTracking/include/DataFormatsGlobalTracking/RecoContainerCreateTracksVariadic.h index 6ceef700151dc..ecce5a4d58b2f 100644 --- a/DataFormats/Detectors/GlobalTracking/include/DataFormatsGlobalTracking/RecoContainerCreateTracksVariadic.h +++ b/DataFormats/Detectors/GlobalTracking/include/DataFormatsGlobalTracking/RecoContainerCreateTracksVariadic.h @@ -17,6 +17,7 @@ #include "Framework/InputSpec.h" #include "DetectorsCommonDataFormats/DetID.h" #include "DataFormatsTPC/WorkflowHelper.h" +#include "DataFormatsTPC/ZeroSuppression.h" #include "DataFormatsTRD/RecoInputContainer.h" #include "DataFormatsITSMFT/CompCluster.h" #include "DataFormatsITS/TrackITS.h" @@ -64,7 +65,7 @@ //________________________________________________________ template -void o2::globaltracking::RecoContainer::createTracksVariadic(T creator) const +void o2::globaltracking::RecoContainer::createTracksVariadic(T creator, GTrackID::mask_t srcSel) const { // We go from most complete tracks to least complete ones, taking into account that some track times // do not bear their own kinematics but just constrain the time @@ -131,7 +132,7 @@ void o2::globaltracking::RecoContainer::createTracksVariadic(T creator) const auto getBCDiff = [startIR = this->startIR, ¤tSource](const o2::InteractionRecord& ir) { auto bcd = ir.differenceInBC(startIR); if (uint64_t(bcd) > o2::constants::lhc::LHCMaxBunches * 256 && BCDiffErrCount < MAXBCDiffErrCount) { - LOGP(alarm, "ATTENTION: wrong bunches diff. {} for current IR {} wrt 1st TF orbit {}, source:{}", bcd, ir, startIR, GTrackID::getSourceName(currentSource)); + LOGP(alarm, "ATTENTION: wrong bunches diff. {} for current IR {} wrt 1st TF orbit {}, source:{}", bcd, ir.asString(), startIR.asString(), GTrackID::getSourceName(currentSource)); BCDiffErrCount++; } return bcd; @@ -140,19 +141,21 @@ void o2::globaltracking::RecoContainer::createTracksVariadic(T creator) const // ITS-TPC-TRD-TOF { currentSource = GTrackID::ITSTPCTRDTOF; - if (matchesITSTPCTRDTOF.size() && (!tofClusters.size() || !tracksITSTPCTRD.size())) { - throw std::runtime_error(fmt::format("Global-TOF tracks ({}) require ITS-TPC-TRD tracks ({}) and TOF clusters ({})", - matchesITSTPCTRDTOF.size(), tracksITSTPCTRD.size(), tofClusters.size())); - } - for (unsigned i = 0; i < matchesITSTPCTRDTOF.size(); i++) { - const auto& match = matchesITSTPCTRDTOF[i]; - auto gidx = match.getTrackRef(); // this should be corresponding ITS-TPC-TRD track - // no need to check isUsed: by construction this ITS-TPC-TRD was not used elsewhere - const auto& tofCl = tofClusters[match.getTOFClIndex()]; - float timeTOFMUS = (tofCl.getTime() - match.getLTIntegralOut().getTOF(o2::track::PID::Pion)) * PS2MUS; // tof time in \mus, FIXME: account for time of flight to R TOF - const float timeErr = 0.010f; // assume 10 ns error FIXME - if (creator(tracksITSTPCTRD[gidx.getIndex()], {i, currentSource}, timeTOFMUS, timeErr)) { - flagUsed(gidx); // flag used ITS-TPC-TRD tracks + if (srcSel[currentSource]) { + if (matchesITSTPCTRDTOF.size() && (!tofClusters.size() || !tracksITSTPCTRD.size())) { + throw std::runtime_error(fmt::format("Global-TOF tracks ({}) require ITS-TPC-TRD tracks ({}) and TOF clusters ({})", + matchesITSTPCTRDTOF.size(), tracksITSTPCTRD.size(), tofClusters.size())); + } + for (unsigned i = 0; i < matchesITSTPCTRDTOF.size(); i++) { + const auto& match = matchesITSTPCTRDTOF[i]; + auto gidx = match.getTrackRef(); // this should be corresponding ITS-TPC-TRD track + // no need to check isUsed: by construction this ITS-TPC-TRD was not used elsewhere + // const auto& tofCl = tofClusters[match.getTOFClIndex()]; + float timeTOFMUS = (match.getSignal() - match.getLTIntegralOut().getTOF(o2::track::PID::Pion)) * PS2MUS; // tof time in \mus, FIXME: account for time of flight to R TOF + const float timeErr = 0.010f; // assume 10 ns error FIXME + if (creator(tracksITSTPCTRD[gidx.getIndex()], {i, currentSource}, timeTOFMUS, timeErr)) { + flagUsed(gidx); // flag used ITS-TPC-TRD tracks + } } } } @@ -160,18 +163,20 @@ void o2::globaltracking::RecoContainer::createTracksVariadic(T creator) const // TPC-TRD-TOF { currentSource = GTrackID::TPCTRDTOF; - if (matchesTPCTRDTOF.size() && (!tofClusters.size() || !tracksTPCTRD.size())) { - throw std::runtime_error(fmt::format("Global-TOF tracks ({}) require TPC-TRD tracks ({}) and TOF clusters ({})", - matchesTPCTRDTOF.size(), tracksTPCTRD.size(), tofClusters.size())); - } - for (unsigned i = 0; i < matchesTPCTRDTOF.size(); i++) { - const auto& match = matchesTPCTRDTOF[i]; - auto gidx = match.getTrackRef(); // this should be corresponding TPC-TRD track - const auto& tofCl = tofClusters[match.getTOFClIndex()]; - float timeTOFMUS = (tofCl.getTime() - match.getLTIntegralOut().getTOF(o2::track::PID::Pion)) * PS2MUS; // tof time in \mus, FIXME: account for time of flight to R TOF - const float timeErr = 0.010f; // assume 10 ns error FIXME - if (creator(tracksTPCTRD[gidx.getIndex()], {i, currentSource}, timeTOFMUS, timeErr)) { - flagUsed(gidx); // flag used TPC-TRD tracks + if (srcSel[currentSource]) { + if (matchesTPCTRDTOF.size() && (!tofClusters.size() || !tracksTPCTRD.size())) { + throw std::runtime_error(fmt::format("Global-TOF tracks ({}) require TPC-TRD tracks ({}) and TOF clusters ({})", + matchesTPCTRDTOF.size(), tracksTPCTRD.size(), tofClusters.size())); + } + for (unsigned i = 0; i < matchesTPCTRDTOF.size(); i++) { + const auto& match = matchesTPCTRDTOF[i]; + auto gidx = match.getTrackRef(); // this should be corresponding TPC-TRD track + // const auto& tofCl = tofClusters[match.getTOFClIndex()]; + float timeTOFMUS = (match.getSignal() - match.getLTIntegralOut().getTOF(o2::track::PID::Pion)) * PS2MUS; // tof time in \mus, FIXME: account for time of flight to R TOF + const float timeErr = 0.010f; // assume 10 ns error FIXME + if (creator(tracksTPCTRD[gidx.getIndex()], {i, currentSource}, timeTOFMUS, timeErr)) { + flagUsed(gidx); // flag used TPC-TRD tracks + } } } } @@ -179,19 +184,25 @@ void o2::globaltracking::RecoContainer::createTracksVariadic(T creator) const // ITS-TPC-TRD { currentSource = GTrackID::ITSTPCTRD; - const auto trigITSTPCTRD = getITSTPCTRDTriggers(); - for (unsigned itr = 0; itr < trigITSTPCTRD.size(); itr++) { - const auto& trig = trigITSTPCTRD[itr]; - auto bcdiff = getBCDiff(trig.getBCData()); - float t0 = bcdiff * o2::constants::lhc::LHCBunchSpacingNS * 1e-3; - for (unsigned int i = trig.getTrackRefs().getFirstEntry(); i < (unsigned int)trig.getTrackRefs().getEntriesBound(); i++) { - const auto& trc = tracksITSTPCTRD[i]; - if (isUsed2(i, currentSource)) { - flagUsed(trc.getRefGlobalTrackId()); // flag seeding ITS-TPC track - continue; - } - if (creator(trc, {i, currentSource}, t0, 1e-3)) { // assign 1ns error to BC - flagUsed(trc.getRefGlobalTrackId()); // flag seeding ITS-TPC track + if (srcSel[currentSource]) { + const auto trigITSTPCTRD = getITSTPCTRDTriggers(); + for (unsigned itr = 0; itr < trigITSTPCTRD.size(); itr++) { + const auto& trig = trigITSTPCTRD[itr]; + auto bcdiff = getBCDiff(trig.getBCData()); + float t0Trig = bcdiff * o2::constants::lhc::LHCBunchSpacingMUS; + for (unsigned int i = trig.getTrackRefs().getFirstEntry(); i < (unsigned int)trig.getTrackRefs().getEntriesBound(); i++) { + const auto& trc = tracksITSTPCTRD[i]; + if (isUsed2(i, currentSource)) { + flagUsed(trc.getRefGlobalTrackId()); // flag seeding ITS-TPC track + continue; + } + float t0Err = 5.e-3; // 5ns nominal error + if (trc.hasPileUpInfo()) { // distance to farthest collision within the pileup integration time + t0Err += trc.getPileUpTimeErrorMUS(); + } + if (creator(trc, {i, currentSource}, t0Trig, t0Err)) { // assign 1ns error to BC + flagUsed(trc.getRefGlobalTrackId()); // flag seeding ITS-TPC track + } } } } @@ -200,22 +211,24 @@ void o2::globaltracking::RecoContainer::createTracksVariadic(T creator) const // ITS-TPC-TOF matches, these are just MatchInfoTOF objects, pointing on ITS-TPC match and TOF cl. { currentSource = GTrackID::ITSTPCTOF; - if (matchesITSTPCTOF.size() && (!tofClusters.size() || !tracksTPCITS.size())) { - throw std::runtime_error(fmt::format("Global-TOF tracks ({}) require ITS-TPC tracks ({}) and TOF clusters ({})", - matchesITSTPCTOF.size(), tracksTPCITS.size(), tofClusters.size())); - } - for (unsigned i = 0; i < matchesITSTPCTOF.size(); i++) { - const auto& match = matchesITSTPCTOF[i]; - auto gidx = match.getTrackRef(); // this should be corresponding ITS-TPC track - if (isUsed(gidx)) { // RS FIXME: THIS IS TEMPORARY, until the TOF matching will use ITS-TPC-TRD as an input - continue; + if (srcSel[currentSource]) { + if (matchesITSTPCTOF.size() && (!tofClusters.size() || !tracksTPCITS.size())) { + throw std::runtime_error(fmt::format("Global-TOF tracks ({}) require ITS-TPC tracks ({}) and TOF clusters ({})", + matchesITSTPCTOF.size(), tracksTPCITS.size(), tofClusters.size())); } - // no need to check isUsed: by construction this ITS-TPC was not used elsewhere - const auto& tofCl = tofClusters[match.getTOFClIndex()]; - float timeTOFMUS = (tofCl.getTime() - match.getLTIntegralOut().getTOF(o2::track::PID::Pion)) * PS2MUS; // tof time in \mus, FIXME: account for time of flight to R TOF - const float timeErr = 0.010f; // assume 10 ns error FIXME - if (creator(tracksTPCITS[gidx.getIndex()], {i, currentSource}, timeTOFMUS, timeErr)) { - flagUsed(gidx); // flag used ITS-TPC tracks + for (unsigned i = 0; i < matchesITSTPCTOF.size(); i++) { + const auto& match = matchesITSTPCTOF[i]; + auto gidx = match.getTrackRef(); // this should be corresponding ITS-TPC track + if (isUsed(gidx)) { // RS FIXME: THIS IS TEMPORARY, until the TOF matching will use ITS-TPC-TRD as an input + continue; + } + // no need to check isUsed: by construction this ITS-TPC was not used elsewhere + // const auto& tofCl = tofClusters[match.getTOFClIndex()]; + float timeTOFMUS = (match.getSignal() - match.getLTIntegralOut().getTOF(o2::track::PID::Pion)) * PS2MUS; // tof time in \mus, FIXME: account for time of flight to R TOF + const float timeErr = 0.010f; // assume 10 ns error FIXME + if (creator(tracksTPCITS[gidx.getIndex()], {i, currentSource}, timeTOFMUS, timeErr)) { + flagUsed(gidx); // flag used ITS-TPC tracks + } } } } @@ -223,19 +236,25 @@ void o2::globaltracking::RecoContainer::createTracksVariadic(T creator) const // TPC-TRD { currentSource = GTrackID::TPCTRD; - const auto trigTPCTRD = getTPCTRDTriggers(); - for (unsigned itr = 0; itr < trigTPCTRD.size(); itr++) { - const auto& trig = trigTPCTRD[itr]; - auto bcdiff = getBCDiff(trig.getBCData()); - float t0 = bcdiff * o2::constants::lhc::LHCBunchSpacingNS * 1e-3; - for (unsigned int i = trig.getTrackRefs().getFirstEntry(); i < (unsigned int)trig.getTrackRefs().getEntriesBound(); i++) { - const auto& trc = tracksTPCTRD[i]; - if (isUsed2(i, currentSource)) { - flagUsed(trc.getRefGlobalTrackId()); // flag seeding TPC track - continue; - } - if (creator(trc, {i, currentSource}, t0, 1e-3)) { // assign 1ns error to BC - flagUsed(trc.getRefGlobalTrackId()); // flag seeding TPC track + if (srcSel[currentSource]) { + const auto trigTPCTRD = getTPCTRDTriggers(); + for (unsigned itr = 0; itr < trigTPCTRD.size(); itr++) { + const auto& trig = trigTPCTRD[itr]; + auto bcdiff = getBCDiff(trig.getBCData()); + float t0Trig = bcdiff * o2::constants::lhc::LHCBunchSpacingMUS; + for (unsigned int i = trig.getTrackRefs().getFirstEntry(); i < (unsigned int)trig.getTrackRefs().getEntriesBound(); i++) { + const auto& trc = tracksTPCTRD[i]; + if (isUsed2(i, currentSource)) { + flagUsed(trc.getRefGlobalTrackId()); // flag seeding TPC track + continue; + } + float t0Err = 5.e-3; // 5ns nominal error + if (trc.hasPileUpInfo()) { // distance to farthest collision within the pileup integration time + t0Err += trc.getPileUpTimeErrorMUS(); + } + if (creator(trc, {i, currentSource}, t0Trig, t0Err)) { // assign 1ns error to BC + flagUsed(trc.getRefGlobalTrackId()); // flag seeding TPC track + } } } } @@ -244,16 +263,18 @@ void o2::globaltracking::RecoContainer::createTracksVariadic(T creator) const // ITS-TPC matches { currentSource = GTrackID::ITSTPC; - for (unsigned i = 0; i < tracksTPCITS.size(); i++) { - const auto& matchTr = tracksTPCITS[i]; - if (isUsed2(i, currentSource)) { - flagUsed(matchTr.getRefITS()); // flag used ITS tracks or AB tracklets (though the latter is not really necessary) - flagUsed(matchTr.getRefTPC()); // flag used TPC tracks - continue; - } - if (creator(matchTr, {i, currentSource}, matchTr.getTimeMUS().getTimeStamp(), matchTr.getTimeMUS().getTimeStampError())) { - flagUsed(matchTr.getRefITS()); // flag used ITS tracks or AB tracklets (though the latter is not really necessary) - flagUsed(matchTr.getRefTPC()); // flag used TPC tracks + if (srcSel[currentSource]) { + for (unsigned i = 0; i < tracksTPCITS.size(); i++) { + const auto& matchTr = tracksTPCITS[i]; + if (isUsed2(i, currentSource)) { + flagUsed(matchTr.getRefITS()); // flag used ITS tracks or AB tracklets (though the latter is not really necessary) + flagUsed(matchTr.getRefTPC()); // flag used TPC tracks + continue; + } + if (creator(matchTr, {i, currentSource}, matchTr.getTimeMUS().getTimeStamp(), matchTr.getTimeMUS().getTimeStampError())) { + flagUsed(matchTr.getRefITS()); // flag used ITS tracks or AB tracklets (though the latter is not really necessary) + flagUsed(matchTr.getRefTPC()); // flag used TPC tracks + } } } } @@ -261,19 +282,21 @@ void o2::globaltracking::RecoContainer::createTracksVariadic(T creator) const // TPC-TOF matches { currentSource = GTrackID::TPCTOF; - if (matchesTPCTOF.size() && !tracksTPCTOF.size()) { - throw std::runtime_error(fmt::format("TPC-TOF matched tracks ({}) require TPCTOF matches ({}) and TPCTOF tracks ({})", - matchesTPCTOF.size(), tracksTPCTOF.size())); - } - for (unsigned i = 0; i < matchesTPCTOF.size(); i++) { - const auto& match = matchesTPCTOF[i]; - const auto& gidx = match.getTrackRef(); // TPC track global idx - if (isUsed(gidx)) { // flag used TPC tracks - continue; + if (srcSel[currentSource]) { + if (matchesTPCTOF.size() && !tracksTPCTOF.size()) { + throw std::runtime_error(fmt::format("TPC-TOF matched tracks ({}) require TPCTOF matches ({}) and TPCTOF tracks ({})", + -1, matchesTPCTOF.size(), tracksTPCTOF.size())); } - const auto& trc = tracksTPCTOF[i]; - if (creator(trc, {i, currentSource}, trc.getTimeMUS().getTimeStamp(), trc.getTimeMUS().getTimeStampError())) { - flagUsed(gidx); // flag used TPC tracks + for (unsigned i = 0; i < matchesTPCTOF.size(); i++) { + const auto& match = matchesTPCTOF[i]; + const auto& gidx = match.getTrackRef(); // TPC track global idx + if (isUsed(gidx)) { // flag used TPC tracks + continue; + } + const auto& trc = tracksTPCTOF[i]; + if (creator(trc, {i, currentSource}, trc.getTimeMUS().getTimeStamp(), trc.getTimeMUS().getTimeStampError())) { + flagUsed(gidx); // flag used TPC tracks + } } } } @@ -281,11 +304,13 @@ void o2::globaltracking::RecoContainer::createTracksVariadic(T creator) const // MFT-MCH tracks { currentSource = GTrackID::MFTMCH; - for (unsigned i = 0; i < tracksMFTMCH.size(); i++) { - const auto& matchTr = tracksMFTMCH[i]; - if (creator(matchTr, {i, currentSource}, matchTr.getTimeMUS().getTimeStamp(), matchTr.getTimeMUS().getTimeStampError())) { - flagUsed2(matchTr.getMFTTrackID(), GTrackID::MFT); - flagUsed2(matchTr.getMCHTrackID(), GTrackID::MCH); + if (srcSel[currentSource]) { + for (unsigned i = 0; i < tracksMFTMCH.size(); i++) { + const auto& matchTr = tracksMFTMCH[i]; + if (creator(matchTr, {i, currentSource}, matchTr.getTimeMUS().getTimeStamp(), matchTr.getTimeMUS().getTimeStampError())) { + flagUsed2(matchTr.getMFTTrackID(), GTrackID::MFT); + flagUsed2(matchTr.getMCHTrackID(), GTrackID::MCH); + } } } } @@ -293,21 +318,23 @@ void o2::globaltracking::RecoContainer::createTracksVariadic(T creator) const // MCH-MID matches { currentSource = GTrackID::MCHMID; - if (matchesMCHMID.size() && !tracksMCH.size()) { - throw std::runtime_error(fmt::format("MCH-MID matched tracks ({}) require MCHMID matches ({}) and MCH tracks ({})", - matchesMCHMID.size(), tracksMCH.size())); - } - for (unsigned i = 0; i < matchesMCHMID.size(); i++) { - const auto& match = matchesMCHMID[i]; - auto [trcTime, isInTF] = match.getTimeMUS(startIR, 256, BCDiffErrCount < MAXBCDiffErrCount); - if (!isInTF && BCDiffErrCount < MAXBCDiffErrCount) { - BCDiffErrCount++; + if (srcSel[currentSource]) { + if (matchesMCHMID.size() && !tracksMCH.size()) { + throw std::runtime_error(fmt::format("MCH-MID matched tracks ({}) require MCHMID matches ({}) and MCH tracks ({})", + -1, matchesMCHMID.size(), tracksMCH.size())); } - auto gidxMCH = match.getMCHRef(); - const auto& trc = tracksMCH[gidxMCH.getIndex()]; - if (creator(trc, {i, currentSource}, trcTime.getTimeStamp(), trcTime.getTimeStampError())) { - flagUsed(gidxMCH); // flag used MCH tracks - flagUsed(match.getMIDRef()); // flag used MID tracks (if requested) + for (unsigned i = 0; i < matchesMCHMID.size(); i++) { + const auto& match = matchesMCHMID[i]; + auto [trcTime, isInTF] = match.getTimeMUS(startIR, 256, BCDiffErrCount < MAXBCDiffErrCount); + if (!isInTF && BCDiffErrCount < MAXBCDiffErrCount) { + BCDiffErrCount++; + } + auto gidxMCH = match.getMCHRef(); + const auto& trc = tracksMCH[gidxMCH.getIndex()]; + if (creator(trc, {i, currentSource}, trcTime.getTimeStamp(), trcTime.getTimeStampError())) { + flagUsed(gidxMCH); // flag used MCH tracks + flagUsed(match.getMIDRef()); // flag used MID tracks (if requested) + } } } } @@ -315,19 +342,21 @@ void o2::globaltracking::RecoContainer::createTracksVariadic(T creator) const // ITS only tracks { currentSource = GTrackID::ITS; - const auto& rofrs = getITSTracksROFRecords(); - for (unsigned irof = 0; irof < rofrs.size(); irof++) { - const auto& rofRec = rofrs[irof]; - auto bcdiff = getBCDiff(rofRec.getBCData()); - float t0 = bcdiff * o2::constants::lhc::LHCBunchSpacingNS * 1e-3; - int trlim = rofRec.getFirstEntry() + rofRec.getNEntries(); - for (int it = rofRec.getFirstEntry(); it < trlim; it++) { - if (isUsed2(it, currentSource)) { // skip used tracks - continue; + if (srcSel[currentSource]) { + const auto& rofrs = getITSTracksROFRecords(); + for (unsigned irof = 0; irof < rofrs.size(); irof++) { + const auto& rofRec = rofrs[irof]; + auto bcdiff = getBCDiff(rofRec.getBCData()); + float t0 = bcdiff * o2::constants::lhc::LHCBunchSpacingNS * 1e-3; + int trlim = rofRec.getFirstEntry() + rofRec.getNEntries(); + for (int it = rofRec.getFirstEntry(); it < trlim; it++) { + if (isUsed2(it, currentSource)) { // skip used tracks + continue; + } + GTrackID gidITS(it, currentSource); + const auto& trc = tracksITS[it]; + creator(trc, gidITS, t0, 0.5); } - GTrackID gidITS(it, currentSource); - const auto& trc = tracksITS[it]; - creator(trc, gidITS, t0, 0.5); } } } @@ -335,19 +364,21 @@ void o2::globaltracking::RecoContainer::createTracksVariadic(T creator) const // MFT only tracks { currentSource = GTrackID::MFT; - const auto& rofrs = getMFTTracksROFRecords(); - for (unsigned irof = 0; irof < rofrs.size(); irof++) { - const auto& rofRec = rofrs[irof]; - auto bcdiff = getBCDiff(rofRec.getBCData()); - float t0 = bcdiff * o2::constants::lhc::LHCBunchSpacingNS * 1e-3; - int trlim = rofRec.getFirstEntry() + rofRec.getNEntries(); - for (int it = rofRec.getFirstEntry(); it < trlim; it++) { - if (isUsed2(it, currentSource)) { - continue; + if (srcSel[currentSource]) { + const auto& rofrs = getMFTTracksROFRecords(); + for (unsigned irof = 0; irof < rofrs.size(); irof++) { + const auto& rofRec = rofrs[irof]; + auto bcdiff = getBCDiff(rofRec.getBCData()); + float t0 = bcdiff * o2::constants::lhc::LHCBunchSpacingNS * 1e-3; + int trlim = rofRec.getFirstEntry() + rofRec.getNEntries(); + for (int it = rofRec.getFirstEntry(); it < trlim; it++) { + if (isUsed2(it, currentSource)) { + continue; + } + GTrackID gidMFT(it, currentSource); + const auto& trc = tracksMFT[it]; + creator(trc, gidMFT, t0, 0.5); } - GTrackID gidMFT(it, currentSource); - const auto& trc = tracksMFT[it]; - creator(trc, gidMFT, t0, 0.5); } } } @@ -355,22 +386,24 @@ void o2::globaltracking::RecoContainer::createTracksVariadic(T creator) const // MCH standalone tracks { currentSource = GTrackID::MCH; - const auto& rofs = getMCHTracksROFRecords(); - for (const auto& rof : rofs) { - if (rof.getNEntries() == 0) { - continue; - } - auto [trcTime, isInTF] = rof.getTimeMUS(startIR, 256, BCDiffErrCount < MAXBCDiffErrCount); - if (!isInTF && BCDiffErrCount < MAXBCDiffErrCount) { - BCDiffErrCount++; - } - for (int idx = rof.getFirstIdx(); idx <= rof.getLastIdx(); ++idx) { - if (isUsed2(idx, currentSource)) { + if (srcSel[currentSource]) { + const auto& rofs = getMCHTracksROFRecords(); + for (const auto& rof : rofs) { + if (rof.getNEntries() == 0) { continue; } - GTrackID gidMCH(idx, currentSource); - const auto& trc = tracksMCH[idx]; - creator(trc, gidMCH, trcTime.getTimeStamp(), trcTime.getTimeStampError()); + auto [trcTime, isInTF] = rof.getTimeMUS(startIR, 256, BCDiffErrCount < MAXBCDiffErrCount); + if (!isInTF && BCDiffErrCount < MAXBCDiffErrCount) { + BCDiffErrCount++; + } + for (int idx = rof.getFirstIdx(); idx <= rof.getLastIdx(); ++idx) { + if (isUsed2(idx, currentSource)) { + continue; + } + GTrackID gidMCH(idx, currentSource); + const auto& trc = tracksMCH[idx]; + creator(trc, gidMCH, trcTime.getTimeStamp(), trcTime.getTimeStampError()); + } } } } @@ -378,28 +411,30 @@ void o2::globaltracking::RecoContainer::createTracksVariadic(T creator) const // MID standalone tracks { currentSource = GTrackID::MID; - const auto& rofs = getMIDTracksROFRecords(); - for (const auto& rof : rofs) { - if (rof.nEntries == 0) { - continue; - } - auto [trcTime, isInTF] = rof.getTimeMUS(startIR, 256, BCDiffErrCount < MAXBCDiffErrCount); - if (!isInTF && BCDiffErrCount < MAXBCDiffErrCount) { - BCDiffErrCount++; - } - if (trcTime.getTimeStamp() < 0.f) { - if (BCDiffErrCount - 1 < MAXBCDiffErrCount) { - LOGP(alarm, "Skipping MID ROF with {} entries since it precedes TF start", rof.nEntries); + if (srcSel[currentSource]) { + const auto& rofs = getMIDTracksROFRecords(); + for (const auto& rof : rofs) { + if (rof.nEntries == 0) { + continue; } - continue; - } - for (int idx = rof.firstEntry; idx < rof.getEndIndex(); ++idx) { - if (isUsed2(idx, currentSource)) { + auto [trcTime, isInTF] = rof.getTimeMUS(startIR, 256, BCDiffErrCount < MAXBCDiffErrCount); + if (!isInTF && BCDiffErrCount < MAXBCDiffErrCount) { + BCDiffErrCount++; + } + if (trcTime.getTimeStamp() < 0.f) { + if (BCDiffErrCount - 1 < MAXBCDiffErrCount) { + LOGP(alarm, "Skipping MID ROF with {} entries since it precedes TF start", rof.nEntries); + } continue; } - GTrackID gidMID(idx, currentSource); - const auto& trc = tracksMID[idx]; - creator(trc, gidMID, trcTime.getTimeStamp(), trcTime.getTimeStampError()); + for (int idx = rof.firstEntry; idx < rof.getEndIndex(); ++idx) { + if (isUsed2(idx, currentSource)) { + continue; + } + GTrackID gidMID(idx, currentSource); + const auto& trc = tracksMID[idx]; + creator(trc, gidMID, trcTime.getTimeStamp(), trcTime.getTimeStampError()); + } } } } @@ -407,13 +442,15 @@ void o2::globaltracking::RecoContainer::createTracksVariadic(T creator) const // TPC only tracks { currentSource = GTrackID::TPC; - int nacc = 0, noffer = 0; - for (unsigned i = 0; i < tracksTPC.size(); i++) { - if (isUsed2(i, currentSource)) { // skip used tracks - continue; + if (srcSel[currentSource]) { + int nacc = 0, noffer = 0; + for (unsigned i = 0; i < tracksTPC.size(); i++) { + if (isUsed2(i, currentSource)) { // skip used tracks + continue; + } + const auto& trc = tracksTPC[i]; + creator(trc, {i, currentSource}, trc.getTime0() + 0.5 * (trc.getDeltaTFwd() - trc.getDeltaTBwd()), 0.5 * (trc.getDeltaTFwd() + trc.getDeltaTBwd())); } - const auto& trc = tracksTPC[i]; - creator(trc, {i, currentSource}, trc.getTime0() + 0.5 * (trc.getDeltaTFwd() - trc.getDeltaTBwd()), 0.5 * (trc.getDeltaTFwd() + trc.getDeltaTBwd())); } } @@ -482,13 +519,7 @@ inline constexpr auto isGlobalFwdTrack() } template -inline constexpr auto isTPCTRDTOFTrack() +inline constexpr auto isBarrelTrack() { - return false; -} // to be implemented - -template -inline constexpr auto isITSTPCTRDTOFTrack() -{ - return false; -} // to be implemented + return std::is_base_of>::value || std::is_base_of>::value; +} diff --git a/DataFormats/Detectors/GlobalTracking/include/DataFormatsGlobalTracking/TrackTuneParams.h b/DataFormats/Detectors/GlobalTracking/include/DataFormatsGlobalTracking/TrackTuneParams.h new file mode 100644 index 0000000000000..3ca1a33325b96 --- /dev/null +++ b/DataFormats/Detectors/GlobalTracking/include/DataFormatsGlobalTracking/TrackTuneParams.h @@ -0,0 +1,56 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \author ruben.shahoyan@cern.ch + +#ifndef ALICEO2_TRACK_TUNE_PARAMS_H +#define ALICEO2_TRACK_TUNE_PARAMS_H + +#include "CommonUtils/ConfigurableParam.h" +#include "CommonUtils/ConfigurableParamHelper.h" +#include + +namespace o2 +{ +namespace globaltracking +{ + +// There are configurable params for tracks ad hoc tuning + +struct TrackTuneParams : public o2::conf::ConfigurableParamHelper { + enum AddCovType { // how to add covariances to tracks + Disable, // do not add + NoCorrelations, // add ignoring correlations (i.e. to diagonal elements only) + WithCorrelations // add adjusting non-diagonal elements to preserve correlation coefficients + }; + AddCovType tpcCovInnerType = AddCovType::Disable; + AddCovType tpcCovOuterType = AddCovType::Disable; + bool sourceLevelTPC = true; // if TPC corrections are allowed, apply them TPC source output level (tracking), otherwise in the global tracking consumers BEFORE update by external detector + bool applyWhenReading = false; // if true, then apply at reading tracks from the file. This better NOT used as there is no way to apply lumi-dependent scaling in the reader + bool useTPCInnerCorr = false; // request to correct TPC inner param + bool useTPCOuterCorr = false; // request to correct TPC outer param + float tpcParInner[5] = {}; // ad hoc correction to be added to TPC param at the inner XRef + float tpcParOuter[5] = {}; // ad hoc correction to be added to TPC param at the outer XRef + float tpcCovInner[5] = {}; // ad hoc errors to be added to TPC cov.matrix at the inner XRef (not squared!) + float tpcCovOuter[5] = {}; // ad hoc errors to be added to TPC outer param cov.matrix at the outer XRef (not squared!) + float tpcCovInnerSlope[5] = {}; // slope of the error with respect to lumi, total correction = [tpcCovInner + lumi*tpcCovInnerSlope]^2 + float tpcCovOuterSlope[5] = {}; // slope of the error with respect to lumi, total correction = [tpcCovOuter + lumi*tpcCovOuterSlope]^2 + + std::array getCovInnerTotal(float scale) const; + std::array getCovOuterTotal(float scale) const; + + O2ParamDef(TrackTuneParams, "trackTuneParams"); +}; + +} // namespace globaltracking +} // end namespace o2 + +#endif diff --git a/DataFormats/Detectors/GlobalTracking/src/DataFormatsGlobalTrackingLinkDef.h b/DataFormats/Detectors/GlobalTracking/src/DataFormatsGlobalTrackingLinkDef.h index 12fbd3bf25e2d..d3519bbaad607 100644 --- a/DataFormats/Detectors/GlobalTracking/src/DataFormatsGlobalTrackingLinkDef.h +++ b/DataFormats/Detectors/GlobalTracking/src/DataFormatsGlobalTrackingLinkDef.h @@ -19,4 +19,7 @@ #pragma link C++ class o2::dataformats::FilteredRecoTF::Header + ; #pragma link C++ class std::vector < o2::dataformats::FilteredRecoTF> + ; +#pragma link C++ class o2::globaltracking::TrackTuneParams + ; +#pragma link C++ class o2::conf::ConfigurableParamHelper < o2::globaltracking::TrackTuneParams> + ; + #endif diff --git a/DataFormats/Detectors/GlobalTracking/src/RecoContainer.cxx b/DataFormats/Detectors/GlobalTracking/src/RecoContainer.cxx index 5c9fa365ecdba..dd206ffe3b70d 100644 --- a/DataFormats/Detectors/GlobalTracking/src/RecoContainer.cxx +++ b/DataFormats/Detectors/GlobalTracking/src/RecoContainer.cxx @@ -23,8 +23,11 @@ #include "ReconstructionDataFormats/VtxTrackRef.h" #include "ReconstructionDataFormats/PrimaryVertex.h" #include "SimulationDataFormat/MCEventLabel.h" +#include "ReconstructionDataFormats/DecayNBodyIndex.h" #include "ReconstructionDataFormats/V0.h" #include "ReconstructionDataFormats/Cascade.h" +#include "ReconstructionDataFormats/Decay3Body.h" +#include "ReconstructionDataFormats/StrangeTrack.h" #include "ReconstructionDataFormats/VtxTrackIndex.h" #include "ReconstructionDataFormats/VtxTrackRef.h" #include "ReconstructionDataFormats/TrackCosmics.h" @@ -39,6 +42,10 @@ #include "Framework/DataRefUtils.h" #include "Framework/CCDBParamSpec.h" +#ifdef ENABLE_UPGRADES +#include "ITS3Reconstruction/TopologyDictionary.h" +#endif + using namespace o2::globaltracking; using namespace o2::framework; namespace o2d = o2::dataformats; @@ -116,6 +123,7 @@ void DataRequest::requestTPCTracks(bool mc) addInput({"trackTPCClRefs", "TPC", "CLUSREFS", 0, Lifetime::Timeframe}); if (requestMap.find("clusTPC") != requestMap.end()) { addInput({"clusTPCshmap", "TPC", "CLSHAREDMAP", 0, Lifetime::Timeframe}); + requestTPCOccMap(); } if (mc) { addInput({"trackTPCMCTR", "TPC", "TRACKSMCLBL", 0, Lifetime::Timeframe}); @@ -223,14 +231,29 @@ void DataRequest::requestITSClusters(bool mc) addInput({"clusITS", "ITS", "COMPCLUSTERS", 0, Lifetime::Timeframe}); addInput({"clusITSPatt", "ITS", "PATTERNS", 0, Lifetime::Timeframe}); addInput({"clusITSROF", "ITS", "CLUSTERSROF", 0, Lifetime::Timeframe}); - addInput({"cldictITS", "ITS", "CLUSDICT", 0, Lifetime::Condition, ccdbParamSpec("ITS/Calib/ClusterDictionary")}); addInput({"alpparITS", "ITS", "ALPIDEPARAM", 0, Lifetime::Condition, ccdbParamSpec("ITS/Config/AlpideParam")}); if (mc) { addInput({"clusITSMC", "ITS", "CLUSTERSMCTR", 0, Lifetime::Timeframe}); } + addInput({"cldictITS", "ITS", "CLUSDICT", 0, Lifetime::Condition, ccdbParamSpec("ITS/Calib/ClusterDictionary")}); requestMap["clusITS"] = mc; } +#ifdef ENABLE_UPGRADES +void DataRequest::requestIT3Clusters(bool mc) +{ + addInput({"clusITS", "ITS", "COMPCLUSTERS", 0, Lifetime::Timeframe}); + addInput({"clusITSPatt", "ITS", "PATTERNS", 0, Lifetime::Timeframe}); + addInput({"clusITSROF", "ITS", "CLUSTERSROF", 0, Lifetime::Timeframe}); + addInput({"alpparITS", "ITS", "ALPIDEPARAM", 0, Lifetime::Condition, ccdbParamSpec("ITS/Config/AlpideParam")}); + if (mc) { + addInput({"clusITSMC", "ITS", "CLUSTERSMCTR", 0, Lifetime::Timeframe}); + } + addInput({"cldictIT3", "IT3", "CLUSDICT", 0, Lifetime::Condition, ccdbParamSpec("IT3/Calib/ClusterDictionary")}); + requestMap["clusIT3"] = mc; +} +#endif + void DataRequest::requestMFTClusters(bool mc) { addInput({"clusMFT", "MFT", "COMPCLUSTERS", 0, Lifetime::Timeframe}); @@ -244,11 +267,21 @@ void DataRequest::requestMFTClusters(bool mc) requestMap["clusMFT"] = mc; } +void DataRequest::requestTPCOccMap() +{ + addInput({"clusTPCoccmap", "TPC", "TPCOCCUPANCYMAP", 0, Lifetime::Timeframe}); + requestMap["TPCOcc"] = false; +} + void DataRequest::requestTPCClusters(bool mc) { addInput({"clusTPC", ConcreteDataTypeMatcher{"TPC", "CLUSTERNATIVE"}, Lifetime::Timeframe}); + if (!getenv("DPL_DISABLE_TPC_TRIGGER_READER") || atoi(getenv("DPL_DISABLE_TPC_TRIGGER_READER")) != 1) { + requestTPCTriggers(); + } if (requestMap.find("trackTPC") != requestMap.end()) { addInput({"clusTPCshmap", "TPC", "CLSHAREDMAP", 0, Lifetime::Timeframe}); + requestTPCOccMap(); } if (mc) { addInput({"clusTPCMC", ConcreteDataTypeMatcher{"TPC", "CLNATIVEMCLBL"}, Lifetime::Timeframe}); @@ -256,6 +289,12 @@ void DataRequest::requestTPCClusters(bool mc) requestMap["clusTPC"] = mc; } +void DataRequest::requestTPCTriggers() +{ + addInput({"trigTPC", "TPC", "TRIGGERWORDS", 0, Lifetime::Timeframe}); + requestMap["trigTPC"] = false; +} + void DataRequest::requestTOFClusters(bool mc) { addInput({"tofcluster", "TOF", "CLUSTERS", 0, Lifetime::Timeframe}); @@ -366,7 +405,7 @@ void DataRequest::requestCoscmicTracks(bool mc) requestMap["Cosmics"] = mc; } -void DataRequest::requestPrimaryVertertices(bool mc) +void DataRequest::requestPrimaryVertices(bool mc) { addInput({"pvtx", "GLO", "PVTX", 0, Lifetime::Timeframe}); addInput({"pvtx_trmtc", "GLO", "PVTX_TRMTC", 0, Lifetime::Timeframe}); // global ids of associated tracks @@ -377,7 +416,7 @@ void DataRequest::requestPrimaryVertertices(bool mc) requestMap["PVertex"] = mc; } -void DataRequest::requestPrimaryVerterticesTMP(bool mc) // primary vertices before global vertex-track matching +void DataRequest::requestPrimaryVerticesTMP(bool mc) // primary vertices before global vertex-track matching { addInput({"pvtx", "GLO", "PVTX", 0, Lifetime::Timeframe}); addInput({"pvtx_cont", "GLO", "PVTX_CONTID", 0, Lifetime::Timeframe}); // global ids of contributors @@ -388,18 +427,33 @@ void DataRequest::requestPrimaryVerterticesTMP(bool mc) // primary vertices befo requestMap["PVertexTMP"] = mc; } -void DataRequest::requestSecondaryVertertices(bool) +void DataRequest::requestSecondaryVertices(bool) { + addInput({"v0sIdx", "GLO", "V0S_IDX", 0, Lifetime::Timeframe}); addInput({"v0s", "GLO", "V0S", 0, Lifetime::Timeframe}); addInput({"p2v0s", "GLO", "PVTX_V0REFS", 0, Lifetime::Timeframe}); + addInput({"cascsIdx", "GLO", "CASCS_IDX", 0, Lifetime::Timeframe}); addInput({"cascs", "GLO", "CASCS", 0, Lifetime::Timeframe}); addInput({"p2cascs", "GLO", "PVTX_CASCREFS", 0, Lifetime::Timeframe}); + addInput({"decay3bodyIdx", "GLO", "DECAYS3BODY_IDX", 0, Lifetime::Timeframe}); + addInput({"decay3body", "GLO", "DECAYS3BODY", 0, Lifetime::Timeframe}); + addInput({"p2decay3body", "GLO", "PVTX_3BODYREFS", 0, Lifetime::Timeframe}); requestMap["SVertex"] = false; // no MC provided for secondary vertices } +void DataRequest::requestStrangeTracks(bool mc) +{ + addInput({"strangetracks", "GLO", "STRANGETRACKS", 0, Lifetime::Timeframe}); + if (mc) { + addInput({"strack_mc", "GLO", "STRANGETRACKS_MC", 0, Lifetime::Timeframe}); + } + requestMap["STracker"] = mc; // no MC for the time being +} + void DataRequest::requestCTPDigits(bool mc) { addInput({"CTPDigits", "CTP", "DIGITS", 0, Lifetime::Timeframe}); + addInput({"CTPLumi", "CTP", "LUMI", 0, Lifetime::Timeframe}); if (mc) { LOG(warning) << "MC truth not implemented for CTP"; // addInput({"CTPDigitsMC", "CTP", "DIGITSMCTR", 0, Lifetime::Timeframe}); @@ -437,18 +491,14 @@ void DataRequest::requestEMCALCells(bool mc) requestMap["EMCCells"] = mc; } -/* void DataRequest::requestHMPMatches(bool mc) { addInput({"matchHMP", "HMP", "MATCHES", 0, Lifetime::Timeframe}); - addInput({"matchTriggerHMP", "HMP", "TRACKREFS", 0, Lifetime::Timeframe}); - addInput({"matchPhotsCharge", "HMP", "PATTERNS", 0, Lifetime::Timeframe}); if (mc) { addInput({"clsHMP_GLO_MCTR", "HMP", "MCLABELS", 0, Lifetime::Timeframe}); } requestMap["matchHMP"] = mc; } -*/ void DataRequest::requestTracks(GTrackID::mask_t src, bool useMC) { @@ -517,47 +567,46 @@ void DataRequest::requestTracks(GTrackID::mask_t src, bool useMC) if (GTrackID::includesDet(DetID::HMP, src)) { requestHMPClusters(useMC); } - // if (src[GTrackID::HMP]) { - // requestHMPMatches(useMC); - // } + if (src[GTrackID::HMP]) { + requestHMPMatches(useMC); + } } -void DataRequest::requestClusters(GTrackID::mask_t src, bool useMC) +void DataRequest::requestClusters(GTrackID::mask_t src, bool useMC, DetID::mask_t skipDetClusters) { // request clusters for detectors of the sources probided by the mask - // clusters needed for refits - if (GTrackID::includesDet(DetID::ITS, src)) { + if (GTrackID::includesDet(DetID::ITS, src) && !skipDetClusters[DetID::ITS]) { requestITSClusters(useMC); } - if (GTrackID::includesDet(DetID::MFT, src)) { + if (GTrackID::includesDet(DetID::MFT, src) && !skipDetClusters[DetID::MFT]) { requestMFTClusters(useMC); } - if (GTrackID::includesDet(DetID::TPC, src)) { + if (GTrackID::includesDet(DetID::TPC, src) && !skipDetClusters[DetID::TPC]) { requestTPCClusters(useMC); } - if (GTrackID::includesDet(DetID::TOF, src)) { + if (GTrackID::includesDet(DetID::TOF, src) && !skipDetClusters[DetID::TOF]) { requestTOFClusters(useMC); } - if (GTrackID::includesDet(DetID::TRD, src)) { + if (GTrackID::includesDet(DetID::TRD, src) && !skipDetClusters[DetID::TRD]) { requestTRDTracklets(useMC); } - if (GTrackID::includesDet(DetID::CTP, src)) { + if (GTrackID::includesDet(DetID::CTP, src) && !skipDetClusters[DetID::CTP]) { requestCTPDigits(false); // RS FIXME: at the moment does not support MC } - if (GTrackID::includesDet(DetID::CPV, src)) { + if (GTrackID::includesDet(DetID::CPV, src) && !skipDetClusters[DetID::CPV]) { requestCPVClusters(useMC); } - if (GTrackID::includesDet(DetID::PHS, src)) { + if (GTrackID::includesDet(DetID::PHS, src) && !skipDetClusters[DetID::PHS]) { requestPHOSCells(useMC); } - if (GTrackID::includesDet(DetID::EMC, src)) { + if (GTrackID::includesDet(DetID::EMC, src) && !skipDetClusters[DetID::EMC]) { requestEMCALCells(useMC); } - if (GTrackID::includesDet(DetID::MCH, src)) { + if (GTrackID::includesDet(DetID::MCH, src) && !skipDetClusters[DetID::MCH]) { requestMCHClusters(useMC); } - if (GTrackID::includesDet(DetID::HMP, src)) { + if (GTrackID::includesDet(DetID::HMP, src) && !skipDetClusters[DetID::HMP]) { requestHMPClusters(useMC); } } @@ -649,14 +698,34 @@ void RecoContainer::collectData(ProcessingContext& pc, const DataRequest& reques addITSClusters(pc, req->second); } +#ifdef ENABLE_UPGRADES + req = reqMap.find("clusIT3"); + if (req != reqMap.end()) { + addIT3Clusters(pc, req->second); + } +#endif + req = reqMap.find("clusMFT"); if (req != reqMap.end()) { addMFTClusters(pc, req->second); } + req = reqMap.find("TPCOcc"); + bool TPCOccDone = false; + if (req != reqMap.end()) { + TPCOccDone = true; + addTPCOccMap(pc); + } + req = reqMap.find("clusTPC"); if (req != reqMap.end()) { - addTPCClusters(pc, req->second, reqMap.find("trackTPC") != reqMap.end()); + auto tracksON = reqMap.find("trackTPC") != reqMap.end(); + addTPCClusters(pc, req->second, tracksON, tracksON && (!TPCOccDone)); + } + + req = reqMap.find("trigTPC"); + if (req != reqMap.end()) { + addTPCTriggers(pc); } req = reqMap.find("clusTOF"); @@ -744,50 +813,65 @@ void RecoContainer::collectData(ProcessingContext& pc, const DataRequest& reques addSVertices(pc, req->second); } + req = reqMap.find("STracker"); + if (req != reqMap.end()) { + addStrangeTracks(pc, req->second); + } + req = reqMap.find("IRFramesITS"); if (req != reqMap.end()) { addIRFramesITS(pc); } - // req = reqMap.find("matchHMP"); - // if (req != reqMap.end()) { - // addHMPMatches(pc, req->second); - // } + req = reqMap.find("matchHMP"); + if (req != reqMap.end()) { + addHMPMatches(pc, req->second); + } } //____________________________________________________________ void RecoContainer::addSVertices(ProcessingContext& pc, bool) { + svtxPool.registerContainer(pc.inputs().get>("v0sIdx"), V0SIDX); svtxPool.registerContainer(pc.inputs().get>("v0s"), V0S); svtxPool.registerContainer(pc.inputs().get>>("p2v0s"), PVTX_V0REFS); + svtxPool.registerContainer(pc.inputs().get>("cascsIdx"), CASCSIDX); svtxPool.registerContainer(pc.inputs().get>("cascs"), CASCS); svtxPool.registerContainer(pc.inputs().get>>("p2cascs"), PVTX_CASCREFS); + svtxPool.registerContainer(pc.inputs().get>("decay3bodyIdx"), DECAY3BODYIDX); + svtxPool.registerContainer(pc.inputs().get>("decay3body"), DECAY3BODY); + svtxPool.registerContainer(pc.inputs().get>>("p2decay3body"), PVTX_3BODYREFS); // no mc } //____________________________________________________________ void RecoContainer::addPVertices(ProcessingContext& pc, bool mc) { - if (!pvtxPool.isLoaded(PVTX)) { // in case was loaded via addPVerticesTMP - pvtxPool.registerContainer(pc.inputs().get>("pvtx"), PVTX); - } + pvtxPool.registerContainer(pc.inputs().get>("pvtx"), PVTX); pvtxPool.registerContainer(pc.inputs().get>("pvtx_trmtc"), PVTX_TRMTC); pvtxPool.registerContainer(pc.inputs().get>("pvtx_tref"), PVTX_TRMTCREFS); - if (mc && !pvtxPool.isLoaded(PVTX_MCTR)) { // in case was loaded via addPVerticesTMP + if (mc) { // in case was loaded via addPVerticesTMP pvtxPool.registerContainer(pc.inputs().get>("pvtx_mc"), PVTX_MCTR); } } //____________________________________________________________ -void RecoContainer::addPVerticesTMP(ProcessingContext& pc, bool mc) +void RecoContainer::addStrangeTracks(ProcessingContext& pc, bool mc) { - if (!pvtxPool.isLoaded(PVTX)) { // in case was loaded via addPVertices - pvtxPool.registerContainer(pc.inputs().get>("pvtx"), PVTX); + strkPool.registerContainer(pc.inputs().get>("strangetracks"), STRACK); + if (mc) { + strkPool.registerContainer(pc.inputs().get>("strack_mc"), STRACK_MC); } +} + +//____________________________________________________________ +void RecoContainer::addPVerticesTMP(ProcessingContext& pc, bool mc) +{ + pvtxPool.registerContainer(pc.inputs().get>("pvtx"), PVTX); pvtxPool.registerContainer(pc.inputs().get>("pvtx_cont"), PVTX_CONTID); pvtxPool.registerContainer(pc.inputs().get>("pvtx_contref"), PVTX_CONTIDREFS); - if (mc && !pvtxPool.isLoaded(PVTX_MCTR)) { // in case was loaded via addPVertices + if (mc) { // in case was loaded via addPVertices pvtxPool.registerContainer(pc.inputs().get>("pvtx_mc"), PVTX_MCTR); } } @@ -804,9 +888,7 @@ void RecoContainer::addCosmicTracks(ProcessingContext& pc, bool mc) //____________________________________________________________ void RecoContainer::addITSTracks(ProcessingContext& pc, bool mc) { - static bool initOnceDone = false; - if (!initOnceDone) { // this params need to be queried only once - initOnceDone = true; + if (pc.services().get().globalRunNumberChanged) { // this params need to be queried only once pc.inputs().get*>("alpparITS"); // note: configurable param does not need finaliseCCDB } commonPool[GTrackID::ITS].registerContainer(pc.inputs().get>("trackITS"), TRACKS); @@ -826,9 +908,7 @@ void RecoContainer::addIRFramesITS(ProcessingContext& pc) //____________________________________________________________ void RecoContainer::addMFTTracks(ProcessingContext& pc, bool mc) { - static bool initOnceDone = false; - if (!initOnceDone) { // this params need to be queried only once - initOnceDone = true; + if (pc.services().get().globalRunNumberChanged) { // this params need to be queried only once pc.inputs().get*>("alpparMFT"); // note: configurable param does not need finaliseCCDB } commonPool[GTrackID::MFT].registerContainer(pc.inputs().get>("trackMFT"), TRACKS); @@ -978,25 +1058,19 @@ void RecoContainer::addTOFMatchesITSTPCTRD(ProcessingContext& pc, bool mc) } } -/* //__________________________________________________________ void RecoContainer::addHMPMatches(ProcessingContext& pc, bool mc) { - commonPool[GTrackID::HMP].registerContainer(pc.inputs().get>("matchHMP"), MATCHES); // HMPID match info, no real tracks - commonPool[GTrackID::HMP].registerContainer(pc.inputs().get>("matchTriggerHMP"), TRACKREFS); // HMPID triggers - commonPool[GTrackID::HMP].registerContainer(pc.inputs().get>("matchPhotsCharge"), PATTERNS); // HMPID photon cluster charges + commonPool[GTrackID::HMP].registerContainer(pc.inputs().get>("matchHMP"), MATCHES); // HMPID match info, no real tracks if (mc) { commonPool[GTrackID::HMP].registerContainer(pc.inputs().get>("clsHMP_GLO_MCTR"), MCLABELS); } } -*/ //__________________________________________________________ void RecoContainer::addITSClusters(ProcessingContext& pc, bool mc) { - static bool initOnceDone = false; - if (!initOnceDone) { // this params need to be queried only once - initOnceDone = true; + if (pc.services().get().globalRunNumberChanged) { // this params need to be queried only once pc.inputs().get("cldictITS"); // just to trigger the finaliseCCDB pc.inputs().get*>("alpparITS"); // note: configurable param does not need finaliseCCDB } @@ -1008,12 +1082,26 @@ void RecoContainer::addITSClusters(ProcessingContext& pc, bool mc) } } +#ifdef ENABLE_UPGRADES +void RecoContainer::addIT3Clusters(ProcessingContext& pc, bool mc) +{ + if (pc.services().get().globalRunNumberChanged) { // this params need to be queried only once + pc.inputs().get*>("alpparITS"); // note: configurable param does not need finaliseCCDB + pc.inputs().get("cldictIT3"); // just to trigger the finaliseCCDB + } + commonPool[GTrackID::ITS].registerContainer(pc.inputs().get>("clusITSROF"), CLUSREFS); + commonPool[GTrackID::ITS].registerContainer(pc.inputs().get>("clusITS"), CLUSTERS); + commonPool[GTrackID::ITS].registerContainer(pc.inputs().get>("clusITSPatt"), PATTERNS); + if (mc) { + mcITSClusters = pc.inputs().get*>("clusITSMC"); + } +} +#endif + //__________________________________________________________ void RecoContainer::addMFTClusters(ProcessingContext& pc, bool mc) { - static bool initOnceDone = false; - if (!initOnceDone) { // this params need to be queried only once - initOnceDone = true; + if (pc.services().get().globalRunNumberChanged) { // this params need to be queried only once pc.inputs().get("cldictMFT"); // just to trigger the finaliseCCDB pc.inputs().get*>("alpparMFT"); // note: configurable param does not need finaliseCCDB } @@ -1026,12 +1114,27 @@ void RecoContainer::addMFTClusters(ProcessingContext& pc, bool mc) } //__________________________________________________________ -void RecoContainer::addTPCClusters(ProcessingContext& pc, bool mc, bool shmap) +void RecoContainer::addTPCOccMap(ProcessingContext& pc) +{ + occupancyMapTPC = pc.inputs().get>("clusTPCoccmap"); +} + +//__________________________________________________________ +void RecoContainer::addTPCClusters(ProcessingContext& pc, bool mc, bool shmap, bool occmap) { inputsTPCclusters = o2::tpc::getWorkflowTPCInput(pc, 0, mc); if (shmap) { clusterShMapTPC = pc.inputs().get>("clusTPCshmap"); } + if (occmap) { + occupancyMapTPC = pc.inputs().get>("clusTPCoccmap"); + } +} + +//__________________________________________________________ +void RecoContainer::addTPCTriggers(ProcessingContext& pc) +{ + commonPool[GTrackID::TPC].registerContainer(pc.inputs().get>("trigTPC"), MATCHES); } //__________________________________________________________ @@ -1082,6 +1185,9 @@ void RecoContainer::addMIDClusters(ProcessingContext& pc, bool mc) void RecoContainer::addCTPDigits(ProcessingContext& pc, bool mc) { commonPool[GTrackID::CTP].registerContainer(pc.inputs().get>("CTPDigits"), CLUSTERS); + if (pc.inputs().get>("CTPLumi").size() == sizeof(o2::ctp::LumiInfo)) { + mCTPLumi = pc.inputs().get("CTPLumi"); + } if (mc) { // pc.inputs().get*>("CTPDigitsMC"); } @@ -1180,7 +1286,16 @@ gsl::span RecoContainer::getTRDCalibratedTrac gsl::span RecoContainer::getTRDTriggerRecords() const { - return inputsTRD->mTriggerRecords; + static int countWarnings = 0; + if (inputsTRD == nullptr) { + if (countWarnings < 1) { + LOG(warning) << "No TRD triggers"; + countWarnings++; + } + return gsl::span(); + } else { + return inputsTRD->mTriggerRecords; + } } const o2::dataformats::MCTruthContainer* RecoContainer::getTRDTrackletsMCLabels() const @@ -1204,6 +1319,16 @@ const o2::track::TrackParCov& RecoContainer::getTrackParamOut(GTrackID gidx) con return getTrack(getTOFMatch(gidx).getTrackRef()).getParamOut(); } else if (trSrc == GTrackID::TPCTOF) { return getTrack(gidx).getParamOut(); + } else if (trSrc == GTrackID::ITSTPCTRDTOF) { // the physical tracks are in ITS-TPC-TRD, need to get reference from match info + return getTrack(getTOFMatch(gidx).getTrackRef()).getOuterParam(); + } + if (trSrc == GTrackID::TPCTRDTOF) { // the physical tracks are in TPC-TRD, need to get reference from match info + return getTrack(getTOFMatch(gidx).getTrackRef()).getOuterParam(); + } else if (trSrc == GTrackID::ITSTPCTRD) { + return getTrack(gidx).getOuterParam(); + } + if (trSrc == GTrackID::TPCTRD) { + return getTrack(gidx).getOuterParam(); } else if (trSrc == GTrackID::ITS) { return getTrack(gidx).getParamOut(); } else if (trSrc == GTrackID::TPC) { @@ -1315,8 +1440,7 @@ RecoContainer::GlobalIDSet RecoContainer::getSingleDetectorRefs(GTrackID gidx) c table[GTrackID::TRD] = parent0.getTrackRef(); // there is no standalone TRD track, so use the index for the ITSTPCTRD track array } else if (src == GTrackID::TPCTRDTOF) { const auto& parent0 = getTOFMatch(gidx); // TPCTRD : TOF - const auto& parent1 = getITSTPCTRDTrack(parent0.getTrackRef()); - const auto& parent2 = getTPCITSTrack(parent1.getRefGlobalTrackId()); + const auto& parent1 = getTPCTRDTrack(parent0.getTrackRef()); table[GTrackID::TPCTRD] = parent0.getTrackRef(); table[GTrackID::TPC] = parent1.getRefGlobalTrackId(); table[GTrackID::TOF] = {unsigned(parent0.getIdxTOFCl()), GTrackID::TOF}; @@ -1341,7 +1465,7 @@ RecoContainer::GlobalIDSet RecoContainer::getSingleDetectorRefs(GTrackID gidx) c table[GTrackID::MCH] = parent0.getMCHRef(); table[GTrackID::MID] = parent0.getMIDRef(); } - return std::move(table); + return table; } //________________________________________________________ @@ -1366,6 +1490,10 @@ GTrackID RecoContainer::getTPCContributorGID(GTrackID gidx) const } else if (src == GTrackID::TPCTOF) { const auto& parent0 = getTPCTOFMatch(gidx); // TPC : TOF return parent0.getTrackRef(); + } else if (src == GTrackID::TPCTRDTOF) { + const auto& parent0 = getTOFMatch(gidx); // TPC/TRD : TOF + const auto& parent1 = getTPCTRDTrack(parent0.getTrackRef()); + return parent1.getRefGlobalTrackId(); } else if (src == GTrackID::TPCTRD) { const auto& parent0 = getTPCTRDTrack(gidx); return parent0.getRefGlobalTrackId(); @@ -1413,3 +1541,133 @@ const o2::dataformats::MCTruthContainer* RecoContainer::getE { return mcEMCCells.get(); } + +//________________________________________________________ +void RecoContainer::getTrackTimeITSTPCTRDTOF(GTrackID gid, float& t, float& tErr) const +{ + const auto& match = getITSTPCTRDTOFMatches()[gid]; + t = (match.getSignal() - match.getLTIntegralOut().getTOF(o2::track::PID::Pion)) * PS2MUS; // tof time in \mus, FIXME: account for time of flight to R TOF + tErr = 0.010f; +} + +//________________________________________________________ +void RecoContainer::getTrackTimeTPCTRDTOF(GTrackID gid, float& t, float& tErr) const +{ + const auto& match = getTPCTRDTOFMatches()[gid]; + t = (match.getSignal() - match.getLTIntegralOut().getTOF(o2::track::PID::Pion)) * PS2MUS; // tof time in \mus, FIXME: account for time of flight to R TOF + tErr = 0.010f; +} + +//________________________________________________________ +void RecoContainer::getTrackTimeITSTPCTOF(GTrackID gid, float& t, float& tErr) const +{ + const auto& match = getITSTPCTOFMatches()[gid]; + t = (match.getSignal() - match.getLTIntegralOut().getTOF(o2::track::PID::Pion)) * PS2MUS; // tof time in \mus, FIXME: account for time of flight to R TOF + tErr = 0.010f; +} + +//________________________________________________________ +void RecoContainer::getTrackTimeITSTPCTRD(GTrackID gid, float& t, float& tErr) const +{ + const auto trigITSTPCTRD = getITSTPCTRDTriggers(); + // very slow: find the trigger this track belongs to + for (const auto& trig : trigITSTPCTRD) { + if (trig.getTrackRefs().getEntriesBound() > gid.getIndex()) { + t = trig.getBCData().differenceInBC(startIR) * o2::constants::lhc::LHCBunchSpacingMUS; + tErr = 5.e-3; + const auto& trc = getITSTPCTRDTracks()[gid]; + if (trc.hasPileUpInfo()) { // distance to farthest collision within the pileup integration time + tErr += trc.getPileUpTimeErrorMUS(); + } + return; + } + } +} + +//________________________________________________________ +void RecoContainer::getTrackTimeTPCTRD(GTrackID gid, float& t, float& tErr) const +{ + const auto trigTPCTRD = getTPCTRDTriggers(); + // very slow: find the trigger this track belongs to + for (const auto& trig : trigTPCTRD) { + if (trig.getTrackRefs().getEntriesBound() > gid.getIndex()) { + t = trig.getBCData().differenceInBC(startIR) * o2::constants::lhc::LHCBunchSpacingMUS; + tErr = 5.e-3; + const auto& trc = getTPCTRDTracks()[gid]; + if (trc.hasPileUpInfo()) { // distance to farthest collision within the pileup integration time + tErr += trc.getPileUpTimeErrorMUS(); + } + return; + } + } +} + +//________________________________________________________ +void RecoContainer::getTrackTimeITSTPC(GTrackID gid, float& t, float& tErr) const +{ + const auto& trc = getTPCITSTrack(gid); + t = trc.getTimeMUS().getTimeStamp(); + tErr = trc.getTimeMUS().getTimeStampError(); +} + +//________________________________________________________ +void RecoContainer::getTrackTimeTPCTOF(GTrackID gid, float& t, float& tErr) const +{ + const auto& trc = getTPCTOFTrack(gid); + t = trc.getTimeMUS().getTimeStamp(); + tErr = trc.getTimeMUS().getTimeStampError(); +} + +//________________________________________________________ +void RecoContainer::getTrackTimeITS(GTrackID gid, float& t, float& tErr) const +{ + const auto& rofrs = getITSTracksROFRecords(); + for (const auto& rof : rofrs) { // slow + if (gid.getIndex() < rof.getFirstEntry() + rof.getNEntries()) { + t = rof.getBCData().differenceInBC(startIR) * o2::constants::lhc::LHCBunchSpacingMUS; + tErr = 0.5; + return; + } + } +} + +//________________________________________________________ +void RecoContainer::getTrackTimeTPC(GTrackID gid, float& t, float& tErr) const +{ + const auto& trc = getTPCTrack(gid); + t = trc.getTime0() + 0.5 * (trc.getDeltaTFwd() - trc.getDeltaTBwd()); + tErr = 0.5 * (trc.getDeltaTFwd() + trc.getDeltaTBwd()); +} + +//________________________________________________________ +void RecoContainer::getTrackTime(GTrackID gid, float& t, float& tErr) const +{ + auto src = gid.getSource(); + if (src == GTrackID::ITSTPCTRDTOF) { + return getTrackTimeITSTPCTRDTOF(gid, t, tErr); + } + if (src == GTrackID::TPCTRDTOF) { + return getTrackTimeTPCTRDTOF(gid, t, tErr); + } + if (src == GTrackID::ITSTPCTOF) { + return getTrackTimeITSTPCTOF(gid, t, tErr); + } + if (src == GTrackID::ITSTPCTRD) { + return getTrackTimeITSTPCTRD(gid, t, tErr); + } + if (src == GTrackID::TPCTRD) { + return getTrackTimeTPCTRD(gid, t, tErr); + } + if (src == GTrackID::ITSTPC) { + return getTrackTimeITSTPC(gid, t, tErr); + } + if (src == GTrackID::TPCTOF) { + return getTrackTimeTPCTOF(gid, t, tErr); + } + if (src == GTrackID::ITS) { + return getTrackTimeITS(gid, t, tErr); + } + if (src == GTrackID::TPC) { + return getTrackTimeTPC(gid, t, tErr); + } +} diff --git a/DataFormats/Detectors/GlobalTracking/src/TrackTuneParams.cxx b/DataFormats/Detectors/GlobalTracking/src/TrackTuneParams.cxx new file mode 100644 index 0000000000000..138f91937b250 --- /dev/null +++ b/DataFormats/Detectors/GlobalTracking/src/TrackTuneParams.cxx @@ -0,0 +1,39 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file TrackTuneParams.h +/// \brief Configurable params for tracks ad hoc tuning +/// \author ruben.shahoyan@cern.ch + +#include "DataFormatsGlobalTracking/TrackTuneParams.h" +O2ParamImpl(o2::globaltracking::TrackTuneParams); + +using namespace o2::globaltracking; + +std::array TrackTuneParams::getCovInnerTotal(float scale) const +{ + std::array cov{}; + for (int i = 0; i < 5; i++) { + cov[i] = tpcCovInner[i] + scale * tpcCovInnerSlope[i]; + cov[i] *= cov[i]; + } + return cov; +} + +std::array TrackTuneParams::getCovOuterTotal(float scale) const +{ + std::array cov{}; + for (int i = 0; i < 5; i++) { + cov[i] = tpcCovOuter[i] + scale * tpcCovOuterSlope[i]; + cov[i] *= cov[i]; + } + return cov; +} diff --git a/DataFormats/Detectors/HMPID/include/DataFormatsHMP/Cluster.h b/DataFormats/Detectors/HMPID/include/DataFormatsHMP/Cluster.h index 3649c13fffb8d..9c777d5afb718 100644 --- a/DataFormats/Detectors/HMPID/include/DataFormatsHMP/Cluster.h +++ b/DataFormats/Detectors/HMPID/include/DataFormatsHMP/Cluster.h @@ -1,4 +1,4 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// Copyright 2020-2022 CERN and copyright holders of ALICE O2. // See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. // All rights not expressly granted are reserved. // @@ -40,18 +40,22 @@ class Cluster kEmp = -1 }; // status flags public: - Cluster() : mCh(-1), mSi(-1), mSt(kEmp), mBox(-1), mNlocMax(-1), mMaxQpad(-1), mMaxQ(-1), mQRaw(0), mQ(0), mErrQ(-1), mXX(0), mErrX(-1), mYY(0), mErrY(-1), mChi2(-1), mParam(o2::hmpid::Param::instanceNoGeo()) { mDigs.clear(); }; - // Cluster(int chamber, int size, int NlocMax, float QRaw, float Q, float X, float Y) - // : mCh(chamber), mSi(size), mNlocMax(NlocMax), mQRaw(QRaw), mQ(Q), mXX(X), mYY(Y) { }; + Cluster() : mCh(-1), mSi(-1), mSt(kEmp), mBox(-1), mNlocMax(-1), mMaxQpad(-1), mMaxQ(-1), mQRaw(0), mQ(0), mErrQ(-1), mXX(0), mErrX(-1), mYY(0), mErrY(-1), mChi2(-1) {} // Methods // void draw(Option_t *opt=""); //overloaded TObject::Print() to draw cluster in current canvas void print(Option_t* opt = "") const; // overloaded TObject::Print() to print cluster info static void fitFunc(int& iNpars, double* deriv, double& chi2, double* par, int iflag); // fit function to be used by MINUIT + void cleanPointers() + { + mDigs = nullptr; + } void coG(); // calculates center of gravity void corrSin(); // sinoidal correction void digAdd(const o2::hmpid::Digit* pDig); // add new digit to the cluster - const o2::hmpid::Digit* dig(int i) const { return mDigs[i]; } // pointer to i-th digi + const o2::hmpid::Digit* dig(int i) const { return mDigs ? (*mDigs)[i] : nullptr; } // pointer to i-th digi + const std::vector* getDigits() const { return mDigs; } + void setDigits(std::vector* v = nullptr) { mDigs = v; } inline bool isInPc(); // check if is in the current PC void reset(); // cleans the cluster // void setClusterParams(float xL, float yL, int iCh); //Set AliCluster3D part @@ -95,26 +99,25 @@ class Cluster // public: protected: - int mCh; // chamber number - int mSi; // size of the formed cluster from which this cluster deduced - int mSt; // flag to mark the quality of the cluster - int mBox; // box contaning this cluster - int mNlocMax; // number of local maxima in formed cluster - int mMaxQpad; // abs pad number of a pad with the highest charge - double mMaxQ; // that max charge value - double mQRaw; // QDC value of the raw cluster - double mQ; // QDC value of the actual cluster - double mErrQ; // error on Q - double mXX; // local x postion, [cm] - double mErrX; // error on x postion, [cm] - double mYY; // local y postion, [cm] - double mErrY; // error on y postion, [cm] - double mChi2; // some estimator of the fit quality - std::vector mDigs; //! list of digits forming this cluster + int mCh; // chamber number + int mSi; // size of the formed cluster from which this cluster deduced + int mSt; // flag to mark the quality of the cluster + int mBox; // box contaning this cluster + int mNlocMax; // number of local maxima in formed cluster + int mMaxQpad; // abs pad number of a pad with the highest charge + double mMaxQ; // that max charge value + double mQRaw; // QDC value of the raw cluster + double mQ; // QDC value of the actual cluster + double mErrQ; // error on Q + double mXX; // local x postion, [cm] + double mErrX; // error on x postion, [cm] + double mYY; // local y postion, [cm] + double mErrY; // error on y postion, [cm] + double mChi2; // some estimator of the fit quality + std::vector* mDigs = nullptr; //! list of digits forming this cluster public: static bool fgDoCorrSin; // flag to switch on/off correction for Sinusoidal to cluster reco - Param* mParam; //! Pointer to AliHMPIDParam ClassDefNV(Cluster, 3); }; diff --git a/DataFormats/Detectors/HMPID/include/DataFormatsHMP/Digit.h b/DataFormats/Detectors/HMPID/include/DataFormatsHMP/Digit.h index b58d12ff6a259..63b04d454e66a 100644 --- a/DataFormats/Detectors/HMPID/include/DataFormatsHMP/Digit.h +++ b/DataFormats/Detectors/HMPID/include/DataFormatsHMP/Digit.h @@ -1,4 +1,4 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// Copyright 2020-2022 CERN and copyright holders of ALICE O2. // See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. // All rights not expressly granted are reserved. // @@ -9,7 +9,6 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. -/// /// \file Digit.h /// \author Antonio Franco - INFN Bari /// \version 1.0 diff --git a/DataFormats/Detectors/HMPID/src/Cluster.cxx b/DataFormats/Detectors/HMPID/src/Cluster.cxx index 56017ef1cd114..e14075c16c842 100644 --- a/DataFormats/Detectors/HMPID/src/Cluster.cxx +++ b/DataFormats/Detectors/HMPID/src/Cluster.cxx @@ -1,4 +1,4 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// Copyright 2020-2022 CERN and copyright holders of ALICE O2. // See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. // All rights not expressly granted are reserved. // @@ -12,6 +12,7 @@ #include #include #include "DataFormatsHMP/Cluster.h" +#include "Framework/Logger.h" #include #include #include @@ -24,84 +25,20 @@ namespace hmpid bool o2::hmpid::Cluster::fgDoCorrSin = true; -/*Cluster::Cluster() -{ - mCh = -1; - mSi = -1; - mSt = kEmp; - mBox = -1; - mNlocMax = -1; - mMaxQpad = -1; - mMaxQ = -1; - mQRaw = 0; - mQ = 0; - mErrQ = -1; - mXX = 0; - mErrX = -1; - mYY = 0; - mErrY = -1; - mChi2 = -1; - mDigs.clear(); - - // delete [] &mDigs; - mParam = (o2::hmpid::Param::instanceNoGeo()); - // mParam = (o2::hmpid::Param()); -}; -*/ -/*Cluster::~Cluster() -{ - delete mDigs; -} -*/ -/*void Cluster::setClusterParams(float xL,float yL,int iCh) -{ - //------------------------------------------------------------------------ - //Set the cluster properties for the AliCluster3D part - //------------------------------------------------------------------------ - mParam = o2::hmpid::Param::instance(); - if(!mParam->getInstType()) { //if there is no geometry we cannot retrieve the volId (only for monitoring) - // new(this) AliCluster3D(); return; - } - //Get the volume ID from the previously set PNEntry - uint16_t volId = AliGeomManager::LayerToVolUID(AliGeomManager::kHMPID, iCh); - - //get L->T cs matrix for a given chamber - const TGeoHMatrix *t2l = AliGeomManager::GetTracking2LocalMatrix(volId); - mParam = o2::hmpid::Param::instance(); - //transformation from the pad cs to local - xL -= 0.5 * mParam->sizeAllX(); //size of all pads with dead zones included - yL -= 0.5 * mParam->sizeAllY(); - - // Get the position in the tracking cs - float posL[3]={xL, yL, 0.}; //this is the LORS of HMPID - float posT[3]; - t2l->MasterToLocal(posL,posT); - - //Get the cluster covariance matrix in the tracking cs - float covL[9] = { 0.8 * 0.8 / 12.0, 0.0, 0.0, //pad size X - 0.0, 0.84 * 0.84 / 12.0, 0.0, //pad size Y - 0.0, 0.0, 0.1 }; //just 1 , no Z dimension ??? - - TGeoHMatrix m; - m.SetRotation(covL); - m.Multiply(t2l); - const TGeoHMatrix& t2li = t2l->Inverse(); - m.MultiplyLeft(&t2li); - float *covT = m.GetRotationMatrix(); - - // ===> new(this) AliCluster3D(volId,posT[0],posT[1],posT[2],covT[0],covT[1],covT[2],covT[4],covT[5],covT[8],0x0); // No MC labels ? -} -*/ // ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ // Calculates naive cluster position as a center of gravity of its digits. void Cluster::coG() { + if (!mDigs) { + LOGP(fatal, "digits are missing in the cluster"); + } + int minPadX = 999; int minPadY = 999; int maxPadX = -1; int maxPadY = -1; // for box finding - if (mDigs.size() == 0) { + if (mDigs->size() == 0) { return; } // no digits in this cluster mXX = mYY = mQRaw = 0; // init summable parameters @@ -110,9 +47,9 @@ void Cluster::coG() int maxQ = -1; // to calculate the pad with the highest charge o2::hmpid::Digit* pDig = nullptr; - for (int iDig = 0; iDig < mDigs.size(); iDig++) { // digits loop + for (int iDig = 0; iDig < mDigs->size(); iDig++) { // digits loop int x, y, mod; - int padId = mDigs[iDig]->getPadID(); + int padId = (*mDigs)[iDig]->getPadID(); o2::hmpid::Digit::pad2Absolute(padId, &mod, &x, &y); if (x > maxPadX) { maxPadX = x; @@ -127,7 +64,7 @@ void Cluster::coG() minPadY = y; } // MinY - float q = mDigs[iDig]->mQ; // get QDC + float q = (*mDigs)[iDig]->mQ; // get QDC mXX += o2::hmpid::Digit::lorsX(padId) * q; mYY += o2::hmpid::Digit::lorsY(padId) * q; // add digit center weighted by QDC mQRaw += q; // increment total charge @@ -144,7 +81,7 @@ void Cluster::coG() mYY /= mQRaw; } // final center of gravity - if (mDigs.size() > 1 && fgDoCorrSin) { + if (mDigs->size() > 1 && fgDoCorrSin) { corrSin(); } // correct it by sinoid @@ -162,12 +99,14 @@ void Cluster::coG() // Correction of cluster x position due to sinoid, see HMPID TDR page 30 void Cluster::corrSin() { + const auto param = o2::hmpid::Param::instanceNoGeo(); int pc; int px; int py; o2::hmpid::Param::lors2Pad(mXX, mYY, pc, px, py); // tmp digit to get it center - double x = mXX - mParam->lorsX(pc, px); // diff between cluster x and center of the pad contaning this cluster - mXX += 3.31267e-2 * sin(2 * M_PI / 0.8 * x) - 2.66575e-3 * sin(4 * M_PI / 0.8 * x) + 2.80553e-3 * sin(6 * M_PI / 0.8 * x) + 0.0070; + double x = mXX - param->lorsX(pc, px); // diff between cluster x and center of the pad contaning this cluster + double xpi8on10 = M_PI / 0.8 * x; + mXX += 3.31267e-2 * sin(2.0 * xpi8on10) - 2.66575e-3 * sin(4.0 * xpi8on10) + 2.80553e-3 * sin(6.0 * xpi8on10) + 0.0070; return; } @@ -197,8 +136,11 @@ void Cluster::fitFunc(int& iNpars, double* deriv, double& chi2, double* par, int for (int i = 0; i < nPads; i++) { // loop on all pads of the cluster double dQpadMath = 0; for (int j = 0; j < iNshape; j++) { // Mathiesons loop as all of them may contribute to this pad - double fracMathi = o2::hmpid::Digit::intMathieson(par[3 * j], par[3 * j + 1], pClu->dig(i)->getPadID()); - dQpadMath += par[3 * j + 2] * fracMathi; // par[3*j+2] is charge par[3*j] is x par[3*j+1] is y of current Mathieson + int baseOff = 3 * j; + int baseOff1 = baseOff + 1; + int baseOff2 = baseOff + 2; + double fracMathi = o2::hmpid::Digit::intMathieson(par[baseOff], par[baseOff1], pClu->dig(i)->getPadID()); + dQpadMath += par[baseOff2] * fracMathi; // par[3*j+2] is charge par[3*j] is x par[3*j+1] is y of current Mathieson } if (dQpadMath > 0 && pClu->dig(i)->mQ > 0) { chi2 += std::pow((pClu->dig(i)->mQ - dQpadMath), 2.0) / pClu->dig(i)->mQ; // chi2 function to be minimized @@ -206,45 +148,46 @@ void Cluster::fitFunc(int& iNpars, double* deriv, double& chi2, double* par, int } //---calculate gradients... if (iflag == 2) { - float** derivPart; - derivPart = new float*[iNpars]; - for (int j = 0; j < iNpars; j++) { - deriv[j] = 0; - derivPart[j] = new float[nPads]; - for (int i = 0; i < nPads; i++) { - derivPart[j][i] = 0; - } - } - for (int i = 0; i < nPads; i++) { // loop on all pads of the cluster + std::vector> derivPart(iNpars, std::vector(nPads, 0.0f)); + double dHalfPadX = o2::hmpid::Param::sizeHalfPadX(); + double dHalfPadY = o2::hmpid::Param::sizeHalfPadY(); + + for (int i = 0; i < nPads; i++) { // loop on all pads of the cluster + int iPadId = pClu->dig(i)->getPadID(); + double lx = o2::hmpid::Digit::lorsX(iPadId); + double ly = o2::hmpid::Digit::lorsY(iPadId); for (int j = 0; j < iNshape; j++) { // Mathiesons loop as all of them may contribute to this pad - double fracMathi = o2::hmpid::Digit::intMathieson(par[3 * j], par[3 * j + 1], pClu->dig(i)->getPadID()); - double lx = o2::hmpid::Digit::lorsX(pClu->dig(i)->getPadID()); - double ly = o2::hmpid::Digit::lorsY(pClu->dig(i)->getPadID()); - derivPart[3 * j][i] += par[3 * j + 2] * (o2::hmpid::Digit::mathiesonX(par[3 * j] - lx - 0.5 * o2::hmpid::Param::sizePadX()) - o2::hmpid::Digit::mathiesonX(par[3 * j] - lx + 0.5 * o2::hmpid::Param::sizePadX())) * - o2::hmpid::Digit::intPartMathiY(par[3 * j + 1], pClu->dig(i)->getPadID()); - derivPart[3 * j + 1][i] += par[3 * j + 2] * (o2::hmpid::Digit::mathiesonY(par[3 * j + 1] - ly - 0.5 * o2::hmpid::Param::sizePadY()) - o2::hmpid::Digit::mathiesonY(par[3 * j + 1] - ly + 0.5 * o2::hmpid::Param::sizePadY())) * - o2::hmpid::Digit::intPartMathiX(par[3 * j], pClu->dig(i)->getPadID()); - derivPart[3 * j + 2][i] += fracMathi; + int baseOff = 3 * j; + int baseOff1 = baseOff + 1; + int baseOff2 = baseOff + 2; + double fracMathi = o2::hmpid::Digit::intMathieson(par[baseOff], par[baseOff1], iPadId); + derivPart[baseOff][i] += par[baseOff2] * (o2::hmpid::Digit::mathiesonX(par[baseOff] - lx - dHalfPadX) - o2::hmpid::Digit::mathiesonX(par[baseOff] - lx + dHalfPadX)) * + o2::hmpid::Digit::intPartMathiY(par[baseOff1], iPadId); + derivPart[baseOff1][i] += par[baseOff2] * (o2::hmpid::Digit::mathiesonY(par[baseOff1] - ly - dHalfPadY) - o2::hmpid::Digit::mathiesonY(par[baseOff1] - ly + dHalfPadY)) * + o2::hmpid::Digit::intPartMathiX(par[baseOff], iPadId); + derivPart[baseOff2][i] += fracMathi; } } // loop on all pads of the cluster - for (int i = 0; i < nPads; i++) { // loop on all pads of the cluster - double dQpadMath = 0; // pad charge collector + for (int i = 0; i < nPads; i++) { // loop on all pads of the cluster + int iPadId = pClu->dig(i)->getPadID(); + double dPadmQ = pClu->dig(i)->mQ; + double dQpadMath = 0.0; // pad charge collector + double twoOverMq = 2.0 / dPadmQ; for (int j = 0; j < iNshape; j++) { // Mathiesons loop as all of them may contribute to this pad - float fracMathi = o2::hmpid::Digit::intMathieson(par[3 * j], par[3 * j + 1], pClu->dig(i)->getPadID()); - dQpadMath += par[3 * j + 2] * fracMathi; - if (dQpadMath > 0 && pClu->dig(i)->mQ > 0) { - deriv[3 * j] += 2 / pClu->dig(i)->mQ * (pClu->dig(i)->mQ - dQpadMath) * derivPart[3 * j][i]; - deriv[3 * j + 1] += 2 / pClu->dig(i)->mQ * (pClu->dig(i)->mQ - dQpadMath) * derivPart[3 * j + 1][i]; - deriv[3 * j + 2] += 2 / pClu->dig(i)->mQ * (pClu->dig(i)->mQ - dQpadMath) * derivPart[3 * j + 2][i]; + int baseOff = 3 * j; + int baseOff1 = baseOff + 1; + int baseOff2 = baseOff + 2; + double fracMathi = o2::hmpid::Digit::intMathieson(par[baseOff], par[baseOff1], iPadId); + dQpadMath += par[baseOff2] * fracMathi; + if (dQpadMath > 0 && dPadmQ > 0) { + double appoggio = twoOverMq * (dPadmQ - dQpadMath); + deriv[baseOff] += appoggio * derivPart[baseOff][i]; + deriv[baseOff1] += appoggio * derivPart[baseOff1][i]; + deriv[baseOff2] += appoggio * derivPart[baseOff2][i]; } } } - // delete array... - for (int i = 0; i < iNpars; i++) { - delete[] derivPart[i]; - } - delete[] derivPart; } //---gradient calculations ended // fit ended. Final calculations } // FitFunction() @@ -301,10 +244,10 @@ void Cluster::print(Option_t* opt) const } printf("%sCLU: ch=%i (X = %7.3f, Y = %7.3f) Q=%8.3f Qraw=%8.3f(%3.0f%%) Size=%2i DimBox=%i LocMax=%i Chi2=%7.3f %s\n", opt, mCh, mXX, mYY, mQ, mQRaw, ratio, mSi, mBox, mNlocMax, mChi2, status); - if (mDigs.size() > 0) { + if (mDigs && mDigs->size() > 0) { std::cout << "Digits of Cluster" << std::endl; - for (int i; i < mDigs.size(); i++) { - std::cout << mDigs[i] << std::endl; + for (int i; i < mDigs->size(); i++) { + std::cout << (*mDigs)[i] << std::endl; } } return; @@ -321,6 +264,10 @@ int Cluster::solve(std::vector* pCluLst, float* pSigmaCut, b // Arguments: pCluLst - cluster list pointer where to add new cluster(s) // isTryUnfold - flag to switch on/off unfolding // Returns: number of local maxima of original cluster + const auto param = o2::hmpid::Param::instanceNoGeo(); + if (!mDigs) { + LOGP(fatal, "digits are missing in the cluster"); + } const int kMaxLocMax = 6; // max allowed number of loc max for fitting coG(); // First calculate CoG for the given cluster int iCluCnt = pCluLst->size(); // get current number of clusters already stored in the list by previous operations @@ -336,13 +283,18 @@ int Cluster::solve(std::vector* pCluLst, float* pSigmaCut, b // setClusterParams(mXX, mYY, mCh); // 2 - flag is set to FALSE // new ((*pCluLst)[iCluCnt++]) Cluster(*this); // 3 - size = 1 pCluLst->push_back(o2::hmpid::Cluster(*this)); + pCluLst->back().cleanPointers(); return 1; // add this raw cluster } // Phase 0. Initialise Fitter - double arglist[10]; + double arglist[10]{0., 0., 0., 0., 0., 0., 0., 0., 0., 0.}; float ierflg = 0.; TVirtualFitter* fitter = TVirtualFitter::Fitter((TObject*)this, 3 * 6); // initialize Fitter + if (fitter == nullptr) { + LOG(fatal) << "TVirtualFitter could not be created"; + return 1; + } arglist[0] = -1; ierflg = fitter->ExecuteCommand("SET PRI", arglist, 1); // no printout ierflg = fitter->ExecuteCommand("SET NOW", arglist, 0); // no warning messages @@ -352,13 +304,13 @@ int Cluster::solve(std::vector* pCluLst, float* pSigmaCut, b // Phase 1. Find number of local maxima. Strategy is to check if the current pad has QDC more then all neigbours. Also find the box contaning the cluster mNlocMax = 0; for (int iDig1 = 0; iDig1 < rawSize; iDig1++) { // first digits loop - auto pDig1 = mDigs.at(iDig1); // take next digit + auto pDig1 = (*mDigs)[iDig1]; // take next digit int iCnt = 0; // counts how many neighbouring pads has QDC more then current one for (int iDig2 = 0; iDig2 < rawSize; iDig2++) { // loop on all digits again if (iDig1 == iDig2) { continue; } // the same digit, no need to compare - auto pDig2 = mDigs.at(iDig2); // take second digit to compare with the first one + auto pDig2 = (*mDigs)[iDig2]; // take second digit to compare with the first one int dist = TMath::Sign(int(pDig1->mX - pDig2->mX), 1) + TMath::Sign(int(pDig1->mY - pDig2->mY), 1); // distance between pads if (dist == 1) { // means dig2 is a neighbour of dig1 if (pDig2->mQ >= pDig1->mQ) { @@ -369,10 +321,10 @@ int Cluster::solve(std::vector* pCluLst, float* pSigmaCut, b if (iCnt == 0 && mNlocMax < kMaxLocMax) { // this pad has Q more then any neighbour so it's local maximum float xStart = o2::hmpid::Digit::lorsX(pDig1->getPadID()); float yStart = o2::hmpid::Digit::lorsY(pDig1->getPadID()); - float xMin = xStart - mParam->sizePadX(); - float xMax = xStart + mParam->sizePadX(); - float yMin = yStart - mParam->sizePadY(); - float yMax = yStart + mParam->sizePadY(); + float xMin = xStart - param->sizePadX(); + float xMax = xStart + param->sizePadX(); + float yMin = yStart - param->sizePadY(); + float yMax = yStart + param->sizePadY(); ierflg = fitter->SetParameter(3 * mNlocMax, Form("x%i", mNlocMax), xStart, 0.1, xMin, xMax); // X,Y,Q initial values of the loc max pad ierflg = fitter->SetParameter(3 * mNlocMax + 1, Form("y%i", mNlocMax), yStart, 0.1, yMin, yMax); // X, Y constrained to be near the loc max ierflg = fitter->SetParameter(3 * mNlocMax + 2, Form("q%i", mNlocMax), pDig1->mQ, 0.1, 0, 10000); // Q constrained to be positive @@ -387,6 +339,7 @@ int Cluster::solve(std::vector* pCluLst, float* pSigmaCut, b mSt = kNoLoc; // setClusterParams(mXX, mYY, mCh); //need to fill the AliCluster3D part pCluLst->push_back(o2::hmpid::Cluster(*this)); // add new unfolded cluster pCluLst->push_back(o2::hmpid::Cluster(*this)); + pCluLst->back().cleanPointers(); return mNlocMax; } @@ -395,10 +348,12 @@ int Cluster::solve(std::vector* pCluLst, float* pSigmaCut, b // setClusterParams(mXX, mYY, mCh); // if # of local maxima exceeds kMaxLocMax... mSt = kMax; pCluLst->push_back(o2::hmpid::Cluster(*this)); //...add this raw cluster - } else { // or resonable number of local maxima to fit and user requested it + pCluLst->back().cleanPointers(); + } else { // or resonable number of local maxima to fit and user requested it // Now ready for minimization step - arglist[0] = 500; // number of steps and sigma on pads charges - arglist[1] = 1.; // + arglist[0] = 500; // number of steps and sigma on pads charges + arglist[1] = 1.; // + /* ierflg = fitter->ExecuteCommand("SIMPLEX", arglist, 2); // start fitting with Simplex if (!ierflg) { fitter->ExecuteCommand("MIGRAD", arglist, 2); // fitting improved by Migrad @@ -413,6 +368,14 @@ int Cluster::solve(std::vector* pCluLst, float* pSigmaCut, b } } } + */ + + double strategy = 2.; + ierflg = fitter->ExecuteCommand("SET STR", &strategy, 1); // change level of strategy + if (!ierflg) { + fitter->ExecuteCommand("MIGRAD", arglist, 2); // fitting improved by Migrad + } + if (ierflg) { mSt = kAbn; // no convergence of the fit... } @@ -427,6 +390,7 @@ int Cluster::solve(std::vector* pCluLst, float* pSigmaCut, b fitter->GetParameter(3 * i + 1, sName, mYY, mErrY, dummy, dummy); // Y fitter->GetParameter(3 * i + 2, sName, mQ, mErrQ, dummy, dummy); // Q fitter->GetStats(mChi2, edm, errdef, nvpar, nparx); // get fit infos + // Printf("********************loc. max. = %i, X= %f, Y = %f, Q = %f**************************",i,mXX,mYY,mQ); if (mNlocMax > 1) { findClusterSize(i, pSigmaCut); // find clustersize for deconvoluted clusters // after this call, fSi temporarly is the calculated size. Later is set again @@ -447,7 +411,9 @@ int Cluster::solve(std::vector* pCluLst, float* pSigmaCut, b } } // setClusterParams(mXX, mYY, mCh); //need to fill the AliCluster3D part + // Printf("********************loc. max. = %i, X= %f, Y = %f, Q = %f**************************",i,mXX,mYY,mQ); pCluLst->push_back(o2::hmpid::Cluster(*this)); // add new unfolded cluster + pCluLst->back().cleanPointers(); if (mNlocMax > 1) { setSize(rawSize); // Original raw size is set again to its proper value } @@ -483,9 +449,13 @@ Bool_t Cluster::isInPc() // Check if (X,Y) position is inside the PC limits // Arguments: // Returns: True or False - int pc = mDigs[0]->getPh(); // (o2::hmpid::Digit*)&mDigs.at(iDig) + const auto param = o2::hmpid::Param::instanceNoGeo(); + if (!mDigs) { + LOGP(fatal, "digits are missing in the cluster"); + } + int pc = (*mDigs)[0]->getPh(); // (o2::hmpid::Digit*)&mDigs.at(iDig) - if (mXX < Param::minPcX(pc) || mXX > Param::maxPcX(pc) || mYY < Param::minPcY(pc) || mYY > Param::maxPcY(pc)) { + if (mXX < param->minPcX(pc) || mXX > param->maxPcX(pc) || mYY < param->minPcY(pc) || mYY > param->maxPcY(pc)) { return false; } @@ -497,12 +467,14 @@ void Cluster::digAdd(const Digit* pDig) // Adds a given digit to the list of digits belonging to this cluster, cluster is not owner of digits // Arguments: pDig - pointer to digit to be added // Returns: none - if (mDigs.size() == 0) { // create list of digits in the first invocation + if (!mDigs) { + LOGP(fatal, "digits are not set to the cluster"); + } + if (mDigs->size() == 0) { // create list of digits in the first invocation mSi = 0; - // std::vector fDigs; } // fDigs->Add(pDig); - mDigs.push_back(pDig); + mDigs->push_back(pDig); mSt = kFrm; mSi++; return; @@ -511,11 +483,7 @@ void Cluster::digAdd(const Digit* pDig) void Cluster::reset() { // - // - // - if (mDigs.size() > 0) { - mDigs.clear(); - } + cleanPointers(); // mDigs={0x0}; mSt = kEmp; mQRaw = mQ = 0; diff --git a/DataFormats/Detectors/HMPID/src/Digit.cxx b/DataFormats/Detectors/HMPID/src/Digit.cxx index 397d1a3cd5d0d..89726a5d28c95 100644 --- a/DataFormats/Detectors/HMPID/src/Digit.cxx +++ b/DataFormats/Detectors/HMPID/src/Digit.cxx @@ -1,4 +1,4 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// Copyright 2020-2022 CERN and copyright holders of ALICE O2. // See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. // All rights not expressly granted are reserved. // @@ -325,8 +325,8 @@ Double_t Digit::qdcTot(Double_t e, Double_t time, Int_t pc, Int_t px, Int_t py, /// @return : a charge fraction [0-1] imposed into the pad double Digit::intPartMathiX(double x, int pad) { - double shift1 = -lorsX(pad) + 0.5 * o2::hmpid::Param::sizePadX(); - double shift2 = -lorsX(pad) - 0.5 * o2::hmpid::Param::sizePadX(); + double shift1 = -lorsX(pad) + o2::hmpid::Param::sizeHalfPadX(); + double shift2 = -lorsX(pad) - o2::hmpid::Param::sizeHalfPadX(); double ux1 = o2::hmpid::Param::sqrtK3x() * tanh(o2::hmpid::Param::k2x() * (x + shift1) / o2::hmpid::Param::pitchAnodeCathode()); double ux2 = o2::hmpid::Param::sqrtK3x() * tanh(o2::hmpid::Param::k2x() * (x + shift2) / o2::hmpid::Param::pitchAnodeCathode()); @@ -343,8 +343,8 @@ double Digit::intPartMathiX(double x, int pad) /// @return : a charge fraction [0-1] imposed into the pad double Digit::intPartMathiY(double y, int pad) { - double shift1 = -lorsY(pad) + 0.5 * o2::hmpid::Param::sizePadY(); - double shift2 = -lorsY(pad) - 0.5 * o2::hmpid::Param::sizePadY(); + double shift1 = -lorsY(pad) + o2::hmpid::Param::sizeHalfPadY(); + double shift2 = -lorsY(pad) - o2::hmpid::Param::sizeHalfPadY(); double uy1 = o2::hmpid::Param::sqrtK3y() * tanh(o2::hmpid::Param::k2y() * (y + shift1) / o2::hmpid::Param::pitchAnodeCathode()); double uy2 = o2::hmpid::Param::sqrtK3y() * tanh(o2::hmpid::Param::k2y() * (y + shift2) / o2::hmpid::Param::pitchAnodeCathode()); diff --git a/DataFormats/Detectors/ITSMFT/CMakeLists.txt b/DataFormats/Detectors/ITSMFT/CMakeLists.txt index 60adadd6778b4..5d5e6da13067b 100644 --- a/DataFormats/Detectors/ITSMFT/CMakeLists.txt +++ b/DataFormats/Detectors/ITSMFT/CMakeLists.txt @@ -12,6 +12,3 @@ add_subdirectory(common) add_subdirectory(ITS) add_subdirectory(MFT) -if(ENABLE_UPGRADES) - add_subdirectory(IT3) -endif() diff --git a/DataFormats/Detectors/ITSMFT/IT3/CMakeLists.txt b/DataFormats/Detectors/ITSMFT/IT3/CMakeLists.txt deleted file mode 100644 index 944ca48df1191..0000000000000 --- a/DataFormats/Detectors/ITSMFT/IT3/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2019-2020 CERN and copyright holders of ALICE O2. -# See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -# All rights not expressly granted are reserved. -# -# This software is distributed under the terms of the GNU General Public -# License v3 (GPL Version 3), copied verbatim in the file "COPYING". -# -# In applying this license CERN does not waive the privileges and immunities -# granted to it by virtue of its status as an Intergovernmental Organization -# or submit itself to any jurisdiction. - -o2_add_library(DataFormatsITS3 - SOURCES src/CompCluster.cxx) - -o2_target_root_dictionary(DataFormatsITS3 - HEADERS include/DataFormatsITS3/CompCluster.h - LINKDEF src/ITS3DataFormatsLinkDef.h) diff --git a/DataFormats/Detectors/ITSMFT/IT3/include/DataFormatsITS3/CompCluster.h b/DataFormats/Detectors/ITSMFT/IT3/include/DataFormatsITS3/CompCluster.h deleted file mode 100644 index 03da492fd56c4..0000000000000 --- a/DataFormats/Detectors/ITSMFT/IT3/include/DataFormatsITS3/CompCluster.h +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// \file CompCluster.h -/// \brief Definition of the ITS3 compact cluster -#ifndef ALICEO2_ITS3_COMPCLUSTER_H -#define ALICEO2_ITS3_COMPCLUSTER_H - -#include - -namespace o2 -{ -namespace its3 -{ - -/// This is a version of the ALPIDE cluster represented by the pattern ID and the address of the -/// top-left (min row,col) pixel of the topololy bounding box - -class CompCluster -{ - public: - static constexpr int NBitsPattID = 11; // number of bits for the pattern ID - private: - static constexpr UShort_t PattIDMask = (0x1 << NBitsPattID) - 1; - static constexpr UShort_t FlagBit = 0x1 << (NBitsPattID); - // - ///< compactified data: bits [0:8] - row, [9-18] - col, [19-30] - pattern ID, bit 31 - special flag - UShort_t mRow; - UShort_t mCol; - UShort_t mPat; - - public: - static constexpr unsigned short InvalidPatternID = (0x1 << NBitsPattID) - 1; // All 11 bits of pattern ID are 1 - CompCluster(UShort_t row = 0, UShort_t col = 0, UShort_t patt = 0) - { - set(row, col, patt); - } - - void set(UShort_t row, UShort_t col, UShort_t patt) - { - setPatternID(patt); - mRow = row; - mCol = col; - } - - UShort_t getRow() const { return mRow; } - UShort_t getCol() const { return mCol; } - UShort_t getPatternID() const { return (mPat & PattIDMask); } - bool getFlag() const { return (mPat & FlagBit) == FlagBit; } - - void setRow(UShort_t r) - { - mRow = r; - } - void setCol(UShort_t c) - { - mCol = c; - }; - void setPatternID(UShort_t p) - { - mPat &= ~(PattIDMask); - mPat |= p; - } - void setFlag(bool v) - { - mPat &= ~FlagBit; - if (v) { - mPat |= FlagBit; - } - } - - void print() const; - - ClassDefNV(CompCluster, 1); -}; - -/// Extension of the compact cluster, augmented by the chipID -/// This is a TEMPORARY class, until we converge to more economical container -class CompClusterExt : public CompCluster -{ - private: - UShort_t mChipID; ///< chip id - - public: - CompClusterExt(UShort_t row = 0, UShort_t col = 0, UShort_t patt = 0, UShort_t chipID = 0) : CompCluster(row, col, patt), mChipID(chipID) - { - } - - void set(UShort_t row, UShort_t col, UShort_t patt, UShort_t chipID) - { - CompCluster::set(row, col, patt); - mChipID = chipID; - } - - UShort_t getChipID() const { return mChipID; } - UShort_t getSensorID() const { return mChipID; } // to have the same signature as BaseCluster - - void setChipID(UShort_t c) { mChipID = c; } - - void print() const; - - ClassDefNV(CompClusterExt, 1); -}; - -} // namespace its3 -} // namespace o2 - -std::ostream& operator<<(std::ostream& stream, const o2::its3::CompCluster& cl); -std::ostream& operator<<(std::ostream& stream, const o2::its3::CompClusterExt& cl); - -#endif /* ALICEO2_ITS3_COMPACTCLUSTER_H */ diff --git a/DataFormats/Detectors/ITSMFT/IT3/src/CompCluster.cxx b/DataFormats/Detectors/ITSMFT/IT3/src/CompCluster.cxx deleted file mode 100644 index 046a27be92919..0000000000000 --- a/DataFormats/Detectors/ITSMFT/IT3/src/CompCluster.cxx +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// \file Cluster.cxx -/// \brief Implementation of the ITSMFT cluster - -#include "DataFormatsITS3/CompCluster.h" -#include -#include - -using namespace o2::its3; - -std::ostream& operator<<(std::ostream& stream, const CompCluster& cl) -{ - stream << " row: " << cl.getRow() << " col: " << cl.getCol() - << " pattID " << cl.getPatternID() << " [flag: " << cl.getFlag() << "] "; - return stream; -} - -std::ostream& operator<<(std::ostream& stream, const CompClusterExt& cl) -{ - stream << " chip: " << cl.getChipID() << ((const CompCluster&)cl); - return stream; -} - -//______________________________________________________________________________ -void CompCluster::print() const -{ - // print itself - std::cout << *this << "\n"; -} - -//______________________________________________________________________________ -void CompClusterExt::print() const -{ - // print itself - std::cout << *this << "\n"; -} diff --git a/DataFormats/Detectors/ITSMFT/IT3/src/ITS3DataFormatsLinkDef.h b/DataFormats/Detectors/ITSMFT/IT3/src/ITS3DataFormatsLinkDef.h deleted file mode 100644 index c139171540f18..0000000000000 --- a/DataFormats/Detectors/ITSMFT/IT3/src/ITS3DataFormatsLinkDef.h +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -#ifdef __CLING__ - -#pragma link off all globals; -#pragma link off all classes; -#pragma link off all functions; - -#pragma link C++ class o2::its3::CompCluster + ; -#pragma link C++ class o2::its3::CompClusterExt + ; -#pragma link C++ class std::vector < o2::its3::CompCluster> + ; -#pragma link C++ class std::vector < o2::its3::CompClusterExt> + ; - -#endif diff --git a/DataFormats/Detectors/ITSMFT/ITS/include/DataFormatsITS/TrackITS.h b/DataFormats/Detectors/ITSMFT/ITS/include/DataFormatsITS/TrackITS.h index 7c2dd982564c6..06d4fba51bd54 100644 --- a/DataFormats/Detectors/ITSMFT/ITS/include/DataFormatsITS/TrackITS.h +++ b/DataFormats/Detectors/ITSMFT/ITS/include/DataFormatsITS/TrackITS.h @@ -35,7 +35,8 @@ namespace its class TrackITS : public o2::track::TrackParCov { enum UserBits { - kNextROF = 1 + kNextROF = 1 << 28, + kSharedClusters = 1 << 29 }; using Cluster = o2::itsmft::Cluster; @@ -45,22 +46,23 @@ class TrackITS : public o2::track::TrackParCov using o2::track::TrackParCov::TrackParCov; // inherit base constructors static constexpr int MaxClusters = 16; - GPUdDefault() TrackITS() = default; - GPUdDefault() TrackITS(const TrackITS& t) = default; - GPUd() TrackITS(const o2::track::TrackParCov& parcov) : o2::track::TrackParCov{parcov} {} - GPUd() TrackITS(const o2::track::TrackParCov& parCov, float chi2, const o2::track::TrackParCov& outer) + GPUhdDefault() TrackITS() = default; + GPUhdDefault() TrackITS(const TrackITS& t) = default; + GPUhd() TrackITS(const o2::track::TrackParCov& parcov) : o2::track::TrackParCov{parcov} {} + GPUhd() TrackITS(const o2::track::TrackParCov& parCov, float chi2, const o2::track::TrackParCov& outer) : o2::track::TrackParCov{parCov}, mParamOut{outer}, mChi2{chi2} {} - GPUdDefault() TrackITS& operator=(const TrackITS& tr) = default; - GPUdDefault() ~TrackITS() = default; + GPUhdDefault() TrackITS& operator=(const TrackITS& tr) = default; + GPUhdDefault() TrackITS& operator=(TrackITS&& tr) = default; + GPUhdDefault() ~TrackITS() = default; // These functions must be provided bool propagate(float alpha, float x, float bz); bool update(const Cluster& c, float chi2); // Other functions - float getChi2() const { return mChi2; } - int getNClusters() const { return mClusRef.getEntries(); } - int getNumberOfClusters() const { return getNClusters(); } + GPUhdi() float getChi2() const { return mChi2; } + GPUhdi() int getNClusters() const { return mClusRef.getEntries(); } + GPUhdi() int getNumberOfClusters() const { return getNClusters(); } int getFirstClusterEntry() const { return mClusRef.getFirstEntry(); } int getClusterEntry(int i) const { return getFirstClusterEntry() + i; } void shiftFirstClusterEntry(int bias) @@ -85,22 +87,23 @@ class TrackITS : public o2::track::TrackParCov } const ClusRefs& getClusterRefs() const { return mClusRef; } - ClusRefs& getClusterRefs() { return mClusRef; } + GPUhdi() ClusRefs& getClusterRefs() { return mClusRef; } - void setChi2(float chi2) { mChi2 = chi2; } + GPUhdi() void setChi2(float chi2) { mChi2 = chi2; } bool isBetter(const TrackITS& best, float maxChi2) const; - o2::track::TrackParCov& getParamIn() { return *this; } - const o2::track::TrackParCov& getParamIn() const { return *this; } + GPUhdi() o2::track::TrackParCov& getParamIn() { return *this; } + GPUhdi() const o2::track::TrackParCov& getParamIn() const { return *this; } - o2::track::TrackParCov& getParamOut() { return mParamOut; } - const o2::track::TrackParCov& getParamOut() const { return mParamOut; } + GPUhdi() o2::track::TrackParCov& getParamOut() { return mParamOut; } + GPUhdi() const o2::track::TrackParCov& getParamOut() const { return mParamOut; } - void setPattern(uint32_t p) { mPattern = p; } - uint32_t getPattern() const { return mPattern; } - bool hasHitOnLayer(int i) { return mPattern & (0x1 << i); } - bool isFakeOnLayer(int i) { return !(mPattern & (0x1 << (16 + i))); } + GPUhdi() void setPattern(uint32_t p) { mPattern = p; } + GPUhdi() uint32_t getPattern() const { return mPattern; } + bool hasHitOnLayer(uint32_t i) const { return mPattern & (0x1 << i); } + bool isFakeOnLayer(uint32_t i) const { return !(mPattern & (0x1 << (16 + i))); } + bool isExtendedOnLayer(uint32_t i) const { return (mPattern & (0x1 << (24 + i))); } // only correct if getNClusters <= 8 on layers <= 8 uint32_t getLastClusterLayer() const { uint32_t r{0}, v{mPattern & ((1 << 16) - 1)}; @@ -117,18 +120,46 @@ class TrackITS : public o2::track::TrackParCov } return s; } - int getNFakeClusters(); + int getNFakeClusters() const; - void setNextROFbit(bool toggle = true) { setUserField((getUserField() & ~kNextROF) | (-toggle & kNextROF)); } - bool hasHitInNextROF() const { return getUserField() & kNextROF; } + void setNextROFbit(bool toggle = true) { mClusterSizes = toggle ? (mClusterSizes | kNextROF) : (mClusterSizes & ~kNextROF); } + bool hasHitInNextROF() const { return mClusterSizes & kNextROF; } + void setSharedClusters(bool toggle = true) { mClusterSizes = toggle ? (mClusterSizes | kSharedClusters) : (mClusterSizes & ~kSharedClusters); } + bool hasSharedClusters() const { return mClusterSizes & kSharedClusters; } + + void setClusterSize(int l, int size) + { + if (l >= 8) { + return; + } + if (size > 15) { + size = 15; + } + mClusterSizes &= ~(0xf << (l * 4)); + mClusterSizes |= (size << (l * 4)); + } + + int getClusterSize(int l) + { + if (l >= 7) { + return 0; + } + return (mClusterSizes >> (l * 4)) & 0xf; + } + + int getClusterSizes() const + { + return mClusterSizes; + } private: o2::track::TrackParCov mParamOut; ///< parameter at largest radius ClusRefs mClusRef; ///< references on clusters float mChi2 = 0.; ///< Chi2 for this track uint32_t mPattern = 0; ///< layers pattern + unsigned int mClusterSizes = 0u; - ClassDefNV(TrackITS, 5); + ClassDefNV(TrackITS, 6); }; class TrackITSExt : public TrackITS @@ -138,21 +169,21 @@ class TrackITSExt : public TrackITS static constexpr int MaxClusters = 16; /// Prepare for overlaps and new detector configurations using TrackITS::TrackITS; // inherit base constructors - GPUd() TrackITSExt(o2::track::TrackParCov&& parCov, short ncl, float chi2, + GPUh() TrackITSExt(o2::track::TrackParCov&& parCov, short ncl, float chi2, o2::track::TrackParCov&& outer, std::array cls) : TrackITS(parCov, chi2, outer), mIndex{cls} { setNumberOfClusters(ncl); } - GPUd() TrackITSExt(o2::track::TrackParCov& parCov, short ncl, float chi2, std::uint32_t rof, + GPUh() TrackITSExt(o2::track::TrackParCov& parCov, short ncl, float chi2, std::uint32_t rof, o2::track::TrackParCov& outer, std::array cls) : TrackITS(parCov, chi2, outer), mIndex{cls} { setNumberOfClusters(ncl); } - GPUdDefault() TrackITSExt(const TrackITSExt& t) = default; + GPUhdDefault() TrackITSExt(const TrackITSExt& t) = default; void setClusterIndex(int l, int i) { @@ -161,9 +192,9 @@ class TrackITSExt : public TrackITS getClusterRefs().setEntries(ncl); } - int getClusterIndex(int lr) const { return mIndex[lr]; } + GPUhdi() const int& getClusterIndex(int lr) const { return mIndex[lr]; } - void setExternalClusterIndex(int layer, int idx, bool newCluster = false) + GPUhdi() void setExternalClusterIndex(int layer, int idx, bool newCluster = false) { if (newCluster) { getClusterRefs().setEntries(getNumberOfClusters() + 1); @@ -174,7 +205,7 @@ class TrackITSExt : public TrackITS mIndex[layer] = idx; } - std::array& getClusterIndexes() + GPUh() std::array& getClusterIndexes() { return mIndex; } diff --git a/DataFormats/Detectors/ITSMFT/ITS/src/TrackITS.cxx b/DataFormats/Detectors/ITSMFT/ITS/src/TrackITS.cxx index 6f0fb6256daf2..0244756647120 100644 --- a/DataFormats/Detectors/ITSMFT/ITS/src/TrackITS.cxx +++ b/DataFormats/Detectors/ITSMFT/ITS/src/TrackITS.cxx @@ -29,31 +29,31 @@ bool TrackITS::operator<(const TrackITS& o) const //----------------------------------------------------------------- // This function compares tracks according to the their curvature //----------------------------------------------------------------- - Float_t co = TMath::Abs(o.getPt()); - Float_t c = TMath::Abs(getPt()); - // Float_t co=o.GetSigmaY2()*o.GetSigmaZ2(); - // Float_t c =GetSigmaY2()*GetSigmaZ2(); + float co = TMath::Abs(o.getPt()); + float c = TMath::Abs(getPt()); + // float co=o.GetSigmaY2()*o.GetSigmaZ2(); + // float c =GetSigmaY2()*GetSigmaZ2(); return (c > co); } -void TrackITS::getImpactParams(Float_t x, Float_t y, Float_t z, Float_t bz, Float_t ip[2]) const +void TrackITS::getImpactParams(float x, float y, float z, float bz, float ip[2]) const { //------------------------------------------------------------------ // This function calculates the transverse and longitudinal impact parameters // with respect to a point with global coordinates (x,y,0) // in the magnetic field "bz" (kG) //------------------------------------------------------------------ - Float_t f1 = getSnp(), r1 = TMath::Sqrt((1. - f1) * (1. + f1)); - Float_t xt = getX(), yt = getY(); - Float_t sn = TMath::Sin(getAlpha()), cs = TMath::Cos(getAlpha()); - Float_t a = x * cs + y * sn; + float f1 = getSnp(), r1 = TMath::Sqrt((1. - f1) * (1. + f1)); + float xt = getX(), yt = getY(); + float sn = TMath::Sin(getAlpha()), cs = TMath::Cos(getAlpha()); + float a = x * cs + y * sn; y = -x * sn + y * cs; x = a; xt -= x; yt -= y; - Float_t rp4 = getCurvature(bz); + float rp4 = getCurvature(bz); if ((TMath::Abs(bz) < Almost0) || (TMath::Abs(rp4) < Almost0)) { ip[0] = -(xt * f1 - yt * r1); ip[1] = getZ() + (ip[0] * f1 - xt) / r1 * getTgl() - z; @@ -63,57 +63,59 @@ void TrackITS::getImpactParams(Float_t x, Float_t y, Float_t z, Float_t bz, Floa sn = rp4 * xt - f1; cs = rp4 * yt + r1; a = 2 * (xt * f1 - yt * r1) - rp4 * (xt * xt + yt * yt); - Float_t rr = TMath::Sqrt(sn * sn + cs * cs); + float rr = TMath::Sqrt(sn * sn + cs * cs); ip[0] = -a / (1 + rr); - Float_t f2 = -sn / rr, r2 = TMath::Sqrt((1. - f2) * (1. + f2)); + float f2 = -sn / rr, r2 = TMath::Sqrt((1. - f2) * (1. + f2)); ip[1] = getZ() + getTgl() / rp4 * TMath::ASin(f2 * r1 - f1 * r2) - z; } -Bool_t TrackITS::propagate(Float_t alpha, Float_t x, Float_t bz) +bool TrackITS::propagate(float alpha, float x, float bz) { if (rotate(alpha)) { if (propagateTo(x, bz)) { - return kTRUE; + return true; } } - return kFALSE; + return false; } -Bool_t TrackITS::update(const Cluster& c, Float_t chi2) +bool TrackITS::update(const Cluster& c, float chi2) { //-------------------------------------------------------------------- // Update track params //-------------------------------------------------------------------- if (!o2::track::TrackParCov::update(static_cast&>(c))) { - return kFALSE; + return false; } mChi2 += chi2; - return kTRUE; + return true; } -Bool_t TrackITS::isBetter(const TrackITS& best, Float_t maxChi2) const +bool TrackITS::isBetter(const TrackITS& best, float maxChi2) const { - Int_t ncl = getNumberOfClusters(); - Int_t nclb = best.getNumberOfClusters(); + int ncl = getNumberOfClusters(); + int nclb = best.getNumberOfClusters(); if (ncl >= nclb) { - Float_t chi2 = getChi2(); + float chi2 = getChi2(); if (chi2 < maxChi2) { if (ncl > nclb || chi2 < best.getChi2()) { - return kTRUE; + return true; } } } - return kFALSE; + return false; } -int TrackITS::getNFakeClusters() +int TrackITS::getNFakeClusters() const { int nFake{0}; - for (int iCl{0}; iCl < getNClusters(); ++iCl) { + auto firstClus = getFirstClusterLayer(); + auto lastClus = firstClus + getNClusters(); + for (auto iCl{firstClus}; iCl < lastClus; ++iCl) { if (hasHitOnLayer(iCl) && isFakeOnLayer(iCl)) { ++nFake; } } return nFake; -} \ No newline at end of file +} diff --git a/DataFormats/Detectors/ITSMFT/MFT/include/DataFormatsMFT/TrackMFT.h b/DataFormats/Detectors/ITSMFT/MFT/include/DataFormatsMFT/TrackMFT.h index 8924016e13c6d..04cca15264047 100644 --- a/DataFormats/Detectors/ITSMFT/MFT/include/DataFormatsMFT/TrackMFT.h +++ b/DataFormats/Detectors/ITSMFT/MFT/include/DataFormatsMFT/TrackMFT.h @@ -62,9 +62,32 @@ class TrackMFT : public o2::track::TrackParCovFwd void print() const; + /// get the number of degrees of freedom of the track + int getNDF() const { return 2 * mClusRef.getEntries() - 5; } + /// get the track normalized chi2 + double getChi2OverNDF() const { return getTrackChi2() / getNDF(); } + const o2::track::TrackParCovFwd& getOutParam() const { return mOutParameters; } ///< Returns track parameters fitted outwards void setOutParam(const o2::track::TrackParCovFwd parcov) { mOutParameters = parcov; } ///< Set track out parameters + void setClusterSize(int l, int size) + { + if (l >= 10) { + return; + } + if (size > 63) { + size = 63; + } + + mClusterSizes &= ~(0x3fULL << (l * 6)); + mClusterSizes |= (static_cast(size) << (l * 6)); + } + + uint64_t getClusterSizes() const + { + return mClusterSizes; + } + private: Bool_t mIsCA = false; ///< Track finding method CA vs. LTF @@ -74,8 +97,9 @@ class TrackMFT : public o2::track::TrackParCovFwd Double_t mSeedinvQPtFitChi2 = 0.; ///< Seed InvQPt Chi2 from FCF clusters X,Y positions Double_t mInvQPtSeed; ///< Seed InvQPt from FCF clusters X,Y positions + uint64_t mClusterSizes = 0; ///< MFT cluster sizes per track - ClassDefNV(TrackMFT, 2); + ClassDefNV(TrackMFT, 3); }; class TrackMFTExt : public TrackMFT @@ -89,14 +113,28 @@ class TrackMFTExt : public TrackMFT using TrackMFT::TrackMFT; // inherit base constructors int getExternalClusterIndex(int i) const { return mExtClsIndex[i]; } + int getExternalClusterSize(int i) const { return mExtClsSize[i]; } + int getExternalClusterLayer(int i) const { return mExtClsLayer[i]; } void setExternalClusterIndex(int np, int idx) { mExtClsIndex[np] = idx; } + void setExternalClusterSize(int np, int clsSize) + { + mExtClsSize[np] = clsSize; + } + + void setExternalClusterLayer(int np, int clsLayer) + { + mExtClsLayer[np] = clsLayer; + } + protected: std::array mExtClsIndex = {-1}; ///< External indices of associated clusters + std::array mExtClsSize = {-1}; ///< Cluster size + std::array mExtClsLayer = {-1}; ///< Cluster layer ClassDefNV(TrackMFTExt, 1); }; diff --git a/DataFormats/Detectors/ITSMFT/common/CMakeLists.txt b/DataFormats/Detectors/ITSMFT/common/CMakeLists.txt index 2adaa9bcf24f2..96d376526a1a4 100644 --- a/DataFormats/Detectors/ITSMFT/common/CMakeLists.txt +++ b/DataFormats/Detectors/ITSMFT/common/CMakeLists.txt @@ -18,6 +18,7 @@ o2_add_library(DataFormatsITSMFT src/ClusterPattern.cxx src/ClusterTopology.cxx src/TopologyDictionary.cxx + src/TimeDeadMap.cxx src/CTF.cxx PUBLIC_LINK_LIBRARIES O2::ITSMFTBase O2::ReconstructionDataFormats @@ -25,9 +26,10 @@ o2_add_library(DataFormatsITSMFT o2_target_root_dictionary(DataFormatsITSMFT HEADERS include/DataFormatsITSMFT/ROFRecord.h - include/DataFormatsITSMFT/Digit.h - include/DataFormatsITSMFT/GBTCalibData.h - include/DataFormatsITSMFT/NoiseMap.h + include/DataFormatsITSMFT/Digit.h + include/DataFormatsITSMFT/GBTCalibData.h + include/DataFormatsITSMFT/NoiseMap.h + include/DataFormatsITSMFT/TimeDeadMap.h include/DataFormatsITSMFT/Cluster.h include/DataFormatsITSMFT/CompCluster.h include/DataFormatsITSMFT/ClusterPattern.h diff --git a/DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/CTF.h b/DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/CTF.h index 60f609bc6a447..314523aa878ba 100644 --- a/DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/CTF.h +++ b/DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/CTF.h @@ -36,7 +36,6 @@ struct CTFHeader : public o2::ctf::CTFDictHeader { uint32_t nPatternBytes = 0; /// number of bytes for explict patterns uint32_t firstOrbit = 0; /// 1st orbit of TF uint16_t firstBC = 0; /// 1st BC of TF - ClassDefNV(CTFHeader, 2); }; @@ -47,12 +46,12 @@ struct CompressedClusters { // ROF header data std::vector firstChipROF; /// 1st chip ID in the ROF - std::vector bcIncROF; /// increment of ROF BC wrt BC of previous ROF - std::vector orbitIncROF; /// increment of ROF orbit wrt orbit of previous ROF + std::vector bcIncROF; /// increment of ROF BC wrt BC of previous ROF + std::vector orbitIncROF; /// increment of ROF orbit wrt orbit of previous ROF std::vector nclusROF; /// number of clusters in ROF // Chip data - std::vector chipInc; /// increment of chipID wrt that of prev. chip + std::vector chipInc; /// increment of chipID wrt that of prev. chip std::vector chipMul; /// clusters in chip std::vector row; /// row of fired pixel std::vector colInc; /// increment of pixel column wrt that of prev. pixel (sometimes can be slightly negative) diff --git a/DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/ClusterPattern.h b/DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/ClusterPattern.h index c9c101bfbc5e6..4594412d19aba 100644 --- a/DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/ClusterPattern.h +++ b/DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/ClusterPattern.h @@ -96,6 +96,7 @@ class ClusterPattern int getNPixels() const; /// Prints the pattern friend std::ostream& operator<<(std::ostream& os, const ClusterPattern& top); + void print() const; /// Sets the pattern void setPattern(int nRow, int nCol, const unsigned char patt[MaxPatternBytes]); /// Sets the whole bitmask: the number of rows, the number of columns and the pattern diff --git a/DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/CompCluster.h b/DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/CompCluster.h index 772a9ae12a81a..361544798dc80 100644 --- a/DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/CompCluster.h +++ b/DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/CompCluster.h @@ -15,6 +15,7 @@ #define ALICEO2_ITSMFT_COMPCLUSTER_H #include +#include namespace o2 { @@ -81,7 +82,13 @@ class CompCluster } } + bool operator==(const CompCluster& cl) const + { + return mData == cl.mData; + } + void print() const; + std::string asString() const; ClassDefNV(CompCluster, 2); }; @@ -91,7 +98,7 @@ class CompCluster class CompClusterExt : public CompCluster { private: - UShort_t mChipID; ///< chip id + UShort_t mChipID; ///< chip id public: CompClusterExt(UShort_t row = 0, UShort_t col = 0, UShort_t patt = 0, UShort_t chipID = 0) : CompCluster(row, col, patt), mChipID(chipID) @@ -110,6 +117,7 @@ class CompClusterExt : public CompCluster void setChipID(UShort_t c) { mChipID = c; } void print() const; + std::string asString() const; ClassDefNV(CompClusterExt, 1); }; diff --git a/DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/NoiseMap.h b/DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/NoiseMap.h index 17de7fd953061..25b7f451b6452 100644 --- a/DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/NoiseMap.h +++ b/DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/NoiseMap.h @@ -69,6 +69,15 @@ class NoiseMap mNoisyPixels[chip][getKey(row, col)]++; } + void increaseNoiseCount(int chip, const std::vector& rowcolKey) + { + assert(chip < (int)mNoisyPixels.size()); + auto& ch = mNoisyPixels[chip]; + for (const auto k : rowcolKey) { + ch[k]++; + } + } + int dumpAboveThreshold(int t = 3) const { int n = 0; @@ -93,9 +102,10 @@ class NoiseMap return dumpAboveThreshold(p * mNumOfStrobes); } - void applyProbThreshold(float t, long int n, float relErr = 0.2f) + void applyProbThreshold(float t, long int n, float relErr = 0.2f, int minChipID = 0, int maxChipID = 24119) { // Remove from the maps all pixels with the firing probability below the threshold + // Apply the cut only for chips between minChipID and maxChipID (included) if (n < 1) { LOGP(alarm, "Cannot apply threshold with {} ROFs scanned", n); return; @@ -119,7 +129,12 @@ class NoiseMap LOGP(alarm, "Requested relative error {} with prob.threshold {} needs > {} ROFs, {} provided: pixels with noise >{} will be masked", relErr, t, req, n, mProbThreshold); } + int currChipID = 0; for (auto& map : mNoisyPixels) { + if (currChipID < minChipID || currChipID > maxChipID) { // chipID range + currChipID++; + continue; + } for (auto it = map.begin(); it != map.end();) { if (it->second < minFired) { it = map.erase(it); @@ -127,6 +142,7 @@ class NoiseMap ++it; } } + currChipID++; } } float getProbThreshold() const { return mProbThreshold; } @@ -153,9 +169,12 @@ class NoiseMap // Methods required by the calibration framework void print(); void fill(const gsl::span data); - void merge(const NoiseMap* prev) {} + const std::map* getChipMap(int chip) const { return chip < (int)mNoisyPixels.size() ? &mNoisyPixels[chip] : nullptr; } + std::map& getChip(int chip) { return mNoisyPixels[chip]; } + const std::map& getChip(int chip) const { return mNoisyPixels[chip]; } + void maskFullChip(int chip, bool cleanNoisyPixels = false) { if (cleanNoisyPixels) { @@ -183,15 +202,30 @@ class NoiseMap return std::ceil((1. + 1. / t) / (relErr * relErr)); } + NoiseMap merge(const NoiseMap* prev) + { + int incre = 0; + for (size_t i = 0; i < mNoisyPixels.size(); ++i) { + for (const auto& prev_np : prev->mNoisyPixels[i]) { // only enters this for loop if the "i" chip exists. + if (mNoisyPixels[i].find(prev_np.first) == mNoisyPixels[i].end()) { + mNoisyPixels[i][prev_np.first] = prev_np.second; + incre++; + } + } // end of for loop on elements of previous noise map + } // end of for loop on i (chip ID) + return (mNoisyPixels); + } // end of void merge + + size_t size() const { return mNoisyPixels.size(); } void setNumOfStrobes(long n) { mNumOfStrobes = n; } void addStrobes(long n) { mNumOfStrobes += n; } long getNumberOfStrobes() const { return mNumOfStrobes; } + static int getKey(int row, int col) { return (row << SHIFT) + col; } + static int key2Row(int key) { return key >> SHIFT; } + static int key2Col(int key) { return key & MASK; } private: static constexpr int SHIFT = 10, MASK = (0x1 << SHIFT) - 1; - int getKey(int row, int col) const { return (row << SHIFT) + col; } - int key2Row(int key) const { return key >> SHIFT; } - int key2Col(int key) const { return key & MASK; } std::vector> mNoisyPixels; ///< Internal noise map representation long int mNumOfStrobes = 0; ///< Accumulated number of ALPIDE strobes float mProbThreshold = 0; ///< Probability threshold for noisy pixels diff --git a/DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/ROFRecord.h b/DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/ROFRecord.h index 1f7ac73d0a131..9a6cd0891b6be 100644 --- a/DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/ROFRecord.h +++ b/DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/ROFRecord.h @@ -31,6 +31,8 @@ class ROFRecord { public: + enum { VtxStdMode = 0, + VtxUPCMode = 1 }; using EvIdx = o2::dataformats::RangeReference; using BCData = o2::InteractionRecord; using ROFtype = unsigned int; @@ -45,6 +47,14 @@ class ROFRecord void setFirstEntry(int idx) { mROFEntry.setFirstEntry(idx); } void setNEntries(int n) { mROFEntry.setEntries(n); } + uint32_t getFlags() const { return mBits; } + void setFlags(uint32_t flags) { mBits = flags; } + void setFlag(uint8_t flagIndex) { mBits |= (1 << flagIndex); } + void resetFlag(uint8_t flagIndex) { mBits &= ~(1 << flagIndex); } + bool getFlag(uint8_t flagIndex) const { return mBits & (1 << flagIndex); } + void clearAllFlags() { mBits = 0; } + void setAllFlags() { mBits = ~0; } + const BCData& getBCData() const { return mBCData; } BCData& getBCData() { return mBCData; } EvIdx getEntry() const { return mROFEntry; } @@ -58,7 +68,6 @@ class ROFRecord mROFEntry.clear(); mBCData.clear(); } - void print() const; template gsl::span getROFData(const gsl::span tfdata) const @@ -84,12 +93,16 @@ class ROFRecord return i < getNEntries() ? &tfdata[getFirstEntry() + i] : nullptr; } + std::string asString() const; + void print() const; + friend std::ostream& operator<<(std::ostream& output, const ROFRecord& rec); + private: o2::InteractionRecord mBCData; // BC data for given trigger EvIdx mROFEntry; //< reference on the 1st object of the ROF in data ROFtype mROFrame = 0; //< frame ID - - ClassDefNV(ROFRecord, 2); + uint32_t mBits = 0; + ClassDefNV(ROFRecord, 3); }; /// this is a simple reference connecting (composed) MC event ID (from the EventRecord of the RunContext) @@ -105,7 +118,10 @@ struct MC2ROFRecord { MC2ROFRecord() = default; MC2ROFRecord(int evID, int rofRecID, ROFtype mnrof, ROFtype mxrof) : eventRecordID(evID), rofRecordID(rofRecID), minROF(mnrof), maxROF(mxrof) {} int getNROFs() const { return (rofRecordID < 0 || minROF > maxROF) ? 0 : (maxROF - minROF); } + std::string asString() const; void print() const; + friend std::ostream& operator<<(std::ostream& output, const MC2ROFRecord& rec); + ClassDefNV(MC2ROFRecord, 1); }; } // namespace itsmft diff --git a/DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/TimeDeadMap.h b/DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/TimeDeadMap.h new file mode 100644 index 0000000000000..6c7c01dc888b7 --- /dev/null +++ b/DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/TimeDeadMap.h @@ -0,0 +1,87 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file TimeDeadMap.h +/// \brief Definition of the ITSMFT time-dependend dead map +#ifndef ALICEO2_ITSMFT_TIMEDEADMAP_H +#define ALICEO2_ITSMFT_TIMEDEADMAP_H + +#include + +#include +#include +#include +#include + +namespace o2 +{ + +namespace itsmft +{ + +class NoiseMap; + +class TimeDeadMap +{ + public: + // Constructor + TimeDeadMap(std::map>& deadmap) + { + mEvolvingDeadMap.swap(deadmap); + } + + /// Constructor + TimeDeadMap() = default; + /// Destructor + ~TimeDeadMap() = default; + + void fillMap(unsigned long firstOrbit, const std::vector& deadVect) + { + mEvolvingDeadMap[firstOrbit] = deadVect; + }; + + void fillMap(const std::vector& deadVect) + { + mStaticDeadMap = deadVect; + } + + void clear() + { + mEvolvingDeadMap.clear(); + mStaticDeadMap.clear(); + } + + void decodeMap(NoiseMap& noisemap) const; + void decodeMap(unsigned long orbit, o2::itsmft::NoiseMap& noisemap, bool includeStaticMap = true, long orbitGapAllowed = 330000) const; + std::string getMapVersion() const { return mMAP_VERSION; }; + + unsigned long getEvolvingMapSize() const { return mEvolvingDeadMap.size(); }; + std::vector getEvolvingMapKeys() const; + void getStaticMap(std::vector& mmap) const { mmap = mStaticDeadMap; }; + long getMapAtOrbit(unsigned long orbit, std::vector& mmap) const; + void setMapVersion(std::string version) { mMAP_VERSION = version; }; + + bool isDefault() const { return mIsDefaultObject; }; + void setAsDefault(bool isdef = true) { mIsDefaultObject = isdef; }; + + private: + bool mIsDefaultObject = false; + std::string mMAP_VERSION = "3"; + std::map> mEvolvingDeadMap; ///< Internal dead chip map representation. key = orbit + std::vector mStaticDeadMap; ///< To store map valid for every orbit. Filled starting from version = 4. + + ClassDefNV(TimeDeadMap, 2); +}; + +} // namespace itsmft +} // namespace o2 + +#endif /* ALICEO2_ITSMFT_TIMEDEADMAP_H */ diff --git a/DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/TopologyDictionary.h b/DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/TopologyDictionary.h index 0fe371ba58248..132601030933e 100644 --- a/DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/TopologyDictionary.h +++ b/DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/TopologyDictionary.h @@ -54,11 +54,11 @@ struct GroupStruct { float mErrZ; ///< Error associated to the hit point in the z direction. float mErr2X; ///< Squared Error associated to the hit point in the x direction. float mErr2Z; ///< Squared Error associated to the hit point in the z direction. - float mXCOG; ///< x position of te COG wrt the boottom left corner of the bounding box - float mZCOG; ///< z position of te COG wrt the boottom left corner of the bounding box + float mXCOG; ///< x position of the COG wrt the bottom left corner of the bounding box + float mZCOG; ///< z position of the COG wrt the bottom left corner of the bounding box int mNpixels; ///< Number of fired pixels ClusterPattern mPattern; ///< Bitmask of pixels. For groups the biggest bounding box for the group is taken, with all - ///the bits set to 1. + /// the bits set to 1. double mFrequency; ///< Frequency of the topology bool mIsGroup; ///< false: common topology; true: group of rare topologies ClassDefNV(GroupStruct, 3); @@ -72,6 +72,7 @@ class TopologyDictionary /// Constructor TopologyDictionary(const std::string& fileName); TopologyDictionary& operator=(const TopologyDictionary& dict) = default; + /// constexpr for the definition of the groups of rare topologies. /// The attritbution of the group ID is stringly dependent on the following parameters: it must be a power of 2. static constexpr int RowClassSpan = 4; ///< Row span of the classes of rare topologies @@ -159,7 +160,7 @@ class TopologyDictionary static void getTopologyDistribution(const TopologyDictionary& dict, TH1F*& histo, const char* histName); /// Returns the number of elements in the dicionary; int getSize() const { return (int)mVectorOfIDs.size(); } - ///Returns the local position of a compact cluster + /// Returns the local position of a compact cluster // array version of getClusterCoordinates template @@ -170,7 +171,7 @@ class TopologyDictionary template math_utils::Point3D getClusterCoordinates(const CompCluster& cl) const; - ///Returns the local position of a compact cluster + /// Returns the local position of a compact cluster template static math_utils::Point3D getClusterCoordinates(const CompCluster& cl, const ClusterPattern& patt, bool isGroup = true); diff --git a/DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/TrkClusRef.h b/DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/TrkClusRef.h index 164fd850fd539..27b059cc61ecf 100644 --- a/DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/TrkClusRef.h +++ b/DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/TrkClusRef.h @@ -25,12 +25,33 @@ namespace itsmft // can refer to max 15 indices in the vector of total length <268435456, i.e. 17895697 tracks in worst case struct TrkClusRef : public o2::dataformats::RangeRefComp<4> { using o2::dataformats::RangeRefComp<4>::RangeRefComp; + uint32_t clsizes = 0; ///< cluster sizes for each layer uint16_t pattern = 0; ///< layers pattern GPUd() int getNClusters() const { return getEntries(); } - bool hasHitOnLayer(int i) { return pattern & (0x1 << i); } - - ClassDefNV(TrkClusRef, 1); + bool hasHitOnLayer(int i) const { return pattern & (0x1 << i); } + + void setClusterSize(int l, int size) + { + if (l >= 8) + return; + if (size > 15) + size = 15; + clsizes &= ~(0xf << (l * 4)); + clsizes |= (size << (l * 4)); + } + + int getClusterSize(int l) const + { + return (l >= 8) ? 0 : (clsizes >> (l * 4)) & 0xf; + } + + int getClusterSizes() const + { + return clsizes; + } + + ClassDefNV(TrkClusRef, 2); }; } // namespace itsmft diff --git a/DataFormats/Detectors/ITSMFT/common/src/Cluster.cxx b/DataFormats/Detectors/ITSMFT/common/src/Cluster.cxx index 968081aca8528..0737475b25caa 100644 --- a/DataFormats/Detectors/ITSMFT/common/src/Cluster.cxx +++ b/DataFormats/Detectors/ITSMFT/common/src/Cluster.cxx @@ -13,7 +13,7 @@ /// \brief Implementation of the ITSMFT cluster #include "DataFormatsITSMFT/Cluster.h" -#include "FairLogger.h" +#include #include #include diff --git a/DataFormats/Detectors/ITSMFT/common/src/ClusterPattern.cxx b/DataFormats/Detectors/ITSMFT/common/src/ClusterPattern.cxx index 7c67a7a3ce4dc..4dda3da3a011d 100644 --- a/DataFormats/Detectors/ITSMFT/common/src/ClusterPattern.cxx +++ b/DataFormats/Detectors/ITSMFT/common/src/ClusterPattern.cxx @@ -102,6 +102,11 @@ std::ostream& operator<<(std::ostream& os, const ClusterPattern& pattern) return os; } +void ClusterPattern::print() const +{ + std::cout << *this << "\n"; +} + int ClusterPattern::getNPixels() const { int n = 0, nBytes = getUsedBytes(); diff --git a/DataFormats/Detectors/ITSMFT/common/src/ClusterTopology.cxx b/DataFormats/Detectors/ITSMFT/common/src/ClusterTopology.cxx index 87b56eeb72986..ae3a0deb293fc 100644 --- a/DataFormats/Detectors/ITSMFT/common/src/ClusterTopology.cxx +++ b/DataFormats/Detectors/ITSMFT/common/src/ClusterTopology.cxx @@ -73,17 +73,15 @@ unsigned int ClusterTopology::hashFunction(const void* key, int len) len -= 4; } // Handle the last few bytes of the input array + // ATTENTION: DO NOT INSERT BREAK after case, its absence is intended!!! switch (len) { case 3: h ^= data[2] << 16; - break; case 2: h ^= data[1] << 8; - break; case 1: h ^= data[0]; h *= m; - break; }; // Do a few final mixes of the hash to ensure the last few // bytes are well-incorporated. diff --git a/DataFormats/Detectors/ITSMFT/common/src/CompCluster.cxx b/DataFormats/Detectors/ITSMFT/common/src/CompCluster.cxx index 95ecd73f6e9d5..1f4be3163b3d2 100644 --- a/DataFormats/Detectors/ITSMFT/common/src/CompCluster.cxx +++ b/DataFormats/Detectors/ITSMFT/common/src/CompCluster.cxx @@ -15,22 +15,32 @@ #include "DataFormatsITSMFT/CompCluster.h" #include #include +#include using namespace o2::itsmft; std::ostream& operator<<(std::ostream& stream, const CompCluster& cl) { - stream << " row: " << cl.getRow() << " col: " << cl.getCol() - << " pattID " << cl.getPatternID() << " [flag: " << cl.getFlag() << "] "; + stream << cl.asString(); return stream; } std::ostream& operator<<(std::ostream& stream, const CompClusterExt& cl) { - stream << " chip: " << cl.getChipID() << ((const CompCluster&)cl); + stream << cl.asString(); return stream; } +std::string CompCluster::asString() const +{ + return std::format(" row: {:4d} col: {:4d} pattID: {:4d} [flag: {:1d}]", getRow(), getCol(), getPatternID(), getFlag()); +} + +std::string CompClusterExt::asString() const +{ + return std::format(" chip: {:5d} row: {:4d} col: {:4d} pattID: {:4d} [flag: {:1d}]", getChipID(), getRow(), getCol(), getPatternID(), getFlag()); +} + //______________________________________________________________________________ void CompCluster::print() const { diff --git a/DataFormats/Detectors/ITSMFT/common/src/ITSMFTDataFormatsLinkDef.h b/DataFormats/Detectors/ITSMFT/common/src/ITSMFTDataFormatsLinkDef.h index 5edd964ce2cd0..fc67fdf028436 100644 --- a/DataFormats/Detectors/ITSMFT/common/src/ITSMFTDataFormatsLinkDef.h +++ b/DataFormats/Detectors/ITSMFT/common/src/ITSMFTDataFormatsLinkDef.h @@ -17,6 +17,7 @@ #pragma link C++ class o2::itsmft::Digit + ; #pragma link C++ class o2::itsmft::NoiseMap + ; +#pragma link C++ class o2::itsmft::TimeDeadMap + ; #pragma link C++ class std::vector < o2::itsmft::Digit> + ; #pragma link C++ class o2::itsmft::GBTCalibData + ; @@ -49,6 +50,8 @@ #pragma link C++ class o2::itsmft::CTF + ; #pragma link C++ class o2::ctf::EncodedBlocks < o2::itsmft::CTFHeader, 10, uint32_t> + ; +#pragma link C++ class std::map < unsigned long, std::vector < uint16_t>> + ; + #pragma link C++ function o2::itsmft::getROFData(const gsl::span tfdata) const; #pragma link C++ function o2::itsmft::getROFData(const gsl::span tfdata) const; #pragma link C++ function o2::itsmft::getROFData(const gsl::span tfdata) const; diff --git a/DataFormats/Detectors/ITSMFT/common/src/ROFRecord.cxx b/DataFormats/Detectors/ITSMFT/common/src/ROFRecord.cxx index fcc86ac7e7dca..8dbde0d580efc 100644 --- a/DataFormats/Detectors/ITSMFT/common/src/ROFRecord.cxx +++ b/DataFormats/Detectors/ITSMFT/common/src/ROFRecord.cxx @@ -9,20 +9,42 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. -#include "DataFormatsITSMFT/ROFRecord.h" #include +#include + +#include "DataFormatsITSMFT/ROFRecord.h" +#include "Framework/Logger.h" using namespace o2::itsmft; +std::string ROFRecord::asString() const +{ + return std::format("ROF: {} | {} entries starting from {} | IR: {}", mROFrame, getNEntries(), getFirstEntry(), mBCData.asString()); +} + void ROFRecord::print() const { - std::cout << "ROF: " << mROFrame << " | " << getNEntries() << " entries starting from " - << getFirstEntry() << std::endl; - mBCData.print(); + LOG(info) << asString(); +} + +std::ostream& operator<<(std::ostream& stream, ROFRecord const& rec) +{ + stream << rec.asString(); + return stream; +} + +std::string MC2ROFRecord::asString() const +{ + return std::format("MCEventID: {} ROFs: {}-{} Entry in ROFRecords: {}", eventRecordID, minROF, maxROF, rofRecordID); } void MC2ROFRecord::print() const { - std::cout << "MCEventID: " << eventRecordID << " ROFs: " << minROF << '-' << maxROF - << " Entry in ROFRecords: " << rofRecordID << std::endl; + LOG(info) << asString(); +} + +std::ostream& operator<<(std::ostream& stream, MC2ROFRecord const& rec) +{ + stream << rec.asString(); + return stream; } diff --git a/DataFormats/Detectors/ITSMFT/common/src/TimeDeadMap.cxx b/DataFormats/Detectors/ITSMFT/common/src/TimeDeadMap.cxx new file mode 100644 index 0000000000000..e3df8e7f91f86 --- /dev/null +++ b/DataFormats/Detectors/ITSMFT/common/src/TimeDeadMap.cxx @@ -0,0 +1,100 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file TimeDeadMap.cxx +/// \brief Implementation of the time-dependent map + +#include "DataFormatsITSMFT/TimeDeadMap.h" +#include "DataFormatsITSMFT/NoiseMap.h" +#include "Framework/Logger.h" + +using namespace o2::itsmft; + +void TimeDeadMap::decodeMap(o2::itsmft::NoiseMap& noisemap) const +{ // for static part only + if (mMAP_VERSION == "3") { + LOG(error) << "Trying to decode static part of deadmap version " << mMAP_VERSION << ". Not implemented, doing nothing."; + return; + } + for (int iel = 0; iel < mStaticDeadMap.size(); iel++) { + uint16_t w = mStaticDeadMap[iel]; + noisemap.maskFullChip(w & 0x7FFF); + if (w & 0x8000) { + for (int w2 = (w & 0x7FFF) + 1; w2 < mStaticDeadMap.at(iel + 1); w2++) { + noisemap.maskFullChip(w2); + } + } + } +} + +void TimeDeadMap::decodeMap(unsigned long orbit, o2::itsmft::NoiseMap& noisemap, bool includeStaticMap, long orbitGapAllowed) const +{ // for time-dependent and (optionally) static part. Use orbitGapAllowed = -1 to ignore check on orbit difference + + if (mMAP_VERSION != "3" && mMAP_VERSION != "4") { + LOG(error) << "Trying to decode time-dependent deadmap version " << mMAP_VERSION << ". Not implemented, doing nothing."; + return; + } + + if (mEvolvingDeadMap.empty()) { + LOG(warning) << "Time-dependent dead map is empty. Doing nothing."; + return; + } + + std::vector closestVec; + long dT = getMapAtOrbit(orbit, closestVec); + + if (orbitGapAllowed >= 0 && std::abs(dT) > orbitGapAllowed) { + LOG(warning) << "Requested orbit " << orbit << ", found " << orbit - dT << ". Orbit gap is too high, skipping time-dependent map."; + closestVec.clear(); + } + + // add static part if requested. something may be masked twice + if (includeStaticMap && mMAP_VERSION != "3") { + closestVec.insert(closestVec.end(), mStaticDeadMap.begin(), mStaticDeadMap.end()); + } + + // vector encoding: if 1<<15 = 0x8000 is set, the word encodes the first element of a range, with mask (1<<15)-1 = 0x7FFF. The last element of the range is the next in the vector. + + for (int iel = 0; iel < closestVec.size(); iel++) { + uint16_t w = closestVec.at(iel); + noisemap.maskFullChip(w & 0x7FFF); + if (w & 0x8000) { + for (int w2 = (w & 0x7FFF) + 1; w2 < closestVec.at(iel + 1); w2++) { + noisemap.maskFullChip(w2); + } + } + } +} + +std::vector TimeDeadMap::getEvolvingMapKeys() const +{ + std::vector keys; + std::transform(mEvolvingDeadMap.begin(), mEvolvingDeadMap.end(), std::back_inserter(keys), + [](const auto& O) { return O.first; }); + return keys; +} + +long TimeDeadMap::getMapAtOrbit(unsigned long orbit, std::vector& mmap) const +{ // fills mmap and returns requested_orbit - found_orbit. Found orbit is the highest key lower or equal to the requested one + if (mEvolvingDeadMap.empty()) { + LOG(warning) << "Requested orbit " << orbit << "from an empty time-dependent map. Doing nothing"; + return (long)orbit; + } + auto closest = mEvolvingDeadMap.upper_bound(orbit); + if (closest != mEvolvingDeadMap.begin()) { + --closest; + mmap = closest->second; + return (long)orbit - closest->first; + } else { + mmap = mEvolvingDeadMap.begin()->second; + return (long)(orbit)-mEvolvingDeadMap.begin()->first; + } +} diff --git a/DataFormats/Detectors/MUON/MCH/CMakeLists.txt b/DataFormats/Detectors/MUON/MCH/CMakeLists.txt index 77b0498e8f8ed..26bf088495ff5 100644 --- a/DataFormats/Detectors/MUON/MCH/CMakeLists.txt +++ b/DataFormats/Detectors/MUON/MCH/CMakeLists.txt @@ -38,3 +38,9 @@ o2_add_test(digit COMPONENT_NAME mch LABELS muon;mch) +o2_add_test(track + SOURCES src/testTrackMCH.cxx + PUBLIC_LINK_LIBRARIES O2::MCHBase + COMPONENT_NAME mch + LABELS muon;mch) + diff --git a/DataFormats/Detectors/MUON/MCH/include/DataFormatsMCH/DsChannelId.h b/DataFormats/Detectors/MUON/MCH/include/DataFormatsMCH/DsChannelId.h index e0e233a81695c..cd11d4b71389c 100644 --- a/DataFormats/Detectors/MUON/MCH/include/DataFormatsMCH/DsChannelId.h +++ b/DataFormats/Detectors/MUON/MCH/include/DataFormatsMCH/DsChannelId.h @@ -54,10 +54,20 @@ class DsChannelId uint32_t value() const { return mChannelId; } + bool isValid() const { return (mChannelId != 0); } + private: uint32_t mChannelId{0}; ClassDefNV(DsChannelId, 1); // class for MCH readout channel }; + +inline bool operator==(const DsChannelId& a, const DsChannelId& b) { return a.value() == b.value(); } +inline bool operator!=(const DsChannelId& a, const DsChannelId& b) { return !(a == b); } +inline bool operator<(const DsChannelId& a, const DsChannelId& b) { return a.value() < b.value(); } +inline bool operator>(const DsChannelId& a, const DsChannelId& b) { return b < a; } +inline bool operator<=(const DsChannelId& a, const DsChannelId& b) { return !(a > b); } +inline bool operator>=(const DsChannelId& a, const DsChannelId& b) { return !(a < b); } + } // namespace o2::mch #endif diff --git a/DataFormats/Detectors/MUON/MCH/include/DataFormatsMCH/TrackMCH.h b/DataFormats/Detectors/MUON/MCH/include/DataFormatsMCH/TrackMCH.h index f5e3de63a7916..8c1427e9a9682 100644 --- a/DataFormats/Detectors/MUON/MCH/include/DataFormatsMCH/TrackMCH.h +++ b/DataFormats/Detectors/MUON/MCH/include/DataFormatsMCH/TrackMCH.h @@ -21,6 +21,8 @@ #include #include "CommonDataFormat/RangeReference.h" +#include "CommonDataFormat/InteractionRecord.h" +#include "CommonDataFormat/TimeStamp.h" namespace o2 { @@ -33,9 +35,11 @@ class TrackMCH using ClusRef = o2::dataformats::RangeRefComp<5>; public: + using Time = o2::dataformats::TimeStampWithError; + TrackMCH() = default; TrackMCH(double z, const TMatrixD& param, const TMatrixD& cov, double chi2, int firstClIdx, int nClusters, - double zAtMID, const TMatrixD& paramAtMID, const TMatrixD& covAtMID); + double zAtMID, const TMatrixD& paramAtMID, const TMatrixD& covAtMID, const Time& time); ~TrackMCH() = default; TrackMCH(const TrackMCH& track) = default; @@ -107,6 +111,19 @@ class TrackMCH /// set the number of the clusters attached to the track and the index of the first one void setClusterRef(int firstClusterIdx, int nClusters) { mClusRef.set(firstClusterIdx, nClusters); } + /// mean time associated to this track, with error + const Time& getTimeMUS() const { return mTimeMUS; } + Time& getTimeMUS() { return mTimeMUS; } + void setTimeMUS(const Time& t) { mTimeMUS = t; } + void setTimeMUS(float t, float te) + { + mTimeMUS.setTimeStamp(t); + mTimeMUS.setTimeStampError(te); + } + + /// interaction record corresponding to the mean track time + InteractionRecord getMeanIR(uint32_t refOrbit) const; + private: static constexpr int SNParams = 5; ///< number of track parameters static constexpr int SCovSize = 15; ///< number of different elements in the symmetric covariance matrix @@ -136,8 +153,9 @@ class TrackMCH double mZAtMID = 0.; ///< z position on the MID side where the parameters are evaluated double mParamAtMID[SNParams] = {0.}; ///< 5 parameters: X (cm), SlopeX, Y (cm), SlopeY, q/pYZ ((GeV/c)^-1) double mCovAtMID[SCovSize] = {0.}; ///< reduced covariance matrix of track parameters, formated as above + Time mTimeMUS{}; ///< associated time in microseconds from the TF start - ClassDefNV(TrackMCH, 2); + ClassDefNV(TrackMCH, 3); }; std::ostream& operator<<(std::ostream& os, const TrackMCH& t); diff --git a/DataFormats/Detectors/MUON/MCH/src/DsChannelGroup.cxx b/DataFormats/Detectors/MUON/MCH/src/DsChannelGroup.cxx deleted file mode 100644 index bcf10d74c95ff..0000000000000 --- a/DataFormats/Detectors/MUON/MCH/src/DsChannelGroup.cxx +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -#include "DataFormatsMCH/DsChannelGroup.h" - -std::string o2::mch::DsChannelId() const -{ -} diff --git a/DataFormats/Detectors/MUON/MCH/src/ROFRecord.cxx b/DataFormats/Detectors/MUON/MCH/src/ROFRecord.cxx index e39354399814c..1926e5e31889e 100644 --- a/DataFormats/Detectors/MUON/MCH/src/ROFRecord.cxx +++ b/DataFormats/Detectors/MUON/MCH/src/ROFRecord.cxx @@ -40,7 +40,7 @@ std::pair ROFRecord::getTimeMUS(const BCData& startIR, ui bool isInTF = bcDiff >= 0 && bcDiff < nOrbits * o2::constants::lhc::LHCMaxBunches; if (printError && !isInTF) { LOGP(alarm, "ATTENTION: wrong bunches diff. {} for current IR {} wrt 1st TF orbit {}, source:MCH", - bcDiff, mBCData, startIR); + bcDiff, mBCData.asString(), startIR.asString()); } return std::make_pair(Time(tMean, tErr), isInTF); } diff --git a/DataFormats/Detectors/MUON/MCH/src/TrackMCH.cxx b/DataFormats/Detectors/MUON/MCH/src/TrackMCH.cxx index 7dcca9bc86e5e..61a4d84c31eb0 100644 --- a/DataFormats/Detectors/MUON/MCH/src/TrackMCH.cxx +++ b/DataFormats/Detectors/MUON/MCH/src/TrackMCH.cxx @@ -29,8 +29,8 @@ namespace mch //__________________________________________________________________________ TrackMCH::TrackMCH(double z, const TMatrixD& param, const TMatrixD& cov, double chi2, int firstClIdx, int nClusters, - double zAtMID, const TMatrixD& paramAtMID, const TMatrixD& covAtMID) - : mZ(z), mChi2(chi2), mClusRef(firstClIdx, nClusters), mZAtMID(zAtMID) + double zAtMID, const TMatrixD& paramAtMID, const TMatrixD& covAtMID, const Time& time) + : mZ(z), mChi2(chi2), mClusRef(firstClIdx, nClusters), mZAtMID(zAtMID), mTimeMUS(time) { /// constructor setParameters(param); @@ -87,6 +87,14 @@ void TrackMCH::setCovariances(const TMatrixD& src, double (&dest)[SCovSize]) } } +//__________________________________________________________________________ +InteractionRecord TrackMCH::getMeanIR(uint32_t refOrbit) const +{ + InteractionRecord startIR(0, refOrbit); + auto trackBCinTF = std::llround(mTimeMUS.getTimeStamp() / constants::lhc::LHCBunchSpacingMUS); + return startIR + trackBCinTF; +} + std::ostream& operator<<(std::ostream& os, const o2::mch::TrackMCH& t) { os << asString(t); diff --git a/DataFormats/Detectors/MUON/MCH/src/testTrackMCH.cxx b/DataFormats/Detectors/MUON/MCH/src/testTrackMCH.cxx new file mode 100644 index 0000000000000..e6cc52945ef5e --- /dev/null +++ b/DataFormats/Detectors/MUON/MCH/src/testTrackMCH.cxx @@ -0,0 +1,57 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include +#define BOOST_TEST_MODULE MCH TrackMCH +#define BOOST_TEST_DYN_LINK +#include +#include +#include + +#include "DataFormatsMCH/TrackMCH.h" + +BOOST_AUTO_TEST_CASE(TrackIRMatchesTrackTime) +{ + uint16_t bc{10}; + uint32_t orbit{2000}; + uint32_t orbitRef = orbit - 120; + o2::InteractionRecord ir{bc, orbit}; + o2::InteractionRecord irRef{0, orbitRef}; + + o2::mch::TrackMCH track; + + auto bcDiff = ir.differenceInBC(irRef); + float tMean = o2::constants::lhc::LHCBunchSpacingMUS * bcDiff; + float tErr = 6.0 * o2::constants::lhc::LHCBunchSpacingMUS; + track.setTimeMUS(tMean, tErr); + + o2::InteractionRecord trackIR = track.getMeanIR(orbitRef); + BOOST_CHECK_EQUAL(ir, trackIR); +} + +BOOST_AUTO_TEST_CASE(TrackIRMatchesNegativeTrackTime) +{ + uint16_t bc{10}; + uint32_t orbit{2000}; + uint32_t orbitRef = orbit + 1; + o2::InteractionRecord ir{bc, orbit}; + o2::InteractionRecord irRef{0, orbitRef}; + + o2::mch::TrackMCH track; + + auto bcDiff = ir.differenceInBC(irRef); + float tMean = o2::constants::lhc::LHCBunchSpacingMUS * bcDiff; + float tErr = 6.0 * o2::constants::lhc::LHCBunchSpacingMUS; + track.setTimeMUS(tMean, tErr); + + o2::InteractionRecord trackIR = track.getMeanIR(orbitRef); + BOOST_CHECK_EQUAL(ir, trackIR); +} diff --git a/DataFormats/Detectors/MUON/MID/CMakeLists.txt b/DataFormats/Detectors/MUON/MID/CMakeLists.txt index 9f278c8cb0676..fe26f93c29848 100644 --- a/DataFormats/Detectors/MUON/MID/CMakeLists.txt +++ b/DataFormats/Detectors/MUON/MID/CMakeLists.txt @@ -25,7 +25,8 @@ o2_add_library( o2_target_root_dictionary( DataFormatsMID - HEADERS include/DataFormatsMID/Cluster.h + HEADERS include/DataFormatsMID/ChEffCounter.h + include/DataFormatsMID/Cluster.h include/DataFormatsMID/ColumnData.h include/DataFormatsMID/ROBoard.h include/DataFormatsMID/ROFRecord.h diff --git a/DataFormats/Detectors/MUON/MID/include/DataFormatsMID/ChEffCounter.h b/DataFormats/Detectors/MUON/MID/include/DataFormatsMID/ChEffCounter.h new file mode 100644 index 0000000000000..59a4a7e7bcf33 --- /dev/null +++ b/DataFormats/Detectors/MUON/MID/include/DataFormatsMID/ChEffCounter.h @@ -0,0 +1,51 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file DataFormatsMID/ChEffCounter.h +/// \brief Chamber efficiency counters +/// \author Diego Stocco +/// \date 12 September 2022 + +#ifndef O2_MID_CHEFFCOUNTER_H +#define O2_MID_CHEFFCOUNTER_H + +#include +#include +#include + +namespace o2 +{ +namespace mid +{ +enum class EffCountType { + BendPlane, ///< Bending plane counters + NonBendPlane, ///< Non-bending plane counters + BothPlanes, ///< Both plane counters + AllTracks ///< All tracks counters +}; + +/// Column data structure for MID +struct ChEffCounter { + uint8_t deId = 0; ///< Index of the detection element + uint8_t columnId = 0; ///< Column in DE + uint8_t lineId = 0; ///< Line in column + std::array counts; ///< Counts + + /// @brief Returns the efficiency counter + /// @param type Efficiency counter type + /// @return Efficiency counter + uint32_t getCounts(EffCountType type) const { return counts[static_cast(type)]; } +}; + +} // namespace mid +} // namespace o2 + +#endif /* O2_MID_CHEFFCOUNTER_H */ diff --git a/DataFormats/Detectors/MUON/MID/include/DataFormatsMID/ROFRecord.h b/DataFormats/Detectors/MUON/MID/include/DataFormatsMID/ROFRecord.h index 889e162aa3fed..e770e15b46bb8 100644 --- a/DataFormats/Detectors/MUON/MID/include/DataFormatsMID/ROFRecord.h +++ b/DataFormats/Detectors/MUON/MID/include/DataFormatsMID/ROFRecord.h @@ -64,7 +64,7 @@ struct ROFRecord { bool isInTF = bcDiff >= 0 && bcDiff < nOrbits * o2::constants::lhc::LHCMaxBunches; if (printError && !isInTF) { LOGP(alarm, "ATTENTION: wrong bunches diff. {} for current IR {} wrt 1st TF orbit {}, source:MID", - bcDiff, interactionRecord, startIR); + bcDiff, interactionRecord.asString(), startIR.asString()); } return std::make_pair(Time(tMean, tErr), isInTF); } diff --git a/DataFormats/Detectors/MUON/MID/include/DataFormatsMID/Track.h b/DataFormats/Detectors/MUON/MID/include/DataFormatsMID/Track.h index 9a24b4ed59e83..060538fc91654 100644 --- a/DataFormats/Detectors/MUON/MID/include/DataFormatsMID/Track.h +++ b/DataFormats/Detectors/MUON/MID/include/DataFormatsMID/Track.h @@ -143,19 +143,26 @@ class Track /// Gets the word allowing to compute the chamber efficiency uint32_t getEfficiencyWord() const { return mEfficiencyWord; } - /// Sets the fired local board for efficiency calculation - /// \param locId local board ID in the range 1-234 - void setFiredLocalBoard(int locId) { setEfficiencyWord(8, 0xFF, locId); } + /// Sets the fired FEE ID + void setFiredFEEId(int uniqueFeeId) { setEfficiencyWord(8, 0x7FFF, uniqueFeeId); } + + /// Gets the fired FEE ID + int getFiredFEEId() const { return (mEfficiencyWord >> 8) & 0x7FFF; } /// Gets the fired local board for efficiency calculation - int getFiredLocalBoard() const { return (mEfficiencyWord >> 8) & 0xFF; } + [[deprecated]] int getFiredLocalBoard() const { return (mEfficiencyWord >> 8) & 0xFF; } + + /// Gets the fired Detection Element ID + int getFiredDEId() const { return (mEfficiencyWord >> 16) & 0x7F; } - /// Sets the fired detection element ID for efficiency calculation - /// \param deId detection element ID in the range 1-234 - void setFiredDeId(int deId) { setEfficiencyWord(16, 0xFF, deId); } + /// Gets the fired Detection Element ID + [[deprecated("Use getFiredDEId instead")]] int getFiredDeId() const { return getFiredDEId(); } - /// Gets the fired detection element ID for efficiency calculation - int getFiredDeId() const { return (mEfficiencyWord >> 16) & 0xFF; } + /// Gets the fired column ID + int getFiredColumnId() const { return (mEfficiencyWord >> 12) & 0x7; } + + /// Gets the fired line ID + int getFiredLineId() const { return (mEfficiencyWord >> 8) & 0x3; } /// Sets the flag for efficiency calculation /// \param effFlag efficiency flag @@ -169,9 +176,21 @@ class Track /// \li \c 3 track can be used to estimate local board efficiency int getEfficiencyFlag() const { return (mEfficiencyWord >> 24) & 0xF; } + /// record that the track misses information on that cathode + void setIncomplete(int cathode) { mIncomplete |= 1 << cathode; } + + /// reset the flag for missing cathode information + void resetIncomplete() { mIncomplete = 0; } + + /// return true is the track misses information on that cathode + bool isIncomplete(int cathode) const { return mIncomplete & (1 << cathode); } + /// Overload ostream operator for MID track friend std::ostream& operator<<(std::ostream& stream, const Track& track); + /// set efficiency word (public function) + void setEfficiencyWord(uint32_t efficiencyWord) { mEfficiencyWord = efficiencyWord; } + private: /// Set portions of the efficiency word /// \param pos Position in the word @@ -185,8 +204,9 @@ class Track float mChi2 = 0.; ///< Chi2 of track int mNDF = 0; ///< Number of chi2 degrees of freedom uint32_t mEfficiencyWord = 0; ///< Efficiency word + uint8_t mIncomplete = 0; //!< Flag for missing cathode information - ClassDefNV(Track, 1); + ClassDefNV(Track, 2); }; } // namespace mid } // namespace o2 diff --git a/DataFormats/Detectors/MUON/MID/src/DataFormatsMIDLinkDef.h b/DataFormats/Detectors/MUON/MID/src/DataFormatsMIDLinkDef.h index d067a3f90eb23..bc7112452bfd1 100644 --- a/DataFormats/Detectors/MUON/MID/src/DataFormatsMIDLinkDef.h +++ b/DataFormats/Detectors/MUON/MID/src/DataFormatsMIDLinkDef.h @@ -15,6 +15,8 @@ #pragma link off all classes; #pragma link off all functions; +#pragma link C++ struct o2::mid::ChEffCounter + ; +#pragma link C++ class std::vector < o2::mid::ChEffCounter> + ; #pragma link C++ struct o2::mid::Cluster + ; #pragma link C++ class std::vector < o2::mid::Cluster> + ; #pragma link C++ struct o2::mid::ColumnData + ; diff --git a/DataFormats/Detectors/MUON/MID/src/Track.cxx b/DataFormats/Detectors/MUON/MID/src/Track.cxx index df0f62e6ca19d..72c2418e44e95 100644 --- a/DataFormats/Detectors/MUON/MID/src/Track.cxx +++ b/DataFormats/Detectors/MUON/MID/src/Track.cxx @@ -177,7 +177,7 @@ std::ostream& operator<<(std::ostream& stream, const Track& track) stream << ((ival == 5) ? ")" : ", "); } stream << fmt::format(" chi2/ndf: {:g}/{:d}", track.getChi2(), track.getNDF()); - stream << fmt::format(" hitMap: 0x{:x} locId: {:d} deId: {:d} effFlag {:d}", track.getHitMap(), track.getFiredLocalBoard(), track.getFiredDeId(), track.getEfficiencyFlag()); + stream << fmt::format(" hitMap: 0x{:x} deId: {:d} columnId: {:d} lineId: {:d} effFlag {:d}", track.getHitMap(), track.getFiredDEId(), track.getFiredColumnId(), track.getFiredLineId(), track.getEfficiencyFlag()); return stream; } diff --git a/DataFormats/Detectors/PHOS/include/DataFormatsPHOS/EventData.h b/DataFormats/Detectors/PHOS/include/DataFormatsPHOS/EventData.h index 4f5d243028177..e503af30d8e7f 100644 --- a/DataFormats/Detectors/PHOS/include/DataFormatsPHOS/EventData.h +++ b/DataFormats/Detectors/PHOS/include/DataFormatsPHOS/EventData.h @@ -13,10 +13,11 @@ #define ALICEO2_PHOS_EVENTDATA_H_ #include +#include #include "CommonDataFormat/InteractionRecord.h" #include "DataFormatsPHOS/Cell.h" #include "DataFormatsPHOS/Cluster.h" - +#include "DataFormatsPHOS/MCLabel.h" namespace o2 { @@ -25,10 +26,11 @@ namespace phos template struct EventData { - InteractionRecord mInteractionRecord; ///< Interaction record for the trigger corresponding to this event - gsl::span mClusters; ///< PHOS clusters - gsl::span mCells; ///< PHOS cells / digits - gsl::span mCellIndices; ///< Cell indices in cluster + InteractionRecord mInteractionRecord; ///< Interaction record for the trigger corresponding to this event + gsl::span mClusters; ///< PHOS clusters + gsl::span mCells; ///< PHOS cells / digits + gsl::span mCellIndices; ///< Cell indices in cluster + std::vector> mMCCellLabels; ///< span of MC labels for each cell /// \brief Reset event structure with empty interaction record and ranges void reset() @@ -37,13 +39,14 @@ struct EventData { mClusters = gsl::span(); mCells = gsl::span(); mCellIndices = gsl::span(); + mMCCellLabels = std::vector>(); } - ClassDefNV(EventData, 1); + ClassDefNV(EventData, 2); }; } // namespace phos } // namespace o2 -#endif // ALICEO2_PHOS_EVENTDATA_H_ \ No newline at end of file +#endif // ALICEO2_PHOS_EVENTDATA_H_ diff --git a/DataFormats/Detectors/PHOS/include/DataFormatsPHOS/EventHandler.h b/DataFormats/Detectors/PHOS/include/DataFormatsPHOS/EventHandler.h index 149bcc5505bf0..0d3285248d7b2 100644 --- a/DataFormats/Detectors/PHOS/include/DataFormatsPHOS/EventHandler.h +++ b/DataFormats/Detectors/PHOS/include/DataFormatsPHOS/EventHandler.h @@ -15,13 +15,16 @@ #include #include #include +#include #include "Rtypes.h" #include "fmt/format.h" #include "DataFormatsPHOS/Cell.h" #include "DataFormatsPHOS/Cluster.h" #include "DataFormatsPHOS/Digit.h" #include "DataFormatsPHOS/EventData.h" +#include "DataFormatsPHOS/MCLabel.h" #include "DataFormatsPHOS/TriggerRecord.h" +#include "SimulationDataFormat/MCTruthContainer.h" namespace o2 { @@ -259,6 +262,13 @@ class EventHandler /// \throw NotInitializedException in case the event handler is not initialized for cell const CellRange getCellsForEvent(int eventID) const; + /// \brief Get vector of MC labels belonging to the given event + /// \param eventID ID of the event + /// \return vector of MC labels for the event + /// \throw RangeException in case the required event ID exceeds the maximum number of events + /// \throw NotInitializedException in case the event handler is not initialized for cell + std::vector> getCellMCLabelForEvent(int eventID) const; + /// \brief Get range of cluster cell indices belonging to the given event /// \param eventID ID of the event /// \return Cluster cell index range for the event @@ -300,6 +310,13 @@ class EventHandler mTriggerRecordsCells = triggers; } + /// \brief Setting the pointer for the MCTruthContainer for cells + /// \param mclabels Pointer to the MCTruthContainer for cells in timeframe + void setCellMCTruthContainer(const o2::dataformats::MCTruthContainer* mclabels) + { + mCellLabels = mclabels; + } + /// \brief Reset containers with empty ranges void reset(); @@ -329,14 +346,15 @@ class EventHandler TriggerRange mTriggerRecordsCellIndices; ///< trigger record for cluster cell index type TriggerRange mTriggerRecordsCells; ///< Trigger record for cell type - ClusterRange mClusters; /// container for clusters in timeframe - CellIndexRange mClusterCellIndices; /// container for cell indices in timeframe - CellRange mCells; /// container for cells in timeframe + ClusterRange mClusters; /// container for clusters in timeframe + CellIndexRange mClusterCellIndices; /// container for cell indices in timeframe + CellRange mCells; /// container for cells in timeframe + const o2::dataformats::MCTruthContainer* mCellLabels = nullptr; /// pointer to the MCTruthContainer for cells in timeframe - ClassDefNV(EventHandler, 1); + ClassDefNV(EventHandler, 2); }; } // namespace phos } // namespace o2 -#endif // ALICEO2_PHOS_EVENTHANDLER_H__ \ No newline at end of file +#endif // ALICEO2_PHOS_EVENTHANDLER_H__ diff --git a/DataFormats/Detectors/PHOS/include/DataFormatsPHOS/MCLabel.h b/DataFormats/Detectors/PHOS/include/DataFormatsPHOS/MCLabel.h index a9aaa30243c0a..aa171e9825870 100644 --- a/DataFormats/Detectors/PHOS/include/DataFormatsPHOS/MCLabel.h +++ b/DataFormats/Detectors/PHOS/include/DataFormatsPHOS/MCLabel.h @@ -23,7 +23,7 @@ namespace phos class MCLabel : public o2::MCCompLabel { private: - float mEdep = 0; //deposited energy + float mEdep = 0; // deposited energy public: MCLabel() = default; @@ -44,7 +44,9 @@ class MCLabel : public o2::MCCompLabel float getEdep() const { return mEdep; } - ClassDefNV(MCLabel, 1); + double getAmplitudeFraction() const { return mEdep; } + + ClassDefNV(MCLabel, 2); }; } // namespace phos } // namespace o2 diff --git a/DataFormats/Detectors/PHOS/src/BadChannelsMap.cxx b/DataFormats/Detectors/PHOS/src/BadChannelsMap.cxx index ad233d563d86e..a2be62ab8544f 100644 --- a/DataFormats/Detectors/PHOS/src/BadChannelsMap.cxx +++ b/DataFormats/Detectors/PHOS/src/BadChannelsMap.cxx @@ -12,7 +12,7 @@ #include "PHOSBase/Geometry.h" #include "DataFormatsPHOS/BadChannelsMap.h" -#include "FairLogger.h" +#include #include diff --git a/DataFormats/Detectors/PHOS/src/CalibParams.cxx b/DataFormats/Detectors/PHOS/src/CalibParams.cxx index a18b4b52d897f..7894c8e8c0095 100644 --- a/DataFormats/Detectors/PHOS/src/CalibParams.cxx +++ b/DataFormats/Detectors/PHOS/src/CalibParams.cxx @@ -12,7 +12,7 @@ #include "DataFormatsPHOS/CalibParams.h" #include "PHOSBase/Geometry.h" -#include "FairLogger.h" +#include #include diff --git a/DataFormats/Detectors/PHOS/src/Digit.cxx b/DataFormats/Detectors/PHOS/src/Digit.cxx index 30420e50a2c2f..0b2a34c2863ad 100644 --- a/DataFormats/Detectors/PHOS/src/Digit.cxx +++ b/DataFormats/Detectors/PHOS/src/Digit.cxx @@ -8,7 +8,7 @@ // In applying this license CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. -#include "FairLogger.h" +#include #include "DataFormatsPHOS/Digit.h" #include "PHOSBase/Hit.h" @@ -23,7 +23,7 @@ Digit::Digit(short absId, float amplitude, float time, int label) { } Digit::Digit(short truId, float amplitude, float time, bool isTrigger2x2, int /*dummy*/) - : DigitBase(time), mIsHighGain(true), mAbsId(truId + NREADOUTCHANNELS), mLabel(-1), mAmplitude(amplitude), mTime(time) + : DigitBase(time), mIsHighGain(true), mAbsId(truId), mLabel(-1), mAmplitude(amplitude), mTime(time) { setHighGain(isTrigger2x2); } diff --git a/DataFormats/Detectors/PHOS/src/EventHandler.cxx b/DataFormats/Detectors/PHOS/src/EventHandler.cxx index 1e74bf132687b..26e5fe15490b5 100644 --- a/DataFormats/Detectors/PHOS/src/EventHandler.cxx +++ b/DataFormats/Detectors/PHOS/src/EventHandler.cxx @@ -108,6 +108,23 @@ const typename EventHandler::CellRange EventHandler +std::vector> EventHandler::getCellMCLabelForEvent(int eventID) const +{ + if (mCellLabels && mTriggerRecordsCells.size()) { + if (eventID >= mTriggerRecordsCells.size()) { + throw RangeException(eventID, mTriggerRecordsCells.size()); + } + auto& trgrecord = mTriggerRecordsCells[eventID]; + std::vector> eventlabels(trgrecord.getNumberOfObjects()); + for (int index = 0; index < trgrecord.getNumberOfObjects(); index++) { + eventlabels[index] = mCellLabels->getLabels(trgrecord.getFirstEntry() + index); + } + return eventlabels; + } + throw NotInitializedException(); +} + template const typename EventHandler::CellIndexRange EventHandler::getClusterCellIndicesForEvent(int eventID) const { @@ -130,6 +147,7 @@ void EventHandler::reset() mClusters = ClusterRange(); mClusterCellIndices = CellIndexRange(); mCells = CellRange(); + mCellLabels = nullptr; } template @@ -146,6 +164,9 @@ EventData EventHandler::buildEvent(int eventID) co if (hasCells()) { outputEvent.mCells = getCellsForEvent(eventID); } + if (mCellLabels) { + outputEvent.mMCCellLabels = getCellMCLabelForEvent(eventID); + } return outputEvent; } diff --git a/DataFormats/Detectors/PHOS/src/Pedestals.cxx b/DataFormats/Detectors/PHOS/src/Pedestals.cxx index 8fa50f338fef6..552cf3022b11f 100644 --- a/DataFormats/Detectors/PHOS/src/Pedestals.cxx +++ b/DataFormats/Detectors/PHOS/src/Pedestals.cxx @@ -10,7 +10,7 @@ // or submit itself to any jurisdiction. #include "DataFormatsPHOS/Pedestals.h" -#include "FairLogger.h" +#include #include using namespace o2::phos; diff --git a/DataFormats/Detectors/PHOS/src/TriggerMap.cxx b/DataFormats/Detectors/PHOS/src/TriggerMap.cxx index e2a5d80292805..ee5e92cd159d7 100644 --- a/DataFormats/Detectors/PHOS/src/TriggerMap.cxx +++ b/DataFormats/Detectors/PHOS/src/TriggerMap.cxx @@ -12,7 +12,7 @@ #include "PHOSBase/Geometry.h" #include "DataFormatsPHOS/TriggerMap.h" -#include "FairLogger.h" +#include #include #include diff --git a/DataFormats/Detectors/TOF/CMakeLists.txt b/DataFormats/Detectors/TOF/CMakeLists.txt index 785fe1f4a5f92..4d41167f7bf1d 100644 --- a/DataFormats/Detectors/TOF/CMakeLists.txt +++ b/DataFormats/Detectors/TOF/CMakeLists.txt @@ -9,6 +9,14 @@ # granted to it by virtue of its status as an Intergovernmental Organization # or submit itself to any jurisdiction. +o2_add_library(DataFormatsParamTOF + SOURCES src/ParameterContainers.cxx + PUBLIC_LINK_LIBRARIES O2::FrameworkLogger) + + +o2_target_root_dictionary(DataFormatsParamTOF + HEADERS include/DataFormatsTOF/ParameterContainers.h) + o2_add_library(DataFormatsTOF SOURCES src/Cluster.cxx src/CalibInfoTOFshort.cxx @@ -22,6 +30,7 @@ o2_add_library(DataFormatsTOF src/TOFFEElightInfo.cxx PUBLIC_LINK_LIBRARIES O2::ReconstructionDataFormats O2::GPUCommon + O2::DataFormatsParamTOF Boost::serialization) o2_target_root_dictionary(DataFormatsTOF diff --git a/DataFormats/Detectors/TOF/include/DataFormatsTOF/CTF.h b/DataFormats/Detectors/TOF/include/DataFormatsTOF/CTF.h index 57d5784dc3b66..881f87ff88971 100644 --- a/DataFormats/Detectors/TOF/include/DataFormatsTOF/CTF.h +++ b/DataFormats/Detectors/TOF/include/DataFormatsTOF/CTF.h @@ -56,14 +56,14 @@ struct CompressedInfos { */ // ROF header data - std::vector bcIncROF; /// increment of ROF BC wrt BC of previous ROF - std::vector orbitIncROF; /// increment of ROF orbit wrt orbit of previous ROF + std::vector bcIncROF; /// increment of ROF BC wrt BC of previous ROF + std::vector orbitIncROF; /// increment of ROF orbit wrt orbit of previous ROF std::vector ndigROF; /// number of digits in ROF std::vector ndiaROF; /// number of diagnostic/pattern words in ROF std::vector ndiaCrate; /// number of diagnostic/pattern words per crate in ROF // Hit data - std::vector timeFrameInc; /// time increment with respect of previous digit in TimeFrame units + std::vector timeFrameInc; /// time increment with respect of previous digit in TimeFrame units std::vector timeTDCInc; /// time increment with respect of previous digit in TDC channel (about 24.4 ps) within timeframe std::vector stripID; /// increment of stripID wrt that of prev. strip std::vector chanInStrip; /// channel in strip 0-95 (ordered in time) diff --git a/DataFormats/Detectors/TOF/include/DataFormatsTOF/CalibInfoTOF.h b/DataFormats/Detectors/TOF/include/DataFormatsTOF/CalibInfoTOF.h index 2389955a9b1fb..218a8c4293cd3 100644 --- a/DataFormats/Detectors/TOF/include/DataFormatsTOF/CalibInfoTOF.h +++ b/DataFormats/Detectors/TOF/include/DataFormatsTOF/CalibInfoTOF.h @@ -40,8 +40,10 @@ class CalibInfoTOF void setTot(float tot) { mTot = tot; } float getTot() const { return mTot; } - void setFlags(int flags) { mFlags = flags; } - float getFlags() const { return mFlags; } + void setFlags(unsigned char flags) { mFlags = flags; } + unsigned char getFlags() const { return mFlags; } + + int getMask() const { return mMask; } // for event time maker float tofSignal() const { return mDeltaTimePi; } @@ -52,6 +54,17 @@ class CalibInfoTOF float tofExpSigmaKa() const { return 500.0; } float tofExpSigmaPr() const { return 500.0; } + enum Flags { + kTPC = BIT(0), + kITSTPC = BIT(1), + kTPCTRD = BIT(2), + kITSTPCTRD = BIT(3), + kBelow = BIT(4), // < 0.5 GeV/c + kAbove = BIT(5), // > 1.5 GeV/c + kNoBC = BIT(6), // no BC in the range + kMultiHit = BIT(7) // multi hit cluster + }; + private: int mTOFChIndex; // index of the TOF channel int mTimestamp; // timestamp in seconds diff --git a/DataFormats/Detectors/TOF/include/DataFormatsTOF/CalibTimeSlewingParamTOF.h b/DataFormats/Detectors/TOF/include/DataFormatsTOF/CalibTimeSlewingParamTOF.h index a6ccf7b0eb912..2b90fe5b40956 100644 --- a/DataFormats/Detectors/TOF/include/DataFormatsTOF/CalibTimeSlewingParamTOF.h +++ b/DataFormats/Detectors/TOF/include/DataFormatsTOF/CalibTimeSlewingParamTOF.h @@ -30,6 +30,8 @@ class CalibTimeSlewingParamTOF static const int NSECTORS = 18; // static const int NCHANNELXSECTOR = NCHANNELS / NSECTORS; // + static constexpr float MAXTOT = 65535 * 1E-3; // max tot -> max short / 1000. + CalibTimeSlewingParamTOF(); CalibTimeSlewingParamTOF(const CalibTimeSlewingParamTOF& source); @@ -37,10 +39,12 @@ class CalibTimeSlewingParamTOF CalibTimeSlewingParamTOF& operator=(const CalibTimeSlewingParamTOF& source) = default; float getChannelOffset(int channel) const; + void setChannelOffset(int channel, float val); float evalTimeSlewing(int channel, float tot) const; void addTimeSlewingInfo(int channel, float tot, float time); + void setTimeSlewingInfo(int channel, float offsetold, int nold, const unsigned short* oldtot, const short* olddt, int nnew, const unsigned short* newtot, const short* newdt); bool updateOffsetInfo(int channel, float residualOffset); diff --git a/DataFormats/Detectors/TOF/include/DataFormatsTOF/Cluster.h b/DataFormats/Detectors/TOF/include/DataFormatsTOF/Cluster.h index 589afc8a2cde9..37d3ca23ddb35 100644 --- a/DataFormats/Detectors/TOF/include/DataFormatsTOF/Cluster.h +++ b/DataFormats/Detectors/TOF/include/DataFormatsTOF/Cluster.h @@ -49,14 +49,18 @@ class Cluster : public o2::BaseCluster kDownRight = 4, // 2^4, 5th bit kDown = 5, // 2^5, 6th bit kDownLeft = 6, // 2^6, 7th bit - kLeft = 7 }; // 2^7, 8th bit + kLeft = 7 // 2^7, 8th bit + }; Cluster() = default; - Cluster(std::int16_t sensid, float x, float y, float z, float sy2, float sz2, float syz, double timeRaw, double time, float tot, int L0L1latency, int deltaBC); + Cluster(std::int16_t sensid, float x, float y, float z, float sy2, float sz2, float syz, double timeRaw, double time, float tot, int L0L1latency, int deltaBC, float geanttime = 0.0, double t0 = 0.0); ~Cluster() = default; + bool isInNominalSector() const { return mInNominalSector; } + void setInNominalSector(bool v = true) { mInNominalSector = v; } + std::int8_t getSector() const { return getCount(); } void setSector(std::int8_t value) { setCount(value); } @@ -134,6 +138,10 @@ class Cluster : public o2::BaseCluster int getDigitInfoCH(int idig) const { return mDigitInfoCh[idig]; } double getDigitInfoT(int idig) const { return mDigitInfoT[idig]; } float getDigitInfoTOT(int idig) const { return mDigitInfoTOT[idig]; } + float getTgeant() const { return mTgeant; } + void setTgeant(float val) { mTgeant = val; } + double getT0true() const { return mT0true; } + void setT0true(double val) { mT0true = val; } private: #if !defined(GPUCA_GPUCODE) && !defined(GPUCA_STANDALONE) @@ -153,8 +161,11 @@ class Cluster : public o2::BaseCluster int mDigitInfoCh[6] = {0, 0, 0, 0, 0, 0}; double mDigitInfoT[6] = {0., 0., 0., 0., 0., 0.}; float mDigitInfoTOT[6] = {0., 0., 0., 0., 0., 0.}; + float mTgeant = 0.0; + bool mInNominalSector = false; + double mT0true = 0.0; - ClassDefNV(Cluster, 4); + ClassDefNV(Cluster, 6); }; #ifndef GPUCA_GPUCODE diff --git a/DataFormats/Detectors/TOF/include/DataFormatsTOF/Diagnostic.h b/DataFormats/Detectors/TOF/include/DataFormatsTOF/Diagnostic.h index df4c03c03dd1c..8adcdb63e9d21 100644 --- a/DataFormats/Detectors/TOF/include/DataFormatsTOF/Diagnostic.h +++ b/DataFormats/Detectors/TOF/include/DataFormatsTOF/Diagnostic.h @@ -32,20 +32,20 @@ class Diagnostic { public: Diagnostic() = default; - int fill(ULong64_t pattern); - int fill(ULong64_t pattern, int frequency); - int getFrequency(ULong64_t pattern) const; // Get frequency - int getFrequencyROW() const { return getFrequency(0); } // Readout window frequency - int getFrequencyEmptyCrate(int crate) const { return getFrequency(getEmptyCrateKey(crate)); } // empty crate frequency - int getFrequencyEmptyTOF() const { return getFrequency(1); } // empty crate frequency - int fillNoisy(int channel, int frequency = 1) { return fill(getNoisyChannelKey(channel), frequency); } - int fillROW() { return fill(0); } - int fillEmptyCrate(int crate, int frequency = 1) { return fill(getEmptyCrateKey(crate), frequency); } - int fillEmptyTOF(int frequency = 1) { return fill(1, frequency); } + uint32_t fill(ULong64_t pattern); + uint32_t fill(ULong64_t pattern, uint32_t frequency); + uint32_t getFrequency(ULong64_t pattern) const; // Get frequency + uint32_t getFrequencyROW() const { return getFrequency(0); } // Readout window frequency + uint32_t getFrequencyEmptyCrate(int crate) const { return getFrequency(getEmptyCrateKey(crate)); } // empty crate frequency + uint32_t getFrequencyEmptyTOF() const { return getFrequency(1); } // empty crate frequency + uint32_t fillNoisy(int channel, int frequency = 1) { return fill(getNoisyChannelKey(channel), frequency); } + uint32_t fillROW() { return fill(0); } + uint32_t fillEmptyCrate(int crate, uint32_t frequency = 1) { return fill(getEmptyCrateKey(crate), frequency); } + uint32_t fillEmptyTOF(uint32_t frequency = 1) { return fill(1, frequency); } static ULong64_t getEmptyCrateKey(int crate); static ULong64_t getNoisyChannelKey(int channel); static ULong64_t getTRMKey(int crate, int trm); - void print() const; + void print(bool longFormat = false) const; void clear() { mVector.clear(); } void fill(const Diagnostic& diag); // for calibration void fill(const gsl::span){}; // for calibration diff --git a/DataFormats/Detectors/TOF/include/DataFormatsTOF/ParameterContainers.h b/DataFormats/Detectors/TOF/include/DataFormatsTOF/ParameterContainers.h new file mode 100644 index 0000000000000..224906e43b8c6 --- /dev/null +++ b/DataFormats/Detectors/TOF/include/DataFormatsTOF/ParameterContainers.h @@ -0,0 +1,272 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file ParameterContainers.h +/// \author Francesco Noferini +/// \author Nicolò Jacazio nicolo.jacazio@cern.ch +/// @since 2022-11-08 +/// \brief Definitions of the containers for the general parameters + +#ifndef O2_TOF_PARAMCONTAINER_H +#define O2_TOF_PARAMCONTAINER_H + +#include +#include +#include +#include + +namespace o2 +{ +namespace tof +{ +using paramvar_t = float; + +template +class Parameters +{ + public: + /// Default constructor + Parameters(std::array parNames, std::string name) : mName{name}, mPar{}, mParNames{parNames} {}; + + /// Default destructor + ~Parameters() = default; + + /// Setter for the parameter at position iparam + /// \param iparam index in the array of the parameters + /// \param value value of the parameter at position iparam + void setParameter(const unsigned int iparam, const paramvar_t value) { mPar[iparam] = value; } + + /// Setter for the parameter, using an array + /// \param param array with parameters + void setParameters(const paramvar_t* params) { std::copy(params, params + mPar.size(), mPar.begin()); } + + /// Setter for the parameter, using a vector + /// \param params vector with parameters + void setParameters(const std::array params) + { + for (int i = 0; i < nPar; i++) { + mPar[i] = params[i]; + } + } + + /// Setter for the parameter, using a parameter object + /// \param params parameter object with parameters + void setParameters(const Parameters params) { setParameters(params.mPar); }; + + /// Setter for the parameter, using a parameter pointer + /// \param params pointer to parameter object with parameters + void setParameters(const Parameters* params) { setParameters(params->mPar); }; + + /// Printer of the parameter values + void print() const + { + LOG(info) << "Parameters '" << mName << "'"; + for (int i = 0; i < nPar; i++) { + LOG(info) << "Parameter " << i << "/" << nPar - 1 << " " << mParNames[i] << " is " << mPar[i]; + } + } + + /// Adds the parameters to the metadata + void addToMetadata(std::map& metadata) const + { + for (int i = 0; i < nPar; i++) { + metadata[Form("p%i", i)] = Form("%f", mPar[i]); + } + } + + /// Loader from file + /// \param FileName name of the input file + /// \param ParamName name of the input object + void loadParamFromFile(const TString FileName, const TString ParamName) + { + TFile f(FileName, "READ"); + if (!f.IsOpen()) { + LOG(fatal) << "Could not open file " << FileName; + } + if (!f.Get(ParamName)) { + f.ls(); + LOG(fatal) << "Did not find parameters " << ParamName << " in file " << FileName; + } + LOG(info) << "Loading parameters " << ParamName << " from TFile " << FileName; + Parameters* p; + f.GetObject(ParamName, p); + if (!p) { + LOG(fatal) << "Could not get parameters " << ParamName << " from file"; + f.ls(); + } + f.Close(); + setParameters(p); + print(); + } + + /// Getter for the parameters + /// \return returns an array of parameters + const paramvar_t* getParameters() const { return mPar.to_array(); } + + /// Getter for the parameters + /// \return returns an array of parameters + const paramvar_t getParameter(int i) const { return mPar[i]; } + + /// Getter for the parameters + /// \return returns an array of parameters + const std::string getParameterName(int i) const { return mParNames[i]; } + + /// Getter for the parameters + /// \return returns an array of parameters + const std::string getName() const { return mName; } + + /// Getter for the size of the parameter + /// \return returns the size of the parameter array + static int size() { return nPar; } + + /// Getter of the parameter at position i + /// \param i index of the parameter to get + /// \return returns the parameter value at position i + paramvar_t operator[](const unsigned int i) const { return mPar[i]; } + + private: + /// Array of the parameter + std::array mPar; + const std::array mParNames; + std::string mName; +}; + +/// \brief Class container to hold different parameters meant to be stored on the CCDB +class ParameterCollection : public TNamed +{ + public: + /// Default constructor + ParameterCollection(TString name = "DefaultParameters") : TNamed(name, name), mParameters{} {}; + + /// Default destructor + ~ParameterCollection() override = default; + + /// @brief Checks if the container has a particular key e.g. a pass + /// @return true if found, false if not + bool hasKey(const std::string& key) const { return (mParameters.find(key) != mParameters.end()); } + + /// @brief Function to load the parameters from the this container into the array based for the asked key, e.g. pass or version + /// Parameters that are not found in storage are kept unchanged in the array. + /// @tparam ParType type of the parameter container + /// @param p parameter list to configure from the stored information + /// @param key key to look for in the stored information e.g. pass + /// @return true if found and configured false if not fully configured + template + bool retrieveParameters(ParType& p, const std::string& key) const + { + if (!hasKey(key)) { // Can't find the required key. Can't load parameters to the object + return false; + } + + const auto& toGet = mParameters.at(key); + for (int i = 0; i < p.size(); i++) { + const auto& name = p.getParameterName(i); + if (toGet.find(name) == toGet.end()) { + LOG(debug) << "Did not find parameter '" << name << "' in collection, keeping preexisting"; + continue; + } + LOG(debug) << "Found parameter '" << name << "' in collection, updating from " << p[i] << " to " << toGet.at(name); + p.setParameter(i, toGet.at(name)); + } + return true; + } + + /// @brief Function to add a single parameter conatiner based on the asked key, e.g. pass or version + /// @param value parameter to add to the stored information + /// @param pass key to look for in the stored information e.g. pass + /// @return true if found and configured false if not fully configured + bool addParameter(const std::string& pass, const std::string& parName, float value); + + /// @return the size of the container i.e. the number of stored keys (or passes) + int getSize(const std::string& pass) const; + + /// @brief Function to push the parameters from the sub container into the collection and store it under a given key + /// @tparam ParType type of the parameter container + /// @param p parameter list to store + /// @param key store key + /// @return true if modified and false if a new key is added + template + bool storeParameters(const ParType& p, const std::string& key) + { + const bool alreadyPresent = hasKey(key); + if (alreadyPresent) { + LOG(debug) << "Changing parametrization corresponding to key " << key << " from size " << mParameters[key].size() << " to " << p.getName() << " of size " << p.size(); + } else { + mParameters[key] = std::unordered_map{}; + LOG(debug) << "Adding new parametrization corresponding to key " << key << ": " << p.getName() << " of size " << p.size(); + } + for (int i = 0; i < p.size(); i++) { + mParameters[key][p.getParameterName(i)] = p[i]; + } + return alreadyPresent; + } + + /// @brief getter for the parameters stored in the container matching to a pass + const std::unordered_map& getPars(const std::string& pass) const + { + if (!hasKey(pass)) { + LOG(fatal) << "Parameters for pass " << pass << " not found!"; + } + return mParameters.at(pass); + } + + /// @brief printing function for the content of the pass + /// @param pass pass to print + void print(const std::string& pass) const; + + /// @brief printing function for the full content of the container + void print() const; + + /// @brief Getter of the full map of parameters stored in the container + /// @return returns the full map of parameters + const std::unordered_map>& getFullMap() const { return mParameters; } + + /// Loader from file + /// \param FileName name of the input file + /// \param ParamName name of the input object + void loadParamFromFile(const TString FileName, const TString ParamName) + { + TFile f(FileName, "READ"); + if (!f.IsOpen()) { + LOG(fatal) << "Could not open file " << FileName; + } + if (!f.Get(ParamName)) { + f.ls(); + LOG(fatal) << "Did not find parameters " << ParamName << " in file " << FileName; + } + LOG(info) << "Loading parameters " << ParamName << " from TFile " << FileName; + ParameterCollection* p; + f.GetObject(ParamName, p); + if (!p) { + LOG(fatal) << "Could not get parameters " << ParamName << " from file"; + f.ls(); + } + f.Close(); + + for (const auto& pass : p->mParameters) { + for (const auto& par : pass.second) { + addParameter(pass.first, par.first, par.second); + } + } + print(); + } + + private: + /// Array of the parameter + std::unordered_map> mParameters; + + ClassDefOverride(ParameterCollection, 1); // Container for containers of parameter of parametrizations. To be used as a manager, in help of CCDB +}; + +} // namespace tof +} // namespace o2 + +#endif // O2_TOF_PARAMCONTAINER_H diff --git a/DataFormats/Detectors/TOF/src/CalibTimeSlewingParamTOF.cxx b/DataFormats/Detectors/TOF/src/CalibTimeSlewingParamTOF.cxx index 0d47e2c224511..01df4f2323edd 100644 --- a/DataFormats/Detectors/TOF/src/CalibTimeSlewingParamTOF.cxx +++ b/DataFormats/Detectors/TOF/src/CalibTimeSlewingParamTOF.cxx @@ -33,8 +33,21 @@ CalibTimeSlewingParamTOF::CalibTimeSlewingParamTOF() //______________________________________________ float CalibTimeSlewingParamTOF::getChannelOffset(int channel) const { - return evalTimeSlewing(channel, 0); + int sector = channel / NCHANNELXSECTOR; + channel = channel % NCHANNELXSECTOR; + + return (*(mGlobalOffset[sector]))[channel]; +} + +//______________________________________________ +void CalibTimeSlewingParamTOF::setChannelOffset(int channel, float val) +{ + int sector = channel / NCHANNELXSECTOR; + channel = channel % NCHANNELXSECTOR; + + (*(mGlobalOffset[sector]))[channel] = val; } +//______________________________________________ void CalibTimeSlewingParamTOF::bind() { mGlobalOffset[0] = &mGlobalOffsetSec0; @@ -55,12 +68,87 @@ void CalibTimeSlewingParamTOF::bind() mGlobalOffset[15] = &mGlobalOffsetSec15; mGlobalOffset[16] = &mGlobalOffsetSec16; mGlobalOffset[17] = &mGlobalOffsetSec17; + + mChannelStart[0] = &mChannelStartSec0; + mChannelStart[1] = &mChannelStartSec1; + mChannelStart[2] = &mChannelStartSec2; + mChannelStart[3] = &mChannelStartSec3; + mChannelStart[4] = &mChannelStartSec4; + mChannelStart[5] = &mChannelStartSec5; + mChannelStart[6] = &mChannelStartSec6; + mChannelStart[7] = &mChannelStartSec7; + mChannelStart[8] = &mChannelStartSec8; + mChannelStart[9] = &mChannelStartSec9; + mChannelStart[10] = &mChannelStartSec10; + mChannelStart[11] = &mChannelStartSec11; + mChannelStart[12] = &mChannelStartSec12; + mChannelStart[13] = &mChannelStartSec13; + mChannelStart[14] = &mChannelStartSec14; + mChannelStart[15] = &mChannelStartSec15; + mChannelStart[16] = &mChannelStartSec16; + mChannelStart[17] = &mChannelStartSec17; + + mTimeSlewing[0] = &mTimeSlewingSec0; + mTimeSlewing[1] = &mTimeSlewingSec1; + mTimeSlewing[2] = &mTimeSlewingSec2; + mTimeSlewing[3] = &mTimeSlewingSec3; + mTimeSlewing[4] = &mTimeSlewingSec4; + mTimeSlewing[5] = &mTimeSlewingSec5; + mTimeSlewing[6] = &mTimeSlewingSec6; + mTimeSlewing[7] = &mTimeSlewingSec7; + mTimeSlewing[8] = &mTimeSlewingSec8; + mTimeSlewing[9] = &mTimeSlewingSec9; + mTimeSlewing[10] = &mTimeSlewingSec10; + mTimeSlewing[11] = &mTimeSlewingSec11; + mTimeSlewing[12] = &mTimeSlewingSec12; + mTimeSlewing[13] = &mTimeSlewingSec13; + mTimeSlewing[14] = &mTimeSlewingSec14; + mTimeSlewing[15] = &mTimeSlewingSec15; + mTimeSlewing[16] = &mTimeSlewingSec16; + mTimeSlewing[17] = &mTimeSlewingSec17; + + mFractionUnderPeak[0] = &mFractionUnderPeakSec0; + mFractionUnderPeak[1] = &mFractionUnderPeakSec1; + mFractionUnderPeak[2] = &mFractionUnderPeakSec2; + mFractionUnderPeak[3] = &mFractionUnderPeakSec3; + mFractionUnderPeak[4] = &mFractionUnderPeakSec4; + mFractionUnderPeak[5] = &mFractionUnderPeakSec5; + mFractionUnderPeak[6] = &mFractionUnderPeakSec6; + mFractionUnderPeak[7] = &mFractionUnderPeakSec7; + mFractionUnderPeak[8] = &mFractionUnderPeakSec8; + mFractionUnderPeak[9] = &mFractionUnderPeakSec9; + mFractionUnderPeak[10] = &mFractionUnderPeakSec10; + mFractionUnderPeak[11] = &mFractionUnderPeakSec11; + mFractionUnderPeak[12] = &mFractionUnderPeakSec12; + mFractionUnderPeak[13] = &mFractionUnderPeakSec13; + mFractionUnderPeak[14] = &mFractionUnderPeakSec14; + mFractionUnderPeak[15] = &mFractionUnderPeakSec15; + mFractionUnderPeak[16] = &mFractionUnderPeakSec16; + mFractionUnderPeak[17] = &mFractionUnderPeakSec17; + + mSigmaPeak[0] = &mSigmaPeakSec0; + mSigmaPeak[1] = &mSigmaPeakSec1; + mSigmaPeak[2] = &mSigmaPeakSec2; + mSigmaPeak[3] = &mSigmaPeakSec3; + mSigmaPeak[4] = &mSigmaPeakSec4; + mSigmaPeak[5] = &mSigmaPeakSec5; + mSigmaPeak[6] = &mSigmaPeakSec6; + mSigmaPeak[7] = &mSigmaPeakSec7; + mSigmaPeak[8] = &mSigmaPeakSec8; + mSigmaPeak[9] = &mSigmaPeakSec9; + mSigmaPeak[10] = &mSigmaPeakSec10; + mSigmaPeak[11] = &mSigmaPeakSec11; + mSigmaPeak[12] = &mSigmaPeakSec12; + mSigmaPeak[13] = &mSigmaPeakSec13; + mSigmaPeak[14] = &mSigmaPeakSec14; + mSigmaPeak[15] = &mSigmaPeakSec15; + mSigmaPeak[16] = &mSigmaPeakSec16; + mSigmaPeak[17] = &mSigmaPeakSec17; } //______________________________________________ float CalibTimeSlewingParamTOF::evalTimeSlewing(int channel, float totIn) const { - // totIn is in ns // the correction is returned in ps @@ -72,23 +160,29 @@ float CalibTimeSlewingParamTOF::evalTimeSlewing(int channel, float totIn) const } int n = (*(mChannelStart[sector]))[channel]; + int nFirst = n; + if (n < 0) { - return 0.; + return (*(mGlobalOffset[sector]))[channel]; } - int nstop = mTimeSlewing[sector]->size(); + if (channel < NCHANNELXSECTOR - 1) { nstop = (*(mChannelStart[sector]))[channel + 1]; } if (n >= nstop) { - return 0.; // something went wrong! + return (*(mGlobalOffset[sector]))[channel]; // something went wrong! } if (totIn == 0) { return (float)((*(mTimeSlewing[sector]))[n].second + (*(mGlobalOffset[sector]))[channel]); } + if (totIn > MAXTOT) { + totIn = MAXTOT; + } + // we convert tot from ns to ps and to unsigned short unsigned short tot = (unsigned short)(totIn * 1000); @@ -98,20 +192,104 @@ float CalibTimeSlewingParamTOF::evalTimeSlewing(int channel, float totIn) const n--; if (n < 0) { // tot is lower than the first available value - return 0; + return (*(mGlobalOffset[sector]))[channel]; } if (n == nstop - 1) { return (float)((*(mTimeSlewing[sector]))[n].second + (*(mGlobalOffset[sector]))[channel]); // use the last value stored for that channel } + if (n < nFirst) { // this is before the first point (probably tot = 0 calibration is missing) + return (*(mGlobalOffset[sector]))[channel]; + } + float w1 = (float)(tot - (*(mTimeSlewing[sector]))[n].first); float w2 = (float)((*(mTimeSlewing[sector]))[n + 1].first - tot); - return (float)((*(mGlobalOffset[sector]))[channel] + (((*(mTimeSlewing[sector]))[n].second * w2 + (*(mTimeSlewing[sector]))[n + 1].second * w1) / ((*(mTimeSlewing[sector]))[n + 1].first - (*(mTimeSlewing[sector]))[n].first))); + return (float)((*(mGlobalOffset[sector]))[channel] + (((*(mTimeSlewing[sector]))[n].second * w2 + (*(mTimeSlewing[sector]))[n + 1].second * w1) / (w1 + w2))); } //______________________________________________ +void CalibTimeSlewingParamTOF::setTimeSlewingInfo(int channel, float offsetold, int nold, const unsigned short* oldtot, const short* olddt, int nnew, const unsigned short* newtot, const short* newdt) +{ + // to be used when updating new calibration from scratch and merging it with the old one (all channels should be called in order) + int sector = channel / NCHANNELXSECTOR; + channel = channel % NCHANNELXSECTOR; + + // work with float to avoid to loose precision due to short (in case offset will be adjust before to move deltat to short) + std::vector deltat; + + float minVal = 100000; + float maxVal = -100000; + + // new scheme + int k1 = 0; + int k2 = 0; + + int ntotFin = 0; + std::vector totFin; + + while (k1 < nnew && k2 < nold) { + if (newtot[k1] == oldtot[k2]) { + k1++; + } else if (newtot[k1] < oldtot[k2]) { + totFin.push_back(newtot[k1]); + if (k2 > 0) { + deltat.push_back(float(newdt[k1]) + olddt[k2 - 1]); + } else { + deltat.push_back(float(newdt[k1] + olddt[k2])); + } + k1++; + } else { + totFin.push_back(oldtot[k2]); + if (k1 > 0) { + deltat.push_back(float(olddt[k2]) + newdt[k1 - 1]); + } else { + deltat.push_back(float(olddt[k2] + newdt[k1])); + } + k2++; + } + } + if (k1 < nnew) { + while (k1 < nnew) { + totFin.push_back(newtot[k1]); + if (nold) { + deltat.push_back(float(newdt[k1]) + olddt[nold - 1]); + } else { + deltat.push_back(float(newdt[k1])); + } + k1++; + } + } else if (k2 < nold) { + while (k2 < nold) { + totFin.push_back(oldtot[k2]); + if (nnew) { + deltat.push_back(float(olddt[k2]) + newdt[nnew - 1]); + } else { + deltat.push_back(float(olddt[k2])); + } + k2++; + } + } + + for (int i = 0; i < deltat.size(); i++) { + if (deltat[i] < minVal) { + minVal = deltat[i]; + } + if (deltat[i] > maxVal) { + maxVal = deltat[i]; + } + } + + float recentering = (minVal + maxVal) * 0.5; + (*(mGlobalOffset[sector]))[channel] = offsetold + recentering; + (*(mChannelStart[sector]))[channel] = mTimeSlewing[sector]->size(); + + for (int i = 0; i < deltat.size(); i++) { + (*(mTimeSlewing[sector])).emplace_back(totFin[i], (short)(deltat[i] - recentering)); + } +} +//______________________________________________ void CalibTimeSlewingParamTOF::addTimeSlewingInfo(int channel, float tot, float time) { // WE ARE ASSUMING THAT: @@ -201,4 +379,6 @@ CalibTimeSlewingParamTOF::CalibTimeSlewingParamTOF(const CalibTimeSlewingParamTO *(mSigmaPeak[i]) = *(source.mSigmaPeak[i]); *(mGlobalOffset[i]) = *(source.mGlobalOffset[i]); } + mStartValidity = source.mStartValidity; + mEndValidity = source.mEndValidity; } diff --git a/DataFormats/Detectors/TOF/src/Cluster.cxx b/DataFormats/Detectors/TOF/src/Cluster.cxx index 8e688ee91e801..a7f3473e0b61c 100644 --- a/DataFormats/Detectors/TOF/src/Cluster.cxx +++ b/DataFormats/Detectors/TOF/src/Cluster.cxx @@ -13,7 +13,7 @@ /// \brief Implementation of the TOF cluster #include "DataFormatsTOF/Cluster.h" -#include "FairLogger.h" +#include #include @@ -23,7 +23,7 @@ using namespace o2::tof; ClassImp(o2::tof::Cluster); -Cluster::Cluster(std::int16_t sensid, float x, float y, float z, float sy2, float sz2, float syz, double timeRaw, double time, float tot, int L0L1Latency, int deltaBC) : o2::BaseCluster(sensid, x, y, z, sy2, sz2, syz), mTimeRaw(timeRaw), mTime(time), mTot(tot), mL0L1Latency(L0L1Latency), mDeltaBC(deltaBC) +Cluster::Cluster(std::int16_t sensid, float x, float y, float z, float sy2, float sz2, float syz, double timeRaw, double time, float tot, int L0L1Latency, int deltaBC, float geanttime, double t0) : o2::BaseCluster(sensid, x, y, z, sy2, sz2, syz), mTimeRaw(timeRaw), mTime(time), mTot(tot), mL0L1Latency(L0L1Latency), mDeltaBC(deltaBC), mTgeant(geanttime), mT0true(t0) { // caching R and phi diff --git a/DataFormats/Detectors/TOF/src/DataFormatsParamTOFLinkDef.h b/DataFormats/Detectors/TOF/src/DataFormatsParamTOFLinkDef.h new file mode 100644 index 0000000000000..2d6ee84bedb92 --- /dev/null +++ b/DataFormats/Detectors/TOF/src/DataFormatsParamTOFLinkDef.h @@ -0,0 +1,17 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifdef __CLING__ + +#pragma link C++ class o2::tof::Parameters < 5> + ; +#pragma link C++ class o2::tof::ParameterCollection + ; + +#endif diff --git a/DataFormats/Detectors/TOF/src/Diagnostic.cxx b/DataFormats/Detectors/TOF/src/Diagnostic.cxx index ae195e86fc296..27ca65d124dd7 100644 --- a/DataFormats/Detectors/TOF/src/Diagnostic.cxx +++ b/DataFormats/Detectors/TOF/src/Diagnostic.cxx @@ -20,9 +20,9 @@ using namespace o2::tof; ClassImp(Diagnostic); -int Diagnostic::fill(ULong64_t pattern) +uint32_t Diagnostic::fill(ULong64_t pattern) { - int frequency = 1; + uint32_t frequency = 1; auto pairC = mVector.find(pattern); @@ -35,7 +35,7 @@ int Diagnostic::fill(ULong64_t pattern) return frequency; } -int Diagnostic::fill(ULong64_t pattern, int frequency) +uint32_t Diagnostic::fill(ULong64_t pattern, uint32_t frequency) { auto pairC = mVector.find(pattern); @@ -49,7 +49,7 @@ int Diagnostic::fill(ULong64_t pattern, int frequency) return frequency; } -int Diagnostic::getFrequency(ULong64_t pattern) const +uint32_t Diagnostic::getFrequency(ULong64_t pattern) const { auto pairC = mVector.find(pattern); if (pairC != mVector.end()) { @@ -59,9 +59,14 @@ int Diagnostic::getFrequency(ULong64_t pattern) const return 0; } -void Diagnostic::print() const +void Diagnostic::print(bool longFormat) const { - LOG(info) << "Diagnostic patterns"; + LOG(info) << "Diagnostic patterns, entries = " << mVector.size(); + + if (!longFormat) { + return; + } + for (const auto& [key, value] : mVector) { std::cout << key << " = " << value << "; "; } @@ -99,7 +104,7 @@ void Diagnostic::merge(const Diagnostic* prev) { LOG(debug) << "Merging diagnostic words"; for (auto const& el : prev->mVector) { - fill(el.first, el.second + getFrequency(el.first)); + fill(el.first, el.second); } } diff --git a/DataFormats/Detectors/TOF/src/ParameterContainers.cxx b/DataFormats/Detectors/TOF/src/ParameterContainers.cxx new file mode 100644 index 0000000000000..91f723873e9cd --- /dev/null +++ b/DataFormats/Detectors/TOF/src/ParameterContainers.cxx @@ -0,0 +1,62 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file ParameterContainers.h +/// \author Francesco Noferini +/// \author Nicolò Jacazio nicolo.jacazio@cern.ch +/// @since 2022-11-08 +/// \brief Implementation of the containers for the general parameters + +#include "DataFormatsTOF/ParameterContainers.h" + +// ClassImp(o2::tof::Parameters); +using namespace o2::tof; + +bool ParameterCollection::addParameter(const std::string& pass, const std::string& parName, float value) +{ + const bool alreadyPresent = hasKey(pass); + if (alreadyPresent) { + LOG(debug) << "Changing parametrization corresponding to key " << pass << " from size " << mParameters[pass].size() << " to " << parName; + } else { + mParameters[pass] = std::unordered_map{}; + LOG(debug) << "Adding new parametrization corresponding to key " << pass << ": " << parName; + } + mParameters[pass][parName] = value; + return true; +} + +int ParameterCollection::getSize(const std::string& pass) const +{ + if (!hasKey(pass)) { + return -1; + } + return mParameters.at(pass).size(); +} + +void ParameterCollection::print() const +{ + for (const auto& [pass, pars] : mParameters) { + print(pass); + } +} + +void ParameterCollection::print(const std::string& pass) const +{ + const auto& size = getSize(pass); + if (size < 0) { + LOG(info) << "empty pass: " << pass; + return; + } + LOG(info) << "Pass \"" << pass << "\" with size " << size; + for (const auto& [par, value] : mParameters.at(pass)) { + LOG(info) << "par name = " << par << ", value = " << value; + } +} diff --git a/DataFormats/Detectors/TPC/CMakeLists.txt b/DataFormats/Detectors/TPC/CMakeLists.txt index 2743dbe0f367b..2cc69e16001a6 100644 --- a/DataFormats/Detectors/TPC/CMakeLists.txt +++ b/DataFormats/Detectors/TPC/CMakeLists.txt @@ -31,9 +31,12 @@ o2_add_library( src/DCS.cxx PUBLIC_LINK_LIBRARIES O2::GPUCommon O2::SimulationDataFormat + O2::ReconstructionDataFormats O2::CommonDataFormat O2::Headers - O2::Algorithm) + O2::DataSampling + O2::Algorithm + ROOT::Minuit) o2_target_root_dictionary( DataFormatsTPC @@ -61,7 +64,9 @@ o2_target_root_dictionary( include/DataFormatsTPC/LtrCalibData.h include/DataFormatsTPC/VDriftCorrFact.h include/DataFormatsTPC/CalibdEdxCorrection.h - include/DataFormatsTPC/BetheBlochAleph.h) + include/DataFormatsTPC/BetheBlochAleph.h + include/DataFormatsTPC/PIDResponse.h + include/DataFormatsTPC/AltroSyncSignal.h) o2_add_test( ClusterNative diff --git a/DataFormats/Detectors/TPC/include/DataFormatsTPC/AltroSyncSignal.h b/DataFormats/Detectors/TPC/include/DataFormatsTPC/AltroSyncSignal.h new file mode 100644 index 0000000000000..6dee49e4ed6c6 --- /dev/null +++ b/DataFormats/Detectors/TPC/include/DataFormatsTPC/AltroSyncSignal.h @@ -0,0 +1,30 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file AltroSyncSignal.h +/// \brief Definition of the timebin from which syncronization starts + +#include "GPUCommonRtypes.h" + +namespace o2::tpc +{ +struct AltroSyncSignal { + int periodTF = 10; // signal repeats every period-th TF + int timebin = 14192; // every 10 TF, orbit 31, Time bin 384, BC 4 -> 14195, but clusters can be affected before that + + int getTB2Cut(uint32_t tfCounter) const + { + return periodTF > 0 && (tfCounter % periodTF) == 1 && tfCounter > periodTF ? timebin : -1; + } + + ClassDefNV(AltroSyncSignal, 1); +}; +} // namespace o2::tpc diff --git a/DataFormats/Detectors/TPC/include/DataFormatsTPC/BetheBlochAleph.h b/DataFormats/Detectors/TPC/include/DataFormatsTPC/BetheBlochAleph.h index 3fff89b44ba0f..28b224298f36f 100644 --- a/DataFormats/Detectors/TPC/include/DataFormatsTPC/BetheBlochAleph.h +++ b/DataFormats/Detectors/TPC/include/DataFormatsTPC/BetheBlochAleph.h @@ -12,26 +12,17 @@ #ifndef AliceO2_TPC_BETHEBLOCH_H_ #define AliceO2_TPC_BETHEBLOCH_H_ -#include +#include "MathUtils/BetheBlochAleph.h" -namespace o2 -{ -namespace tpc +namespace o2::tpc { template -inline T BetheBlochAleph(T bg, T kp1, T kp2, T kp3, T kp4, T kp5) +GPUdi() T BetheBlochAleph(T bg, T kp1, T kp2, T kp3, T kp4, T kp5) { - T beta = bg / std::sqrt(static_cast(1.) + bg * bg); - - T aa = std::pow(beta, kp4); - T bb = std::pow(static_cast(1.) / bg, kp5); - bb = std::log(kp3 + bb); - - return (kp2 - aa - bb) * kp1 / aa; + return o2::common::BetheBlochAleph(bg, kp1, kp2, kp3, kp4, kp5); } -} // namespace tpc -} // namespace o2 +} // namespace o2::tpc #endif diff --git a/DataFormats/Detectors/TPC/include/DataFormatsTPC/CTF.h b/DataFormats/Detectors/TPC/include/DataFormatsTPC/CTF.h index 1e57c9654932c..1f4f08db9d1b9 100644 --- a/DataFormats/Detectors/TPC/include/DataFormatsTPC/CTF.h +++ b/DataFormats/Detectors/TPC/include/DataFormatsTPC/CTF.h @@ -27,14 +27,15 @@ namespace tpc struct CTFHeader : public ctf::CTFDictHeader, public CompressedClustersCounters { enum : uint32_t { CombinedColumns = 0x1 }; uint32_t flags = 0; - - ClassDefNV(CTFHeader, 2); + uint32_t firstOrbitTrig = 0; /// orbit of 1st trigger + uint16_t nTriggers = 0; /// number of triggers + ClassDefNV(CTFHeader, 3); }; /// wrapper for the Entropy-encoded clusters of the TF -struct CTF : public o2::ctf::EncodedBlocks { +struct CTF : public o2::ctf::EncodedBlocks { - using container_t = o2::ctf::EncodedBlocks; + using container_t = o2::ctf::EncodedBlocks; static constexpr size_t N = getNBlocks(); static constexpr int NBitsQTot = 16; @@ -66,9 +67,14 @@ struct CTF : public o2::ctf::EncodedBlocks { BLCsigmaPadU, BLCsigmaTimeU, // can be combined with BLCsigmaPadU BLCnTrackClusters, - BLCnSliceRowClusters }; + BLCnSliceRowClusters, + // trigger info + BLCTrigOrbitInc, + BLCTrigBCInc, + BLCTrigType + }; - ClassDefNV(CTF, 2); + ClassDefNV(CTF, 4); }; } // namespace tpc diff --git a/DataFormats/Detectors/TPC/include/DataFormatsTPC/CalibdEdxCorrection.h b/DataFormats/Detectors/TPC/include/DataFormatsTPC/CalibdEdxCorrection.h index 277875e8edfde..024d6189593e9 100644 --- a/DataFormats/Detectors/TPC/include/DataFormatsTPC/CalibdEdxCorrection.h +++ b/DataFormats/Detectors/TPC/include/DataFormatsTPC/CalibdEdxCorrection.h @@ -22,6 +22,7 @@ #ifndef GPUCA_GPUCODE_DEVICE #include #include +#include #endif // o2 includes @@ -30,6 +31,11 @@ namespace o2::tpc { +namespace conf_dedx_corr +{ +GPUconstexpr() float TglScale[4] = {1.9, 1.5, 1.22, 1.02}; ///< Max Tgl values for each ROC type +} + class CalibdEdxCorrection { public: @@ -43,9 +49,9 @@ class CalibdEdxCorrection } CalibdEdxCorrection(std::string_view fileName) { loadFromFile(fileName); } #else - CalibdEdxCorrection() CON_DEFAULT; + CalibdEdxCorrection() = default; #endif - ~CalibdEdxCorrection() CON_DEFAULT; + ~CalibdEdxCorrection() = default; GPUd() float getCorrection(const StackID& stack, ChargeType charge, float tgl = 0, float snp = 0) const { @@ -54,7 +60,8 @@ class CalibdEdxCorrection return 1; } - tgl = o2::gpu::CAMath::Abs(tgl); + // limit to the fit range in the respective region + tgl = o2::gpu::CAMath::Min(conf_dedx_corr::TglScale[stack.type], o2::gpu::CAMath::Abs(tgl)); auto p = mParams[stackIndex(stack, charge)]; float result = p[0]; // Tgl part @@ -84,8 +91,33 @@ class CalibdEdxCorrection void clear(); - void writeToFile(std::string_view fileName) const; - void loadFromFile(std::string_view fileName); + void writeToFile(std::string_view fileName, std::string_view objName = "ccdb_object") const; + void loadFromFile(std::string_view fileName, std::string_view objName = "ccdb_object"); + + /// \param outFileName name of the output file + void dumpToTree(const char* outFileName = "calib_dedx.root") const; + + /// Parameters averaged over all stacks + const std::array getMeanParams(ChargeType charge) const; + + /// Parameters averaged over all sectors for a stack type + const std::array getMeanParams(const GEMstack stack, ChargeType charge) const; + + /// Single fit parameters averaged over all sectors for a stack type + float getMeanParam(ChargeType charge, uint32_t param) const; + + /// Single fit parameters averaged over all sectors for a stack type + float getMeanParam(const GEMstack stack, ChargeType charge, uint32_t param) const; + + /// Single fit parameters averaged over all sectors for a stack type + float getMeanEntries(ChargeType charge) const; + + /// Single fit parameters averaged over all sectors for a stack type + float getMeanEntries(const GEMstack stack, ChargeType charge) const; + + /// set all corrections to 1, used for default initialization and to reset corrections + void setUnity(); + #endif private: diff --git a/DataFormats/Detectors/TPC/include/DataFormatsTPC/ClusterNative.h b/DataFormats/Detectors/TPC/include/DataFormatsTPC/ClusterNative.h index 49d61007092a8..f3070d456afb1 100644 --- a/DataFormats/Detectors/TPC/include/DataFormatsTPC/ClusterNative.h +++ b/DataFormats/Detectors/TPC/include/DataFormatsTPC/ClusterNative.h @@ -31,7 +31,7 @@ template class ConstMCTruthContainer; template class ConstMCTruthContainerView; -} +} // namespace dataformats } // namespace o2 namespace o2 @@ -76,7 +76,7 @@ struct ClusterNative { GPUd() static float unpackPad(uint16_t pad) { return float(pad) * (1.f / scalePadPacked); } GPUd() static float unpackTime(uint32_t time) { return float(time) * (1.f / scaleTimePacked); } - GPUdDefault() ClusterNative() CON_DEFAULT; + GPUdDefault() ClusterNative() = default; GPUd() ClusterNative(uint32_t time, uint8_t flags, uint16_t pad, uint8_t sigmaTime, uint8_t sigmaPad, uint16_t qmax, uint16_t qtot) : padPacked(pad), sigmaTimePacked(sigmaTime), sigmaPadPacked(sigmaPad), qMax(qmax), qTot(qtot) { setTimePackedFlags(time, flags); @@ -156,6 +156,17 @@ struct ClusterNative { return (this->getFlags() < rhs.getFlags()); } } + + GPUd() bool operator==(const ClusterNative& rhs) const + { + return this->getTimePacked() == rhs.getTimePacked() && + this->padPacked == rhs.padPacked && + this->sigmaTimePacked == rhs.sigmaTimePacked && + this->sigmaPadPacked == rhs.sigmaPadPacked && + this->qMax == rhs.qMax && + this->qTot == rhs.qTot && + this->getFlags() == rhs.getFlags(); + } }; // This is an index struct to access TPC clusters inside sectors and rows. It shall not own the data, but just point to diff --git a/DataFormats/Detectors/TPC/include/DataFormatsTPC/CompressedClusters.h b/DataFormats/Detectors/TPC/include/DataFormatsTPC/CompressedClusters.h index cbdc64fde9d3a..18ad5c6819344 100644 --- a/DataFormats/Detectors/TPC/include/DataFormatsTPC/CompressedClusters.h +++ b/DataFormats/Detectors/TPC/include/DataFormatsTPC/CompressedClusters.h @@ -29,9 +29,11 @@ struct CompressedClustersCounters { unsigned int nUnattachedClusters = 0; unsigned int nAttachedClustersReduced = 0; unsigned int nSliceRows = 36 * 152; - unsigned char nComppressionModes = 0; + unsigned char nComppressionModes = 0; // Don't fix this name due to ROOT dictionaries! + float solenoidBz = -1e6f; + int maxTimeBin = -1e6; - ClassDefNV(CompressedClustersCounters, 2); + ClassDefNV(CompressedClustersCounters, 3); }; template @@ -63,7 +65,7 @@ struct CompressedClustersPtrs_x { TSHORT nTrackClusters = 0; //! TINT nSliceRowClusters = 0; //! - ClassDefNV(CompressedClustersPtrs_x, 2); + ClassDefNV(CompressedClustersPtrs_x, 3); }; struct CompressedClustersPtrs : public CompressedClustersPtrs_x { @@ -74,28 +76,30 @@ struct CompressedClustersOffsets : public CompressedClustersPtrs_x #include #include +#include #include #include #include @@ -29,9 +30,13 @@ #include "Framework/Logger.h" #include "DataFormatsTPC/Defs.h" +#include "MathUtils/fit.h" using namespace o2::tpc; +class TLinearFitter; +class TTree; + namespace o2::tpc::dcs { @@ -50,6 +55,8 @@ struct DataPoint { bool equalTime(const DataPoint& other) const { return time == other.time; } bool operator<(const DataPoint& other) const { return time < other.time; } bool operator<(const TimeStampType timeStamp) const { return time < timeStamp; } + DataPoint operator+(const DataPoint& other) const { return DataPoint{(time + other.time) / TimeStampType{2}, value + other.value}; } + DataPoint operator/(const DataType denom) const { return DataPoint{time, value / denom}; } ClassDefNV(DataPoint, 1); }; @@ -63,6 +70,19 @@ struct DataPointVector { uint32_t sensorNumber{}; std::vector data; + /// \brief convert data points to a vector of pairs: pair.first -> data and pair.second -> time + auto getPairOfVector() const + { + std::pair, std::vector> pairs; + pairs.first.reserve(data.size()); + pairs.second.reserve(data.size()); + for (const auto& dp : data) { + pairs.first.emplace_back(dp.value); + pairs.second.emplace_back(dp.time); + } + return pairs; + } + void fill(const TimeStampType time, const T& value) { data.emplace_back(DPType{time, value}); } void fill(const DPType& dataPoint) { data.emplace_back(dataPoint); } @@ -83,6 +103,11 @@ struct DataPointVector { void clear() { data.clear(); } + void append(const DataPointVector& other) + { + data.insert(data.end(), other.data.begin(), other.data.end()); + } + /// return value at the last valid time stamp /// /// values are valid unitl the next time stamp @@ -90,7 +115,32 @@ struct DataPointVector { { const auto i = std::upper_bound(data.begin(), data.end(), DPType{timeStamp, {}}); return (i == data.begin()) ? (*i).value : (*(i - 1)).value; - }; + } + + /// calculate average value for `timeStamp`, extending the range by +- range elements + const T getAverageValueForTime(const TimeStampType timeStamp, const long range) const + { + return getAverageValueForTime(timeStamp, timeStamp, range); + } + + /// calculate average value between `from` and `unil`, extending the range by +- `range` elements + const std::pair getSumAndPoints(const TimeStampType from, const TimeStampType until, const long range) const + { + const auto iFrom = std::upper_bound(data.begin(), data.end(), DPType{from, {}}); + const auto iUntil = (from == until) ? iFrom : std::upper_bound(data.begin(), data.end(), DPType{until, {}}); + const auto distFrom = std::distance(data.begin(), iFrom); + const auto distUntil = std::distance(iUntil, data.end()); + const auto nFrom = std::min(distFrom, range + 1); + const auto nUntil = std::min(distUntil, range); + const auto nPoints = std::distance(iFrom - nFrom, iUntil + nUntil); + return {std::accumulate(iFrom - nFrom, iUntil + nUntil, DPType{(*(iFrom - nFrom)).time, T{}}).value, nPoints}; + } + + const T getAverageValueForTime(const TimeStampType from, const TimeStampType until, const long range) const + { + const auto sumAndPoints = getSumAndPoints(from, until, range); + return sumAndPoints.first / static_cast(sumAndPoints.second); + } ClassDefNV(DataPointVector, 1); }; @@ -111,6 +161,70 @@ void doClear(std::vector>& dataVector) } } +template +void doAppend(std::vector>& a, const std::vector>& b) +{ + if (a.size() != b.size()) { + LOGP(warning, "Trying to append std::vector>s of different size: {} != {}", a.size(), b.size()); + } + for (size_t i = 0; i < a.size(); ++i) { + a[i].append(b[i]); + } +} + +template +const T getAverageValueForTime(const std::vector>& dpVec, const TimeStampType from, const TimeStampType until, const long range) +{ + T ret{}; + long nPoints{}; + + for (const auto& dps : dpVec) { + const auto sumAndPoints = dps.getSumAndPoints(from, until, range); + ret += sumAndPoints.first; + nPoints += sumAndPoints.second; + } + return (nPoints > 0) ? ret / static_cast(nPoints) : T{}; +} + +template +dcs::TimeStampType getMinTime(const std::vector>& data, const bool roundToInterval, dcs::TimeStampType fitInterval) +{ + constexpr auto max = std::numeric_limits::max(); + dcs::TimeStampType firstTime = std::numeric_limits::max(); + for (const auto& sensor : data) { + const auto time = sensor.data.size() ? sensor.data.front().time : max; + firstTime = std::min(firstTime, time); + } + + // mFitInterval is is seconds. Round to full amount. + // if e.g. mFitInterval = 5min, then round 10:07:20.510 to 10:05:00.000 + if (roundToInterval) { + firstTime -= (firstTime % fitInterval); + } + + return firstTime; +} + +template +dcs::TimeStampType getMaxTime(const std::vector>& data) +{ + constexpr auto min = 0; + dcs::TimeStampType lastTime = 0; + for (const auto& sensor : data) { + const auto time = sensor.data.size() ? sensor.data.back().time : 0; + lastTime = std::max(lastTime, time); + } + + // mFitInterval is is seconds. Round to full amount. + // if e.g. mFitInterval = 5min, then round 10:07:20.510 to 10:05:00.000 + // TODO: fix this + // if (mRoundToInterval) { + // lastTime -= (lastTime % mFitInterval); + //} + + return lastTime; +} + using RawDPsF = DataPointVector; // using RawDPsI = DataPointVector; @@ -152,11 +266,20 @@ struct Temperature { static constexpr auto& getSensorPosition(const size_t sensor) { return SensorPosition[sensor]; } + /// \brief make fit of the mean temperature and gradients in time intervals + /// \param Side TPC side for which to make the fit + /// \param fitInterval time interval for the fits + /// \param roundToInterval round min time + void fitTemperature(Side side, dcs::TimeStampType fitInterval = 5 * 60 * 1000, const bool roundToInterval = false); + struct Stats { DataType mean{}; ///< average temperature in K DataType gradX{}; ///< horizontal temperature gradient in K/cm DataType gradY{}; ///< vertical temperature gradient in K/cm + Stats operator+(const Stats& other) const { return Stats{mean + other.mean, gradX + other.gradX, gradY + other.gradY}; } + Stats operator/(const DataType val) const { return Stats{mean / val, gradX / val, gradY / val}; } + ClassDefNV(Stats, 1); }; using StatsDPs = DataPointVector; @@ -170,6 +293,11 @@ struct Temperature { return (s == Side::A) ? statsA.getValueForTime(timeStamp) : statsC.getValueForTime(timeStamp); } + DataType getMeanTempRaw() + { + return getAverageValueForTime(raw, 0, 9999999999999, 0); + } + void fill(std::string_view sensor, const TimeStampType time, const DataType temperature) { raw[SensorNameMap.at(sensor.data())].fill(time, temperature); @@ -177,6 +305,8 @@ struct Temperature { void sortAndClean() { + statsA.sortAndClean(); + statsC.sortAndClean(); doSortAndClean(raw); } @@ -187,6 +317,16 @@ struct Temperature { statsC.clear(); } + void append(const Temperature& other) + { + statsA.append(other.statsA); + statsC.append(other.statsC); + doAppend(raw, other.raw); + } + + private: + bool makeFit(TLinearFitter& fitter, const int nDim, std::vector& xVals, std::vector& temperatures); + ClassDefNV(Temperature, 1); }; @@ -195,8 +335,7 @@ struct Temperature { /// struct HV { - HV() - noexcept; + HV() noexcept; // Exmple strings // TPC_HV_A03_I_G1B_I @@ -295,6 +434,13 @@ struct HV { doClear(states); } + void append(const HV& other) + { + doAppend(voltages, other.voltages); + doAppend(currents, other.currents); + doAppend(states, other.states); + } + ClassDefNV(HV, 1); }; @@ -382,6 +528,18 @@ struct Gas { o2Sensor.clear(); } + void append(const Gas& other) + { + neon.append(other.neon); + co2.append(other.co2); + n2.append(other.n2); + argon.append(other.argon); + h2o.append(other.h2o); + o2.append(other.o2); + h2oSensor.append(other.h2oSensor); + o2Sensor.append(other.o2Sensor); + } + TimeStampType getMinTime() const; TimeStampType getMaxTime() const; @@ -389,5 +547,77 @@ struct Gas { ClassDefNV(Gas, 1); }; +struct RobustPressure { + using Stats = o2::math_utils::RollingStats; + Stats surfaceAtmosPressure; ///< rolling statistics of surface sensor + Stats cavernAtmosPressure; ///< rolling statistics of cavern sensor 1 + Stats cavernAtmosPressure2; ///< rolling statistics of cavern sensor 2 + Stats cavernAtmosPressure12; ///< rolling statistics of cavernAtmosPressure/cavernAtmosPressure2 + Stats cavernAtmosPressure1S; ///< rolling statistics of cavernAtmosPressure/surfaceAtmosPressure + Stats cavernAtmosPressure2S; ///< rolling statistics of cavernAtmosPressure2/surfaceAtmosPressure + std::vector isOk; ///< bit mask of valid sensors: cavernBit 0, cavern2Bit = 1, surfaceBit = 2 + std::vector robustPressure; ///< combined robust pressure value that should be used + std::vector time; ///< time stamps of all pressure values + TimeStampType timeInterval; ///< time interval used for rolling statistics + TimeStampType timeIntervalRef; ///< reference time interval used for normalization of pressure sensors + float maxDist{}; ///< maximum allowed time distance between sensors to be accepted for robust pressure calculation + float maxDiff{0.2f}; ///< maximum allowed pressure difference between sensors to be accepted for robust pressure calculation + + ClassDefNV(RobustPressure, 2); +}; + +struct Pressure { + + /// \brief fill pressure data + /// \param sensor name of the sensor from DCS data stream + /// \param time measurement time + /// \param value pressure value + void fill(std::string_view sensor, const TimeStampType time, const DataType value); + + /// sort pressure values and remove obvious outliers + /// \param pMin min accepted pressure + /// \param pMax max accepted pressure + void sortAndClean(float pMin = 800, float pMax = 1100); + + /// \clear all stored data except the buffer + void clear(); + + /// append other pressure values + void append(const Pressure& other); + + /// \return get minimum time of stored data + TimeStampType getMinTime() const; + + /// \return get maximum time of stored data + TimeStampType getMaxTime() const; + + /// \brief average pressure values for given time interval + /// \param timeInterval time interval for which the pressure values are averaged + /// \param timeIntervalRef time interval used to calculate the normalization values for the pressure + /// \param tStart min time of the data + /// \param tEnd max time of the data + /// \param nthreads numbe rof threads used for some calculations + void makeRobustPressure(TimeStampType timeInterval = 100 * 1000, TimeStampType timeIntervalRef = 24 * 60 * 1000, TimeStampType tStart = 1, TimeStampType tEnd = 0, const int nthreads = 1); + + /// set aliases for the cuts used in the calculation of the robust pressure + static void setAliases(TTree* tree); + + RawDPsF cavernAtmosPressure{}; ///< raw pressure in the cavern from sensor 1 + RawDPsF cavernAtmosPressure2{}; ///< raw pressure in the cavern from sensor 2 + RawDPsF surfaceAtmosPressure{}; ///< raw pressure at the surface + RobustPressure robustPressure{}; ///< combined robust pressure estimator from all three sensors + + std::pair, std::vector> mCavernAtmosPressure1Buff{}; ///, std::vector> mCavernAtmosPressure2Buff{}; ///, std::vector> mSurfaceAtmosPressureBuff{}; ///, std::vector> mPressure12Buff{}; ///, std::vector> mPressure1SBuff{}; ///, std::vector> mPressure2SBuff{}; ///, std::vector> mRobPressureBuff{}; /// +#include #include namespace o2::tpc @@ -30,16 +30,55 @@ struct LtrCalibData { uint64_t firstTime{}; ///< first time stamp of processed TFs uint64_t lastTime{}; ///< last time stamp of processed TFs long creationTime{}; ///< time of creation - float dvCorrectionA{}; ///< drift velocity correction factor A-Side (inverse multiplicative) - float dvCorrectionC{}; ///< drift velocity correction factor C-Side (inverse multiplicative) + float dvCorrectionA{1.f}; ///< drift velocity correction factor A-Side (inverse multiplicative) + float dvCorrectionC{1.f}; ///< drift velocity correction factor C-Side (inverse multiplicative) float dvOffsetA{}; ///< drift velocity trigger offset A-Side float dvOffsetC{}; ///< drift velocity trigger offset C-Side float refVDrift{}; ///< reference vdrift for which factor was extracted + float refTimeOffset{0.}; ///< additive time offset reference (\mus) + float timeOffsetCorr{0.}; ///< additive time offset correction (\mus) uint16_t nTracksA{}; ///< number of tracks used for A-Side fit uint16_t nTracksC{}; ///< number of tracks used for C-Side fit - std::vector matchedLtrIDs; ///< list of matched laser track IDs + std::vector matchedLtrIDs; ///< matched laser track IDs + std::vector nTrackTF; ///< number of laser tracks per TF + std::vector dEdx; ///< dE/dx of each track + float tp{0.f}; ///< temperature over pressure ratio - float getDriftVCorrection() const { return 0.5f * (dvCorrectionA + dvCorrectionC); } + bool isValid() const + { + return (std::abs(dvCorrectionA - 1.f) < 0.2) || (std::abs(dvCorrectionC - 1.f) < 0.2); + } + + float getDriftVCorrection() const + { + float correction = 0; + int nCorr = 0; + // only allow +- 20% around reference correction + if (std::abs(dvCorrectionA - 1.f) < 0.2) { + correction += dvCorrectionA; + ++nCorr; + } else { + LOGP(warning, "abs(dvCorrectionA ({}) - 1) >= 0.2, not using for combined estimate", dvCorrectionA); + } + + if (std::abs(dvCorrectionC - 1.f) < 0.2) { + correction += dvCorrectionC; + ++nCorr; + } else { + LOGP(warning, "abs(dvCorrectionC ({}) - 1) >= 0.2, not using for combined estimate", dvCorrectionC); + } + + if (nCorr == 0) { + LOGP(error, "no valid drift velocity correction"); + return 1.f; + } + + return correction / nCorr; + } + + float getVDrift() const { return refVDrift / getDriftVCorrection(); } + + float getTimeOffset() const { return refTimeOffset + timeOffsetCorr; } // renormalize reference and correction either to provided new reference (if >0) or to correction 1 wrt current reference void normalize(float newVRef = 0.f) @@ -61,6 +100,25 @@ struct LtrCalibData { dvCorrectionC *= fact; } + // similarly, the time offset reference is set to provided newRefTimeOffset (if > -998) or modified to have timeOffsetCorr to + // be 0 otherwise + + void normalizeOffset(float newRefTimeOffset = -999.) + { + if (newRefTimeOffset > -999.) { + timeOffsetCorr = getTimeOffset() - newRefTimeOffset; + refTimeOffset = newRefTimeOffset; + } else { + refTimeOffset = getTimeOffset(); + timeOffsetCorr = 0.; + } + } + + float getT0A() const { return (250.f * (1.f - dvCorrectionA) - dvOffsetA) / refVDrift; } + float getT0C() const { return (250.f * (1.f - dvCorrectionC) + dvOffsetC) / refVDrift; } + float getZOffsetA() const { return (250.f * (1.f - dvCorrectionA) - dvOffsetA) / dvCorrectionA; } + float getZOffsetC() const { return (250.f * (1.f - dvCorrectionC) + dvOffsetC) / dvCorrectionC; } + void reset() { processedTFs = 0; @@ -74,10 +132,14 @@ struct LtrCalibData { nTracksA = 0; nTracksC = 0; refVDrift = 0; + refTimeOffset = 0; + timeOffsetCorr = 0; matchedLtrIDs.clear(); + nTrackTF.clear(); + dEdx.clear(); } - ClassDefNV(LtrCalibData, 2); + ClassDefNV(LtrCalibData, 5); }; } // namespace o2::tpc diff --git a/DataFormats/Detectors/TPC/include/DataFormatsTPC/PIDResponse.h b/DataFormats/Detectors/TPC/include/DataFormatsTPC/PIDResponse.h new file mode 100644 index 0000000000000..277011a260631 --- /dev/null +++ b/DataFormats/Detectors/TPC/include/DataFormatsTPC/PIDResponse.h @@ -0,0 +1,145 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// +/// @file PIDResponse.h +/// @author Tuba Gündem, tuba.gundem@cern.ch +/// + +#ifndef AliceO2_TPC_PIDResponse_H +#define AliceO2_TPC_PIDResponse_H + +// o2 includes +#include "GPUCommonDef.h" +#include "GPUCommonRtypes.h" +#include "GPUCommonMath.h" +#include "ReconstructionDataFormats/PID.h" +#include "DataFormatsTPC/PIDResponse.h" +#include "DataFormatsTPC/BetheBlochAleph.h" +#include "DataFormatsTPC/TrackTPC.h" + +namespace o2::tpc +{ +class TrackTPC; + +/// \brief PID response class +/// +/// This class is used to handle the TPC PID response. +/// + +class PIDResponse +{ + public: + /// default constructor + PIDResponse() = default; + + /// default destructor + ~PIDResponse() = default; + + /// setters + GPUd() void setBetheBlochParams(const float betheBlochParams[5]); + GPUd() void setMIP(float mip) { mMIP = mip; } + GPUd() void setChargeFactor(float chargeFactor) { mChargeFactor = chargeFactor; } + + /// getters + GPUd() const float* getBetheBlochParams() const { return mBetheBlochParams; } + GPUd() float getMIP() const { return mMIP; } + GPUd() float getChargeFactor() const { return mChargeFactor; } + + /// get expected signal of the track + GPUd() float getExpectedSignal(const TrackTPC& track, const o2::track::PID::ID id) const; + + /// get most probable PID of the track + GPUd() o2::track::PID::ID getMostProbablePID(const TrackTPC& track, float PID_EKrangeMin, float PID_EKrangeMax, float PID_EPrangeMin, float PID_EPrangeMax, float PID_EDrangeMin, float PID_EDrangeMax, float PID_ETrangeMin, float PID_ETrangeMax, char PID_useNsigma, float PID_sigma) const; + + private: + float mBetheBlochParams[5] = {0.19310481, 4.26696118, 0.00522579, 2.38124907, 0.98055396}; // BBAleph average fit parameters + float mMIP = 50.f; + float mChargeFactor = 2.299999952316284f; + + ClassDefNV(PIDResponse, 1); +}; + +GPUd() void PIDResponse::setBetheBlochParams(const float betheBlochParams[5]) +{ + for (int i = 0; i < 5; i++) { + mBetheBlochParams[i] = betheBlochParams[i]; + } +} + +GPUd() float PIDResponse::getExpectedSignal(const TrackTPC& track, const o2::track::PID::ID id) const +{ + const float bg = static_cast(track.getP() / o2::track::pid_constants::sMasses[id]); + if (bg < 0.05) { + return -999.; + } + const float bethe = mMIP * o2::tpc::BetheBlochAleph(bg, mBetheBlochParams[0], mBetheBlochParams[1], mBetheBlochParams[2], mBetheBlochParams[3], mBetheBlochParams[4]) * o2::gpu::GPUCommonMath::Pow(static_cast(o2::track::pid_constants::sCharges[id]), mChargeFactor); + return bethe >= 0. ? bethe : -999.; +} + +// get most probable PID +GPUd() o2::track::PID::ID PIDResponse::getMostProbablePID(const TrackTPC& track, float PID_EKrangeMin, float PID_EKrangeMax, float PID_EPrangeMin, float PID_EPrangeMax, float PID_EDrangeMin, float PID_EDrangeMax, float PID_ETrangeMin, float PID_ETrangeMax, char PID_useNsigma, float PID_sigma) const +{ + const float dEdx = track.getdEdx().dEdxTotTPC; + + if (dEdx < 10.) { + return o2::track::PID::Pion; + } + + auto id = o2::track::PID::Electron; + float distanceMin = 0.; + float dEdxExpected = getExpectedSignal(track, id); + if (PID_useNsigma) { + // using nSigma + distanceMin = o2::gpu::GPUCommonMath::Abs((dEdx - dEdxExpected) / (PID_sigma * dEdxExpected)); + } else { + // using absolute distance + distanceMin = o2::gpu::GPUCommonMath::Abs(dEdx - dEdxExpected); + } + + // calculate the distance to the expected dEdx signals + // start from Pion to exlude Muons + for (o2::track::PID::ID i = o2::track::PID::Pion; i < o2::track::PID::NIDs; i++) { + float distance = 0.; + dEdxExpected = getExpectedSignal(track, i); + if (PID_useNsigma) { + // using nSigma + distance = o2::gpu::GPUCommonMath::Abs((dEdx - dEdxExpected) / (PID_sigma * dEdxExpected)); + } else { + // using absolute distance + distance = o2::gpu::GPUCommonMath::Abs(dEdx - dEdxExpected); + } + if (distance < distanceMin) { + id = i; + distanceMin = distance; + } + } + + // override the electrons to the closest alternative PID in the bands crossing regions + if (id == o2::track::PID::Electron) { + const float p = track.getP(); + if ((p > PID_EKrangeMin) && (p < PID_EKrangeMax)) { + id = o2::track::PID::Kaon; + } else if ((p > PID_EPrangeMin) && (p < PID_EPrangeMax)) { + id = o2::track::PID::Proton; + } else if ((p > PID_EDrangeMin) && (p < PID_EDrangeMax)) { + id = o2::track::PID::Deuteron; + } else if ((p > PID_ETrangeMin) && (p < PID_ETrangeMax)) { + id = o2::track::PID::Triton; + } + } + + return id; +} + +} // namespace o2::tpc + +#endif diff --git a/DataFormats/Detectors/TPC/include/DataFormatsTPC/RawDataTypes.h b/DataFormats/Detectors/TPC/include/DataFormatsTPC/RawDataTypes.h index 481a7739e54fb..db96280bde534 100644 --- a/DataFormats/Detectors/TPC/include/DataFormatsTPC/RawDataTypes.h +++ b/DataFormats/Detectors/TPC/include/DataFormatsTPC/RawDataTypes.h @@ -25,9 +25,10 @@ namespace o2::tpc::raw_data_types enum Type : char { RAWDATA = 0, ///< GBT raw data LinkZS = 1, ///< Link-based Zero Suppression - ZS = 2, ///< final Zero Suppression + ZS = 2, ///< final Zero Suppression (can be ILBZS, DLBZS) IDC = 3, ///< Integrated Digitial Currents, with priority bit to end up in separate buffer SAC = 4, ///< Sampled Analogue Currents from the current monitor + CMV = 5, ///< Common mode values }; const std::unordered_map TypeNameMap{ @@ -36,6 +37,7 @@ const std::unordered_map TypeNameMap{ {Type::ZS, "ZS"}, {Type::IDC, "IDC"}, {Type::SAC, "SAC"}, + {Type::CMV, "CMV"}, }; } // namespace o2::tpc::raw_data_types diff --git a/DataFormats/Detectors/TPC/include/DataFormatsTPC/SAC.h b/DataFormats/Detectors/TPC/include/DataFormatsTPC/SAC.h index 53cbb66435ee6..416522185dd7d 100644 --- a/DataFormats/Detectors/TPC/include/DataFormatsTPC/SAC.h +++ b/DataFormats/Detectors/TPC/include/DataFormatsTPC/SAC.h @@ -81,7 +81,7 @@ struct dataDef { }; /// /// uint32_t pktNumber = 0; ///< packet number of this front end card. Should always increase by 1 - uint32_t timeStamp = 0; ///< tims stamp + uint32_t timeStamp = 0; ///< time stamp char dataWords[DataSize]; ///< ASCI encoded SAC data uint32_t crc32 = 0; ///< CRC32 checksum uint32_t trailer = 0; ///< trailer magic word, should always be TrailerWord diff --git a/DataFormats/Detectors/TPC/include/DataFormatsTPC/TrackCuts.h b/DataFormats/Detectors/TPC/include/DataFormatsTPC/TrackCuts.h index f8bcc601cf659..451acfe9d1fa0 100644 --- a/DataFormats/Detectors/TPC/include/DataFormatsTPC/TrackCuts.h +++ b/DataFormats/Detectors/TPC/include/DataFormatsTPC/TrackCuts.h @@ -46,13 +46,16 @@ class TrackCuts void setNClusMin(float NClusMin) { mNClusMin = NClusMin; } void setdEdxMin(float dEdxMin) { mdEdxMin = dEdxMin; } void setdEdxMax(float dEdxMax) { mdEdxMax = dEdxMax; } + /// try to remove looper cutting on (abs(z_out) - abs(z_in)) < -10), not very precise + void setCutLooper(bool cut) { mCutLooper = cut; } private: - float mPMin{0}; ///< min momentum allowed - float mPMax{1e10}; ///< max momentum allowed - float mNClusMin{0}; ///< min number of clusters in track allowed - float mdEdxMin{0}; ///< min dEdx - float mdEdxMax{1e10}; ///< max dEdx + float mPMin{0}; ///< min momentum allowed + float mPMax{1e10}; ///< max momentum allowed + float mNClusMin{0}; ///< min number of clusters in track allowed + float mdEdxMin{0}; ///< min dEdx + float mdEdxMax{1e10}; ///< max dEdx + bool mCutLooper{false}; ///< cut looper comparing zout-zin ClassDefNV(TrackCuts, 1) }; diff --git a/DataFormats/Detectors/TPC/include/DataFormatsTPC/TrackTPC.h b/DataFormats/Detectors/TPC/include/DataFormatsTPC/TrackTPC.h index adc3ff45b30eb..e7e23dea91e88 100644 --- a/DataFormats/Detectors/TPC/include/DataFormatsTPC/TrackTPC.h +++ b/DataFormats/Detectors/TPC/include/DataFormatsTPC/TrackTPC.h @@ -67,6 +67,7 @@ class TrackTPC : public o2::track::TrackParCov GPUd() float getChi2() const { return mChi2; } GPUd() const o2::track::TrackParCov& getOuterParam() const { return mOuterParam; } GPUd() const o2::track::TrackParCov& getParamOut() const { return mOuterParam; } // to have method with same name as other tracks + GPUd() o2::track::TrackParCov& getParamOut() { return mOuterParam; } // to have method with same name as other tracks GPUd() void setTime0(float v) { mTime0 = v; } GPUd() void setChi2(float v) { mChi2 = v; } GPUd() void setOuterParam(o2::track::TrackParCov&& v) { mOuterParam = v; } @@ -78,8 +79,8 @@ class TrackTPC : public o2::track::TrackParCov GPUd() void setClusterRef(uint32_t entry, uint16_t ncl) { mClustersReference.set(entry, ncl); } template - GPUd() static inline void getClusterReference(T& clinfo, int nCluster, - uint8_t& sectorIndex, uint8_t& rowIndex, uint32_t& clusterIndex, const ClusRef& ref) + GPUdi() static void getClusterReference(T& clinfo, int nCluster, + uint8_t& sectorIndex, uint8_t& rowIndex, uint32_t& clusterIndex, const ClusRef& ref) { // data for given tracks starts at clinfo[ ref.getFirstEntry() ], // 1st ref.getEntries() cluster indices are stored as uint32_t @@ -94,15 +95,15 @@ class TrackTPC : public o2::track::TrackParCov } template - GPUd() inline void getClusterReference(T& clinfo, int nCluster, - uint8_t& sectorIndex, uint8_t& rowIndex, uint32_t& clusterIndex) const + GPUdi() void getClusterReference(T& clinfo, int nCluster, + uint8_t& sectorIndex, uint8_t& rowIndex, uint32_t& clusterIndex) const { getClusterReference(clinfo, nCluster, sectorIndex, rowIndex, clusterIndex, mClustersReference); } template - GPUd() static inline const o2::tpc::ClusterNative& getCluster(T& clinfo, int nCluster, - const o2::tpc::ClusterNativeAccess& clusters, uint8_t& sectorIndex, uint8_t& rowIndex, const ClusRef& ref) + GPUdi() static const o2::tpc::ClusterNative& getCluster(T& clinfo, int nCluster, + const o2::tpc::ClusterNativeAccess& clusters, uint8_t& sectorIndex, uint8_t& rowIndex, const ClusRef& ref) { uint32_t clusterIndex; getClusterReference(clinfo, nCluster, sectorIndex, rowIndex, clusterIndex, ref); @@ -110,15 +111,15 @@ class TrackTPC : public o2::track::TrackParCov } template - GPUd() inline const o2::tpc::ClusterNative& getCluster(T& clinfo, int nCluster, - const o2::tpc::ClusterNativeAccess& clusters, uint8_t& sectorIndex, uint8_t& rowIndex) const + GPUdi() const o2::tpc::ClusterNative& getCluster(T& clinfo, int nCluster, + const o2::tpc::ClusterNativeAccess& clusters, uint8_t& sectorIndex, uint8_t& rowIndex) const { return getCluster(clinfo, nCluster, clusters, sectorIndex, rowIndex, mClustersReference); } template - GPUd() inline const o2::tpc::ClusterNative& getCluster(T& clinfo, int nCluster, - const o2::tpc::ClusterNativeAccess& clusters) const + GPUdi() const o2::tpc::ClusterNative& getCluster(T& clinfo, int nCluster, + const o2::tpc::ClusterNativeAccess& clusters) const { uint8_t sectorIndex, rowIndex; return (getCluster(clinfo, nCluster, clusters, sectorIndex, rowIndex)); @@ -127,6 +128,9 @@ class TrackTPC : public o2::track::TrackParCov GPUd() const dEdxInfo& getdEdx() const { return mdEdx; } GPUd() void setdEdx(const dEdxInfo& v) { mdEdx = v; } + GPUd() const dEdxInfo& getdEdxAlt() const { return mdEdxAlt; } + GPUd() void setdEdxAlt(const dEdxInfo& v) { mdEdxAlt = v; } + private: float mTime0 = 0.f; ///< Assumed time of the vertex that created the track in TPC time bins, 0 for triggered data float mDeltaTFwd = 0; ///< max possible increment to mTime0 @@ -135,9 +139,10 @@ class TrackTPC : public o2::track::TrackParCov float mChi2 = 0.f; // Chi2 of the track o2::track::TrackParCov mOuterParam; // Track parameters at outer end of TPC. dEdxInfo mdEdx; // dEdx Information + dEdxInfo mdEdxAlt; // dEdx alternative Information ClusRef mClustersReference; // reference to externale cluster indices - ClassDefNV(TrackTPC, 4); + ClassDefNV(TrackTPC, 5); }; } // namespace tpc diff --git a/DataFormats/Detectors/TPC/include/DataFormatsTPC/VDriftCorrFact.h b/DataFormats/Detectors/TPC/include/DataFormatsTPC/VDriftCorrFact.h index 09e0968e8c734..a20c37e9b2cee 100644 --- a/DataFormats/Detectors/TPC/include/DataFormatsTPC/VDriftCorrFact.h +++ b/DataFormats/Detectors/TPC/include/DataFormatsTPC/VDriftCorrFact.h @@ -20,34 +20,61 @@ #define AliceO2_TPC_VDRIFT_CORRFACT_H #include "GPUCommonRtypes.h" +#include "CommonConstants/LHCConstants.h" namespace o2::tpc { struct VDriftCorrFact { - long firstTime{}; ///< first time stamp of processed TFs - long lastTime{}; ///< last time stamp of processed TFs - long creationTime{}; ///< time of creation - float corrFact{1.0}; ///< drift velocity correction factor (multiplicative) - float corrFactErr{0.0}; ///< stat error of correction factor - float refVDrift{0.}; ///< reference vdrift for which factor was extracted + long firstTime{}; ///< first time stamp of processed TFs + long lastTime{}; ///< last time stamp of processed TFs + long creationTime{}; ///< time of creation + float corrFact{1.0}; ///< drift velocity correction factor (multiplicative) + float corrFactErr{0.0}; ///< stat error of correction factor + float refVDrift{0.}; ///< reference vdrift for which factor was extracted + float refTimeOffset{0.}; ///< additive time offset reference (\mus) + float timeOffsetCorr{0.}; ///< additive time offset correction (\mus) + float refTP{0.}; ///< reference temperature / pressure for which refVDrift was extracted float getVDrift() const { return refVDrift * corrFact; } float getVDriftError() const { return refVDrift * corrFactErr; } - // renormalize reference and correction either to provided new reference (if >0) or to correction 1 wrt current reference - void normalize(float newVRef = 0.f) + float getTimeOffset() const { return refTimeOffset + timeOffsetCorr; } + + // renormalize VDrift reference and correction either to provided new reference (if >0) or to correction 1 wrt current reference + void normalize(float newVRef = 0.f, float tp = 0.f) { + float normVDrift = newVRef; if (newVRef == 0.f) { - newVRef = refVDrift * corrFact; + normVDrift = refVDrift * corrFact; + newVRef = normVDrift; + if ((tp > 0) && (refTP > 0)) { + // linear scaling based on relative change of T/P + normVDrift *= refTP / tp; + refTP = tp; // update reference T/P + } } - float fact = refVDrift / newVRef; + float fact = refVDrift / normVDrift; refVDrift = newVRef; corrFactErr *= fact; corrFact *= fact; } - ClassDefNV(VDriftCorrFact, 1); + // similarly, the time offset reference is set to provided newRefTimeOffset (if > -998) or modified to have timeOffsetCorr to + // be 0 otherwise + + void normalizeOffset(float newRefTimeOffset = -999.) + { + if (newRefTimeOffset > -999.) { + timeOffsetCorr = getTimeOffset() - newRefTimeOffset; + refTimeOffset = newRefTimeOffset; + } else { + refTimeOffset = getTimeOffset(); + timeOffsetCorr = 0.; + } + } + + ClassDefNV(VDriftCorrFact, 3); }; } // namespace o2::tpc diff --git a/DataFormats/Detectors/TPC/include/DataFormatsTPC/WorkflowHelper.h b/DataFormats/Detectors/TPC/include/DataFormatsTPC/WorkflowHelper.h index 30b40ed70b9c7..f4a318bc30101 100644 --- a/DataFormats/Detectors/TPC/include/DataFormatsTPC/WorkflowHelper.h +++ b/DataFormats/Detectors/TPC/include/DataFormatsTPC/WorkflowHelper.h @@ -21,6 +21,7 @@ #include "Framework/DataRefUtils.h" #include #include "Framework/InputRecordWalker.h" +#include "DataSampling/DataSamplingHeader.h" #include "DataFormatsTPC/TrackTPC.h" #include "SimulationDataFormat/MCCompLabel.h" #include "SimulationDataFormat/MCTruthContainer.h" @@ -80,7 +81,7 @@ static auto getWorkflowTPCInput(o2::framework::ProcessingContext& pc, int verbos if (do_digits) { std::fill(inputDigitsMCIndex.begin(), inputDigitsMCIndex.end(), -1); } - for (auto const& ref : o2::framework::InputRecordWalker(pc.inputs(), filter)) { + for (auto const& ref : o2::framework::InputRecordWalker(pc.inputs(), filter)) { auto const* sectorHeader = o2::framework::DataRefUtils::getHeader(ref); if (sectorHeader == nullptr) { // FIXME: think about error policy @@ -127,7 +128,7 @@ static auto getWorkflowTPCInput(o2::framework::ProcessingContext& pc, int verbos {"check", o2::framework::ConcreteDataTypeMatcher{o2::header::gDataOriginTPC, "CLUSTERNATIVE"}, o2::framework::Lifetime::Timeframe}, }; unsigned long recvMask = 0; - for (auto const& ref : o2::framework::InputRecordWalker(pc.inputs(), filter)) { + for (auto const& ref : o2::framework::InputRecordWalker(pc.inputs(), filter)) { auto const* sectorHeader = o2::framework::DataRefUtils::getHeader(ref); if (sectorHeader == nullptr) { throw std::runtime_error("sector header missing on header stack"); diff --git a/DataFormats/Detectors/TPC/include/DataFormatsTPC/ZeroSuppression.h b/DataFormats/Detectors/TPC/include/DataFormatsTPC/ZeroSuppression.h index 70c6cf60e166d..b1df9445bcf42 100644 --- a/DataFormats/Detectors/TPC/include/DataFormatsTPC/ZeroSuppression.h +++ b/DataFormats/Detectors/TPC/include/DataFormatsTPC/ZeroSuppression.h @@ -19,6 +19,7 @@ #include // for size_t #endif #include "GPUCommonDef.h" +#include "GPUCommonRtypes.h" namespace o2 { @@ -29,6 +30,8 @@ enum ZSVersion : unsigned char { ZSVersionRowBased10BitADC = 1, ZSVersionRowBased12BitADC = 2, ZSVersionLinkBasedWithMeta = 3, + ZSVersionDenseLinkBased = 4, + ZSVersionDenseLinkBasedV2 = 5 }; struct TPCZSHDR { @@ -43,20 +46,28 @@ struct TPCZSHDR { // 1: original row-based format with 10-bit ADC values // 2: original row-based format with 12-bit ADC values // 3: improved link-based format with extra META header - // 4: tightly-packed link based - unsigned char nTimeBins; // Number of time bins in this raw page + // 4: dense link based + unsigned char nTimeBinSpan; // Span of time bins in this raw page, i.e. last timeBin is <= timeOffset + nTimeBinSpan unsigned short cruID; // CRU id unsigned short timeOffset; // Time offset in BC after orbit in RDH unsigned short nADCsamples; // Total number of ADC samples in this raw page }; struct TPCZSHDRV2 : public TPCZSHDR { - static constexpr unsigned int TPC_ZS_NBITS_V3 = 12; + static constexpr unsigned int TPC_ZS_NBITS_V34 = 12; static constexpr bool TIGHTLY_PACKED_V3 = false; - static constexpr unsigned int SAMPLESPER64BIT = 64 / TPC_ZS_NBITS_V3; // 5 12-bit samples with 4 bit padding per 64 bit word for non-TIGHTLY_PACKED data + static constexpr unsigned int SAMPLESPER64BIT = 64 / TPC_ZS_NBITS_V34; // 5 12-bit samples with 4 bit padding per 64 bit word for non-TIGHTLY_PACKED data + static constexpr unsigned int TRIGGER_WORD_SIZE = 16; // trigger word size in bytes + enum ZSFlags : unsigned char { + TriggerWordPresent = 1, + nTimeBinSpanBit8 = 2, + payloadExtendsToNextPage = 4 + }; - unsigned short firstZSDataOffset; // Offset (after the TPCZSHDRV2 header) in 128bit words to first ZS data (in between can be trigger words, etc.) + unsigned short firstZSDataOffset; // zs Version 3: Offset (after the TPCZSHDRV2 header) in 128bit words to first ZS data (in between can be trigger words, etc.) + // zs Version >=4: Offset (from beginning of page) in bytes of the first ZS data. unsigned short nTimebinHeaders; // Number of timebin headers - unsigned short reserved1; // 16 reserved bits, header is 128 bit + unsigned char flags; // flag field (zs version 4 only): 0 = triggerWordPresent, 1 = bit 8 of nTimeBinSpan (i.e. nTimeBinSpan += 256) + unsigned char reserved1; // 16 reserved bits, header is 128 bit unsigned char reserved2; // 8 reserved bits, header is 128 bit unsigned char magicWord; // Magic word }; @@ -74,6 +85,35 @@ struct ZeroSuppressedContainer { // Struct for the TPC zero suppressed data form TPCZSHDR hdr; // ZS header }; +/// Trigger word for dense link-base ZS +/// +/// Trigger word is always 128bit and occurs always in the last page of a HBF before the meta header +struct TriggerWordDLBZS { + static constexpr uint16_t MaxTriggerEntries = 8; ///< Maximum number of trigger information + + /// trigger types as in the ttype bits + enum TriggerType : uint8_t { + PhT = 1, ///< Physics Trigger + PP = 2, ///< Pre Pulse for calibration + Cal = 4, ///< Laser (Calibration trigger) + }; + uint16_t triggerEntries[MaxTriggerEntries] = {}; + + uint16_t getTriggerBC(int entry = 0) const { return triggerEntries[entry] & 0xFFF; } + uint16_t getTriggerType(int entry = 0) const { return (triggerEntries[entry] >> 12) & 0x7; } + bool isValid(int entry = 0) const { return triggerEntries[entry] & 0x8000; } + + ClassDefNV(TriggerWordDLBZS, 1); +}; + +/// Trigger info including the orbit +struct TriggerInfoDLBZS { + TriggerWordDLBZS triggerWord{}; ///< trigger Word information + uint32_t orbit{}; ///< orbit of the trigger word + + ClassDefNV(TriggerInfoDLBZS, 1); +}; + } // namespace tpc } // namespace o2 #endif diff --git a/DataFormats/Detectors/TPC/include/DataFormatsTPC/ZeroSuppressionLinkBased.h b/DataFormats/Detectors/TPC/include/DataFormatsTPC/ZeroSuppressionLinkBased.h index 996b47d1830c1..a753f24aec11f 100644 --- a/DataFormats/Detectors/TPC/include/DataFormatsTPC/ZeroSuppressionLinkBased.h +++ b/DataFormats/Detectors/TPC/include/DataFormatsTPC/ZeroSuppressionLinkBased.h @@ -19,6 +19,7 @@ #include "GPUCommonDef.h" #ifndef GPUCA_GPUCODE +#include #include #endif @@ -54,7 +55,8 @@ struct CommonHeader { uint32_t bunchCrossing : 12; ///< bunch crossing number uint32_t numWordsPayload : 4; ///< number of 128bit words with 12bit ADC values uint32_t syncOffsetBC : 8; ///< sync offset in bunch crossings - uint32_t fecInPartition : 16; ///< fecInPartition, only used in improved link-based format + uint32_t fecInPartition : 8; ///< fecInPartition, only used in improved link-based format + uint32_t CMC : 8; ///< CMC, only used in improved link-based format uint32_t magicWord : 8; ///< magic word, identifies package }; }; @@ -310,7 +312,7 @@ struct TriggerInfoV3 { uint64_t word0 = 0; uint64_t word1 = 0; - bool hasTrigger() const { return word0 & TriggerWord::HasTrigger; } + bool hasTrigger() const { return word0 != 0x0; } uint16_t getFirstBC() const { return word0 & 0xFFF; } }; #endif // !defined(GPUCA_GPUCODE) diff --git a/DataFormats/Detectors/TPC/include/DataFormatsTPC/dEdxInfo.h b/DataFormats/Detectors/TPC/include/DataFormatsTPC/dEdxInfo.h index 63b64a50b63dc..bdf9c41aae421 100644 --- a/DataFormats/Detectors/TPC/include/DataFormatsTPC/dEdxInfo.h +++ b/DataFormats/Detectors/TPC/include/DataFormatsTPC/dEdxInfo.h @@ -22,24 +22,24 @@ namespace o2 namespace tpc { struct dEdxInfo { - float dEdxTotIROC; - float dEdxTotOROC1; - float dEdxTotOROC2; - float dEdxTotOROC3; - float dEdxTotTPC; - float dEdxMaxIROC; - float dEdxMaxOROC1; - float dEdxMaxOROC2; - float dEdxMaxOROC3; - float dEdxMaxTPC; - unsigned char NHitsIROC; - unsigned char NHitsSubThresholdIROC; - unsigned char NHitsOROC1; - unsigned char NHitsSubThresholdOROC1; - unsigned char NHitsOROC2; - unsigned char NHitsSubThresholdOROC2; - unsigned char NHitsOROC3; - unsigned char NHitsSubThresholdOROC3; + float dEdxTotIROC = 0.f; + float dEdxTotOROC1 = 0.f; + float dEdxTotOROC2 = 0.f; + float dEdxTotOROC3 = 0.f; + float dEdxTotTPC = 0.f; + float dEdxMaxIROC = 0.f; + float dEdxMaxOROC1 = 0.f; + float dEdxMaxOROC2 = 0.f; + float dEdxMaxOROC3 = 0.f; + float dEdxMaxTPC = 0.f; + unsigned char NHitsIROC = 0; + unsigned char NHitsSubThresholdIROC = 0; + unsigned char NHitsOROC1 = 0; + unsigned char NHitsSubThresholdOROC1 = 0; + unsigned char NHitsOROC2 = 0; + unsigned char NHitsSubThresholdOROC2 = 0; + unsigned char NHitsOROC3 = 0; + unsigned char NHitsSubThresholdOROC3 = 0; ClassDefNV(dEdxInfo, 1); }; } // namespace tpc diff --git a/DataFormats/Detectors/TPC/src/CalibdEdxCorrection.cxx b/DataFormats/Detectors/TPC/src/CalibdEdxCorrection.cxx index 7f28b74bbfd81..152feacb41937 100644 --- a/DataFormats/Detectors/TPC/src/CalibdEdxCorrection.cxx +++ b/DataFormats/Detectors/TPC/src/CalibdEdxCorrection.cxx @@ -11,10 +11,13 @@ #include "DataFormatsTPC/CalibdEdxCorrection.h" +#include #include // o2 includes +#include "Framework/Logger.h" #include "DataFormatsTPC/Defs.h" +#include "CommonUtils/TreeStreamRedirector.h" // root includes #include "TFile.h" @@ -34,17 +37,147 @@ void CalibdEdxCorrection::clear() mDims = -1; } -void CalibdEdxCorrection::writeToFile(std::string_view fileName) const +void CalibdEdxCorrection::writeToFile(std::string_view fileName, std::string_view objName) const { std::unique_ptr file(TFile::Open(fileName.data(), "recreate")); - file->WriteObject(this, "CalibdEdxCorrection"); + if (!file) { + LOGP(error, "Failed to open file {} for writing", fileName.data()); + return; + } + + file->WriteObject(this, objName.data()); } -void CalibdEdxCorrection::loadFromFile(std::string_view fileName) +void CalibdEdxCorrection::loadFromFile(std::string_view fileName, std::string_view objName) { std::unique_ptr file(TFile::Open(fileName.data())); - auto tmp = file->Get("CalibdEdxCorrection"); + if (!file || file->IsZombie()) { + LOGP(error, "Failed to open file {}", fileName.data()); + return; + } + + auto tmp = file->Get(objName.data()); if (tmp != nullptr) { *this = *tmp; + } else { + LOGP(error, "Failed to load object with name {} from file {}", objName.data(), fileName.data()); + } +} + +void CalibdEdxCorrection::dumpToTree(const char* outFileName) const +{ + o2::utils::TreeStreamRedirector pcstream(outFileName, "RECREATE"); + pcstream.GetFile()->cd(); + + for (int sector = 0; sector < 2 * SECTORSPERSIDE; ++sector) { + for (int roc = 0; roc < GEMSTACKSPERSECTOR; ++roc) { + tpc::StackID stack{ + sector, + static_cast(roc)}; + + std::vector qMaxCorrOut; + std::vector qTotCorrOut; + std::vector tglOut; + std::vector snpOut; + + for (float tgl = 0; tgl < 2; tgl += 0.01) { + for (float snp = 0; snp < 1; snp += 0.1) { + qMaxCorrOut.emplace_back(getCorrection(stack, ChargeType::Max, tgl, snp)); + qTotCorrOut.emplace_back(getCorrection(stack, ChargeType::Tot, tgl, snp)); + tglOut.emplace_back(tgl); + snpOut.emplace_back(snp); + } + } + + pcstream << "tree" + << "qMaxCorr=" << qMaxCorrOut + << "qTotCorr=" << qTotCorrOut + << "tgl=" << tglOut + << "snp=" << snpOut + << "roc=" << roc + << "sector=" << sector + << "\n"; + } + } +} + +const std::array CalibdEdxCorrection::getMeanParams(ChargeType charge) const +{ + std::array params{}; + + for (int index = 0; index < FitSize / 2; ++index) { + std::transform(params.begin(), params.end(), mParams[index + charge * FitSize / 2], params.begin(), std::plus<>()); + } + std::for_each(params.begin(), params.end(), [](auto& val) { val /= (0.5f * FitSize); }); + return params; +} + +const std::array CalibdEdxCorrection::getMeanParams(const GEMstack stack, ChargeType charge) const +{ + std::array params{}; + + for (int index = 0; index < SECTORSPERSIDE * SIDES; ++index) { + std::transform(params.begin(), params.end(), getParams(StackID{index, stack}, charge), params.begin(), std::plus<>()); + } + std::for_each(params.begin(), params.end(), [](auto& val) { val /= (SECTORSPERSIDE * SIDES); }); + return params; +} + +float CalibdEdxCorrection::getMeanParam(ChargeType charge, uint32_t param) const +{ + if (param >= ParamSize) { + return 0.f; + } + float mean{}; + for (int index = 0; index < FitSize / 2; ++index) { + mean += mParams[index + charge * FitSize / 2][param]; + } + + return mean / (0.5f * FitSize); +} + +float CalibdEdxCorrection::getMeanParam(const GEMstack stack, ChargeType charge, uint32_t param) const +{ + if (param >= ParamSize) { + return 0.f; + } + float mean{}; + for (int index = 0; index < SECTORSPERSIDE * SIDES; ++index) { + mean += getParams(StackID{index, stack}, charge)[param]; + } + + return mean / (SECTORSPERSIDE * SIDES); +} + +float CalibdEdxCorrection::getMeanEntries(ChargeType charge) const +{ + float mean{}; + for (int index = 0; index < FitSize / 2; ++index) { + mean += mEntries[index + charge * FitSize / 2]; + } + + return mean / (0.5f * FitSize); +} + +float CalibdEdxCorrection::getMeanEntries(const GEMstack stack, ChargeType charge) const +{ + float mean{}; + for (int index = 0; index < SECTORSPERSIDE * SIDES; ++index) { + mean += getEntries(StackID{index, stack}, charge); + } + + return mean / (SECTORSPERSIDE * SIDES); +} + +void CalibdEdxCorrection::setUnity() +{ + for (int i = 0; i < FitSize; ++i) { + for (int j = 0; j < ParamSize; ++j) { + mParams[i][j] = 0.f; + } + mParams[i][0] = 1.f; // constant term = 1 + mChi2[i] = 0.f; + mEntries[i] = 0; } + mDims = 0; } diff --git a/DataFormats/Detectors/TPC/src/ClusterNativeHelper.cxx b/DataFormats/Detectors/TPC/src/ClusterNativeHelper.cxx index 28bd810453ba3..a1268c02a2740 100644 --- a/DataFormats/Detectors/TPC/src/ClusterNativeHelper.cxx +++ b/DataFormats/Detectors/TPC/src/ClusterNativeHelper.cxx @@ -17,7 +17,7 @@ #include "DataFormatsTPC/ClusterNativeHelper.h" #include "Algorithm/Parser.h" #include -#include +#include #include using namespace o2::tpc; diff --git a/DataFormats/Detectors/TPC/src/CompressedClusters.cxx b/DataFormats/Detectors/TPC/src/CompressedClusters.cxx index f4e5ff3cedd6d..400eed3848c64 100644 --- a/DataFormats/Detectors/TPC/src/CompressedClusters.cxx +++ b/DataFormats/Detectors/TPC/src/CompressedClusters.cxx @@ -13,7 +13,7 @@ /// \brief Container to store compressed TPC cluster data #include "DataFormatsTPC/CompressedClusters.h" - +#include "GPUCommonLogger.h" #include #ifndef GPUCA_STANDALONE #include "DataFormatsTPC/CompressedClustersHelpers.h" @@ -55,6 +55,26 @@ void CompressedClustersFlat::setForward(const CompressedClusters* p) ptrForward = p; } +void CompressedClusters::dump() +{ + LOG(info) << "nTracks" << nTracks; + for (unsigned int i = 0; i < nTracks; i++) { + LOG(info) << " " << i << ":" << (unsigned int)qPtA[i] << " " << (unsigned int)rowA[i] << " " << (unsigned int)sliceA[i] << " " << (unsigned int)timeA[i] << " " << (unsigned int)padA[i]; + } + LOG(info) << "nAttachedClusters" << nAttachedClusters; + for (unsigned int i = 0; i < nAttachedClusters; i++) { + LOG(info) << " " << i << ":" << (unsigned int)qTotA[i] << " " << (unsigned int)qMaxA[i] << " " << (unsigned int)flagsA[i] << " " << (unsigned int)sigmaPadA[i] << " " << (unsigned int)sigmaTimeA[i]; + } + LOG(info) << "nAttachedClustersReduced" << nAttachedClustersReduced; + for (unsigned int i = 0; i < nAttachedClustersReduced; i++) { + LOG(info) << " " << i << ":" << (unsigned int)rowDiffA[i] << " " << (unsigned int)sliceLegDiffA[i] << " " << (unsigned int)padResA[i] << " " << (unsigned int)timeResA[i]; + } + LOG(info) << "nUnattachedClusters" << nUnattachedClusters; + for (unsigned int i = 0; i < nUnattachedClusters; i++) { + LOG(info) << " " << i << ":" << (unsigned int)qTotU[i] << " " << (unsigned int)qMaxU[i] << " " << (unsigned int)flagsU[i] << " " << (unsigned int)padDiffU[i] << " " << (unsigned int)timeDiffU[i] << " " << (unsigned int)sigmaPadU[i] << " " << (unsigned int)sigmaTimeU[i]; + } +} + #ifndef GPUCA_STANDALONE void CompressedClustersROOT::Streamer(TBuffer& R__b) { diff --git a/DataFormats/Detectors/TPC/src/DCS.cxx b/DataFormats/Detectors/TPC/src/DCS.cxx index 131f6c9526647..b56d07acd7c73 100644 --- a/DataFormats/Detectors/TPC/src/DCS.cxx +++ b/DataFormats/Detectors/TPC/src/DCS.cxx @@ -16,6 +16,8 @@ #include #include "DataFormatsTPC/DCS.h" +#include "TLinearFitter.h" +#include "TTree.h" using namespace o2::tpc::dcs; @@ -117,3 +119,431 @@ TimeStampType Gas::getMaxTime() const return *std::max_element(times.begin(), times.end()); } + +TimeStampType Pressure::getMinTime() const +{ + constexpr auto max = std::numeric_limits::max(); + const std::vector times{ + cavernAtmosPressure.data.size() ? cavernAtmosPressure.data.front().time : max, + cavernAtmosPressure2.data.size() ? cavernAtmosPressure2.data.front().time : max, + surfaceAtmosPressure.data.size() ? surfaceAtmosPressure.data.front().time : max, + }; + + return *std::min_element(times.begin(), times.end()); +} + +TimeStampType Pressure::getMaxTime() const +{ + constexpr auto min = 0; + const std::vector times{ + cavernAtmosPressure.data.size() ? cavernAtmosPressure.data.back().time : min, + cavernAtmosPressure2.data.size() ? cavernAtmosPressure2.data.back().time : min, + surfaceAtmosPressure.data.size() ? surfaceAtmosPressure.data.back().time : min, + }; + + return *std::max_element(times.begin(), times.end()); +} + +bool Temperature::makeFit(TLinearFitter& fitter, const int nDim, std::vector& xVals, std::vector& temperatures) +{ + const int minPointsForFit = 5; + if (temperatures.empty() || (temperatures.size() < minPointsForFit)) { + LOGP(warning, "Number of points {} for fit smaller than minimum of {}!", temperatures.size(), minPointsForFit); + return false; + } + + fitter.ClearPoints(); + fitter.AssignData(temperatures.size(), nDim, xVals.data(), temperatures.data()); + int status = fitter.Eval(); + if (status == 1) { + LOGP(warning, "Fit failed!"); + return false; + } + return true; +} + +void Temperature::fitTemperature(Side side, dcs::TimeStampType fitInterval, const bool roundToInterval) +{ + // clear old data + auto& stats = (side == Side::A) ? statsA : statsC; + stats.clear(); + + // temperature fits in x-y + const int nDim = 2; + TLinearFitter fitter(nDim, "1 ++ x0 ++ x1", ""); + std::array startPos{}; + const size_t sensorOffset = (side == Side::C) ? dcs::Temperature::SensorsPerSide : 0; + + const dcs::TimeStampType refTime = getMinTime(raw, fitInterval, roundToInterval); + const dcs::TimeStampType refTimeMax = getMaxTime(raw); + + // calculate number of intervals and see if the last interval should be merged into the previous one + const int lastIntervalDuration = (refTimeMax - refTime) % fitInterval; + + // process the last interval only if it contains more than 50% of the interval duration + const bool procLastInt = (lastIntervalDuration / fitInterval > 0.5); + int numIntervals = (refTimeMax - refTime) / fitInterval + procLastInt; + if (numIntervals == 0) { + numIntervals = 1; + } + + // buffer for fit values + std::vector xVals; + std::vector temperatures; + xVals.reserve(2 * 1000); + temperatures.reserve(1000); + + for (int interval = 0; interval < numIntervals; ++interval) { + const dcs::TimeStampType timeStart = refTime + interval * fitInterval; + + // clear buffer + xVals.clear(); + temperatures.clear(); + + // TODO: check if we should use refTime + dcs::TimeStampType firstTime = std::numeric_limits::max(); + dcs::TimeStampType LastTime = 0; + + for (size_t iSensor = 0; iSensor < dcs::Temperature::SensorsPerSide; ++iSensor) { + const auto& sensor = raw[iSensor + sensorOffset]; + + LOGP(debug, "sensor {}, start {}, size {}", sensor.sensorNumber, startPos[iSensor], sensor.data.size()); + while (startPos[iSensor] < sensor.data.size()) { + const auto& dataPoint = sensor.data[startPos[iSensor]]; + if (((dataPoint.time - timeStart) >= fitInterval) && (interval != numIntervals - 1)) { + LOGP(debug, "sensor {}, {} - {} >= {}", sensor.sensorNumber, dataPoint.time, timeStart, fitInterval); + break; + } + firstTime = std::min(firstTime, dataPoint.time); + LastTime = std::max(LastTime, dataPoint.time); + const auto temperature = dataPoint.value; + // sanity check + ++startPos[iSensor]; + if (temperature < 15 || temperature > 25) { + continue; + } + const auto& pos = dcs::Temperature::SensorPosition[iSensor + sensorOffset]; + xVals.emplace_back(pos.x); + xVals.emplace_back(pos.y); + temperatures.emplace_back(temperature); + } + } + if (firstTime < std::numeric_limits::max() && !temperatures.empty()) { + const bool fitOk = makeFit(fitter, nDim, xVals, temperatures); + if (!fitOk) { + continue; + } + auto& stat = stats.data.emplace_back(); + stat.time = (firstTime + LastTime) / 2; + stat.value.mean = fitter.GetParameter(0); + stat.value.gradX = fitter.GetParameter(1); + stat.value.gradY = fitter.GetParameter(2); + + // check if data contains outliers + const float maxDeltaT = 1; + const float meanTemp = fitter.GetParameter(0); + const bool isDataGood = std::all_of(temperatures.begin(), temperatures.end(), [meanTemp, maxDeltaT](double t) { return std::abs(t - meanTemp) < maxDeltaT; }); + + // do second iteration only in case of outliers + if (!isDataGood) { + std::vector xVals2; + std::vector temperatures2; + xVals2.reserve(xVals.size()); + temperatures2.reserve(temperatures.size()); + for (int i = 0; i < temperatures.size(); ++i) { + if (std::abs(temperatures[i] - meanTemp) < maxDeltaT) { + const int idx = 2 * i; + xVals2.emplace_back(xVals[idx]); + xVals2.emplace_back(xVals[idx + 1]); + temperatures2.emplace_back(temperatures[i]); + } + } + const bool fitOk2 = makeFit(fitter, nDim, xVals2, temperatures2); + if (fitOk2) { + stat.value.mean = fitter.GetParameter(0); + stat.value.gradX = fitter.GetParameter(1); + stat.value.gradY = fitter.GetParameter(2); + } + } + } + } +} + +void Pressure::fill(std::string_view sensor, const TimeStampType time, const DataType value) +{ + if (sensor == "CavernAtmosPressure") { + cavernAtmosPressure.fill(time, value); + } else if (sensor == "CavernAtmosPressure2") { + cavernAtmosPressure2.fill(time, value); + } else if (sensor == "SurfaceAtmosPressure") { + surfaceAtmosPressure.fill(time, value); + } else { + LOGP(warning, "Unknown pressure sensor {}", sensor); + } +} + +void Pressure::sortAndClean(float pMin, float pMax) +{ + cavernAtmosPressure.sortAndClean(); + cavernAtmosPressure2.sortAndClean(); + surfaceAtmosPressure.sortAndClean(); + + auto removeOutliers = [](auto& dataVec, auto minVal, auto maxVal) { + dataVec.erase( + std::remove_if(dataVec.begin(), dataVec.end(), + [minVal, maxVal](const auto& dp) { + return (dp.value < minVal || dp.value > maxVal); + }), + dataVec.end()); + }; + + removeOutliers(cavernAtmosPressure.data, pMin, pMax); + removeOutliers(cavernAtmosPressure2.data, pMin, pMax); + removeOutliers(surfaceAtmosPressure.data, pMin, pMax); +} + +void Pressure::clear() +{ + cavernAtmosPressure.clear(); + cavernAtmosPressure2.clear(); + surfaceAtmosPressure.clear(); + robustPressure = RobustPressure(); +} + +void Pressure::append(const Pressure& other) +{ + cavernAtmosPressure.append(other.cavernAtmosPressure); + cavernAtmosPressure2.append(other.cavernAtmosPressure2); + surfaceAtmosPressure.append(other.surfaceAtmosPressure); +} + +void fillBuffer(std::pair, std::vector>& buffer, const std::pair, std::vector>& values, TimeStampType tStart, const int minPoints) +{ + const auto itStartBuff = std::lower_bound(buffer.second.begin(), buffer.second.end(), tStart); + size_t idxStartBuffer = std::distance(buffer.second.begin(), itStartBuff); + if (buffer.first.size() - idxStartBuffer < minPoints) { + if (buffer.first.size() < minPoints) { + idxStartBuffer = 0; + } else { + idxStartBuffer = buffer.first.size() - minPoints; + } + } + + std::pair, std::vector> buffTmp; + auto& [buffVals, buffTimes] = buffTmp; + + // Preallocate enough capacity to avoid reallocations + buffVals.reserve(buffer.first.size() - idxStartBuffer + values.first.size()); + buffTimes.reserve(buffer.second.size() - idxStartBuffer + values.second.size()); + // Insert the kept part of the old buffer + buffVals.insert(buffVals.end(), buffer.first.begin() + idxStartBuffer, buffer.first.end()); + buffTimes.insert(buffTimes.end(), buffer.second.begin() + idxStartBuffer, buffer.second.end()); + // Insert the new values + buffVals.insert(buffVals.end(), values.first.begin(), values.first.end()); + buffTimes.insert(buffTimes.end(), values.second.begin(), values.second.end()); + + // this should not happen + if (!std::is_sorted(buffTimes.begin(), buffTimes.end())) { + LOGP(info, "Pressure buffer not sorted after filling - sorting it"); + std::vector idx(buffTimes.size()); + o2::math_utils::SortData(buffTimes, idx); + o2::math_utils::Reorder(buffVals, idx); + o2::math_utils::Reorder(buffTimes, idx); + } + + buffer = std::move(buffTmp); +} + +void Pressure::makeRobustPressure(TimeStampType timeInterval, TimeStampType timeIntervalRef, TimeStampType tStart, TimeStampType tEnd, const int nthreads) +{ + const auto surfaceAtmosPressurePair = surfaceAtmosPressure.getPairOfVector(); + const auto cavernAtmosPressurePair = cavernAtmosPressure.getPairOfVector(); + const auto cavernAtmosPressure2Pair = cavernAtmosPressure2.getPairOfVector(); + + // round to second + tStart = tStart / 1000 * 1000; + const TimeStampType tStartRef = (tStart - timeIntervalRef); + const int minPointsRef = 50; + fillBuffer(mCavernAtmosPressure1Buff, cavernAtmosPressurePair, tStartRef, minPointsRef); + fillBuffer(mCavernAtmosPressure2Buff, cavernAtmosPressure2Pair, tStartRef, minPointsRef); + fillBuffer(mSurfaceAtmosPressureBuff, surfaceAtmosPressurePair, tStartRef, minPointsRef); + + int nIntervals = std::round((tEnd - tStart) / timeInterval); + if (nIntervals == 0) { + nIntervals = 1; // at least one interval + } + std::vector times; + times.reserve(nIntervals); + for (int i = 0; i < nIntervals; ++i) { + times.emplace_back(tStart + (i + 0.5) * timeInterval); + } + + /// minimum number of points in the interval - otherwise use the n closest points + const int minPoints = 4; + const auto cavernAtmosPressureStats = o2::math_utils::getRollingStatistics(mCavernAtmosPressure1Buff.second, mCavernAtmosPressure1Buff.first, times, timeInterval, nthreads, minPoints, minPoints); + const auto cavernAtmosPressure2Stats = o2::math_utils::getRollingStatistics(mCavernAtmosPressure2Buff.second, mCavernAtmosPressure2Buff.first, times, timeInterval, nthreads, minPoints, minPoints); + const auto surfaceAtmosPressureStats = o2::math_utils::getRollingStatistics(mSurfaceAtmosPressureBuff.second, mSurfaceAtmosPressureBuff.first, times, timeInterval, nthreads, minPoints, minPoints); + + // subtract the moving median values from the different sensors if they are ok + std::pair, std::vector> cavernAtmosPressure12; + std::pair, std::vector> cavernAtmosPressure1S; + std::pair, std::vector> cavernAtmosPressure2S; + cavernAtmosPressure12.first.reserve(nIntervals); + cavernAtmosPressure1S.first.reserve(nIntervals); + cavernAtmosPressure2S.first.reserve(nIntervals); + cavernAtmosPressure12.second.reserve(nIntervals); + cavernAtmosPressure1S.second.reserve(nIntervals); + cavernAtmosPressure2S.second.reserve(nIntervals); + + for (int i = 0; i < nIntervals; i++) { + // coarse check if data is close by + const int maxDist = 600 * 1000; + const bool cavernOk = (cavernAtmosPressureStats.median[i] > 0) && (cavernAtmosPressureStats.closestDistanceL[i] < maxDist) && (cavernAtmosPressureStats.closestDistanceR[i] < maxDist); + const bool cavern2Ok = (cavernAtmosPressure2Stats.median[i] > 0) && (cavernAtmosPressure2Stats.closestDistanceL[i] < maxDist) && (cavernAtmosPressure2Stats.closestDistanceR[i] < maxDist); + const bool surfaceOk = (surfaceAtmosPressureStats.median[i] > 0) && (surfaceAtmosPressureStats.closestDistanceL[i] < maxDist) && (surfaceAtmosPressureStats.closestDistanceR[i] < maxDist); + + if (cavernOk && cavern2Ok) { + cavernAtmosPressure12.first.emplace_back(cavernAtmosPressureStats.median[i] - cavernAtmosPressure2Stats.median[i]); + cavernAtmosPressure12.second.emplace_back(times[i]); + } + if (cavernOk && surfaceOk) { + cavernAtmosPressure1S.first.emplace_back(cavernAtmosPressureStats.median[i] - surfaceAtmosPressureStats.median[i]); + cavernAtmosPressure1S.second.emplace_back(times[i]); + } + if (cavern2Ok && surfaceOk) { + cavernAtmosPressure2S.first.emplace_back(cavernAtmosPressure2Stats.median[i] - surfaceAtmosPressureStats.median[i]); + cavernAtmosPressure2S.second.emplace_back(times[i]); + } + } + + fillBuffer(mPressure12Buff, cavernAtmosPressure12, tStartRef, minPointsRef); + fillBuffer(mPressure1SBuff, cavernAtmosPressure1S, tStartRef, minPointsRef); + fillBuffer(mPressure2SBuff, cavernAtmosPressure2S, tStartRef, minPointsRef); + + // get long term median of diffs - this is used for normalization of the pressure values - + const auto cavernAtmosPressure12Stats = o2::math_utils::getRollingStatistics(mPressure12Buff.second, mPressure12Buff.first, times, timeIntervalRef, nthreads, 3, minPointsRef); + const auto cavernAtmosPressure1SStats = o2::math_utils::getRollingStatistics(mPressure1SBuff.second, mPressure1SBuff.first, times, timeIntervalRef, nthreads, 3, minPointsRef); + const auto cavernAtmosPressure2SStats = o2::math_utils::getRollingStatistics(mPressure2SBuff.second, mPressure2SBuff.first, times, timeIntervalRef, nthreads, 3, minPointsRef); + + // calculate diffs of median values + const float maxDist = 20 * timeInterval; + const float maxDiff = 0.2; + std::pair, std::vector> robustPressureTmp; + robustPressureTmp.first.reserve(nIntervals); + robustPressureTmp.second.reserve(nIntervals); + std::vector isOk(nIntervals); + + for (int i = 0; i < nIntervals; ++i) { + // difference beween pressure values corrected for the long term median + const float delta12 = cavernAtmosPressureStats.median[i] - cavernAtmosPressure2Stats.median[i] - cavernAtmosPressure12Stats.median[i]; + const float delta1S = cavernAtmosPressureStats.median[i] - surfaceAtmosPressureStats.median[i] - cavernAtmosPressure1SStats.median[i]; + const float delta2S = cavernAtmosPressure2Stats.median[i] - surfaceAtmosPressureStats.median[i] - cavernAtmosPressure2SStats.median[i]; + + const auto distCavernAtmosPressureL = cavernAtmosPressureStats.closestDistanceL[i]; + const auto distCavernAtmosPressure2L = cavernAtmosPressure2Stats.closestDistanceL[i]; + const auto distSurfaceAtmosPressureL = surfaceAtmosPressureStats.closestDistanceL[i]; + const auto distCavernAtmosPressureR = cavernAtmosPressureStats.closestDistanceR[i]; + const auto distCavernAtmosPressure2R = cavernAtmosPressure2Stats.closestDistanceR[i]; + const auto distSurfaceAtmosPressureR = surfaceAtmosPressureStats.closestDistanceR[i]; + + // check if data is ok + const bool cavernDistOk = (cavernAtmosPressureStats.median[i] > 0) && ((distCavernAtmosPressureL < maxDist) || (distCavernAtmosPressureR < maxDist)); + const bool cavern2DistOk = (cavernAtmosPressure2Stats.median[i] > 0) && ((distCavernAtmosPressure2L < maxDist) || (distCavernAtmosPressure2R < maxDist)); + const bool surfaceDistOk = (surfaceAtmosPressureStats.median[i] > 0) && ((distSurfaceAtmosPressureL < maxDist) || (distSurfaceAtmosPressureR < maxDist)); + const bool onlyOneSensor = (cavernDistOk + cavern2DistOk + surfaceDistOk) == 1; // check if only 1 sensor exists, if so use that sensor + + uint8_t maskIsOkTmp = 0; + const int cavernBit = 0; // val 1 + const int cavern2Bit = 1; // val 2 + const int surfaceBit = 2; // val 4 + + // check if ratio sensor 1 and 2 are good + // maskIsOkTmp = 3 + if (((std::abs(delta12) < maxDiff) && (cavernDistOk && cavern2DistOk)) || onlyOneSensor) { + if (cavernDistOk) { + maskIsOkTmp |= (1 << cavernBit); + } + if (cavern2DistOk) { + maskIsOkTmp |= (1 << cavern2Bit); + } + } + + // check if ratio sensor 1 and surface are good + // maskIsOkTmp = 5 + if ((std::abs(delta1S) < maxDiff) && ((cavernDistOk && surfaceDistOk)) || onlyOneSensor) { + if (cavernDistOk) { + maskIsOkTmp |= (1 << cavernBit); + } + if (surfaceDistOk) { + maskIsOkTmp |= (1 << surfaceBit); + } + } + + // check if ratio sensor 2 and surface are good + // maskIsOkTmp = 6 + if ((std::abs(delta2S) < maxDiff) && ((cavern2DistOk && surfaceDistOk)) || onlyOneSensor) { + if (cavern2DistOk) { + maskIsOkTmp |= (1 << cavern2Bit); + } + if (surfaceDistOk) { + maskIsOkTmp |= (1 << surfaceBit); + } + } + + // calculate robust pressure + float pressure = 0; + int pressureCount = 0; + if ((maskIsOkTmp >> cavernBit) & 1) { + pressure += cavernAtmosPressureStats.median[i]; + pressureCount++; + } + + if ((maskIsOkTmp >> cavern2Bit) & 1) { + pressure += cavernAtmosPressure2Stats.median[i] + cavernAtmosPressure12Stats.median[i]; + pressureCount++; + } + + if ((maskIsOkTmp >> surfaceBit) & 1) { + pressure += surfaceAtmosPressureStats.median[i] + cavernAtmosPressure1SStats.median[i]; + pressureCount++; + } + + isOk[i] = maskIsOkTmp; + if (pressureCount > 0) { + pressure /= pressureCount; + robustPressureTmp.first.emplace_back(pressure); + robustPressureTmp.second.emplace_back(times[i]); + } + } + + fillBuffer(mRobPressureBuff, robustPressureTmp, tStartRef, minPointsRef); + + RobustPressure& pOut = robustPressure; + pOut.surfaceAtmosPressure = std::move(surfaceAtmosPressureStats); + pOut.cavernAtmosPressure2 = std::move(cavernAtmosPressure2Stats); + pOut.cavernAtmosPressure = std::move(cavernAtmosPressureStats); + pOut.cavernAtmosPressure12 = std::move(cavernAtmosPressure12Stats); + pOut.cavernAtmosPressure1S = std::move(cavernAtmosPressure1SStats); + pOut.cavernAtmosPressure2S = std::move(cavernAtmosPressure2SStats); + pOut.isOk = std::move(isOk); + pOut.robustPressure = o2::math_utils::getRollingStatistics(mRobPressureBuff.second, mRobPressureBuff.first, times, timeInterval, nthreads, 1, 5).median; + pOut.time = std::move(times); + pOut.timeInterval = timeInterval; + pOut.timeIntervalRef = timeIntervalRef; + pOut.maxDist = maxDist; + pOut.maxDiff = maxDiff; +} + +void Pressure::setAliases(TTree* tree) +{ + tree->SetAlias("cavernDistOk", "robustPressure.cavernAtmosPressure.median>0 && (robustPressure.cavernAtmosPressure.closestDistanceRSetAlias("cavern2DistOk", "robustPressure.cavernAtmosPressure2.median>0 && (robustPressure.cavernAtmosPressure2.closestDistanceRSetAlias("surfaceDistOk", "robustPressure.surfaceAtmosPressure.median>0 && (robustPressure.surfaceAtmosPressure.closestDistanceRSetAlias("onlyOneSensor", "(cavernDistOk + cavern2DistOk + surfaceDistOk) == 1"); + tree->SetAlias("delta12", "robustPressure.cavernAtmosPressure.median - robustPressure.cavernAtmosPressure2.median - robustPressure.cavernAtmosPressure12.median"); + tree->SetAlias("delta1S", "robustPressure.cavernAtmosPressure.median - robustPressure.surfaceAtmosPressure.median - robustPressure.cavernAtmosPressure1S.median"); + tree->SetAlias("delta2S", "robustPressure.surfaceAtmosPressure.median - robustPressure.cavernAtmosPressure2.median - robustPressure.cavernAtmosPressure2S.median"); + tree->SetAlias("delta12_Ok", "abs(delta12)SetAlias("delta1S_Ok", "abs(delta1S)SetAlias("delta2S_Ok", "abs(delta2S) + ; +#pragma link C++ class o2::ctf::EncodedBlocks < o2::tpc::CTFHeader, 26, uint32_t> + ; #pragma link C++ enum o2::tpc::StatisticsType; #pragma link C++ class o2::tpc::TrackCuts + ; #pragma link C++ class o2::tpc::KrCluster + ; @@ -69,5 +69,12 @@ #pragma link C++ class o2::tpc::dcs::DataPoint < o2::tpc::dcs::HV::StackState> + ; #pragma link C++ class o2::tpc::dcs::DataPointVector < o2::tpc::dcs::HV::StackState> + ; #pragma link C++ class o2::tpc::dcs::Gas + ; +#pragma link C++ class o2::tpc::dcs::Pressure + ; +#pragma link C++ class o2::tpc::dcs::RobustPressure + ; +#pragma link C++ class o2::tpc::PIDResponse + ; +#pragma link C++ class o2::tpc::TriggerWordDLBZS + ; +#pragma link C++ class o2::tpc::TriggerInfoDLBZS + ; +#pragma link C++ class std::vector < o2::tpc::TriggerInfoDLBZS> + ; +#pragma link C++ class o2::tpc::AltroSyncSignal + ; #endif diff --git a/DataFormats/Detectors/TPC/src/LaserTrack.cxx b/DataFormats/Detectors/TPC/src/LaserTrack.cxx index e44bcf7e20bd6..cbce09beb774a 100644 --- a/DataFormats/Detectors/TPC/src/LaserTrack.cxx +++ b/DataFormats/Detectors/TPC/src/LaserTrack.cxx @@ -18,7 +18,7 @@ #include #include #include -#include +#include #include "TFile.h" #include "TTree.h" diff --git a/DataFormats/Detectors/TPC/src/TrackCuts.cxx b/DataFormats/Detectors/TPC/src/TrackCuts.cxx index 7211583a75e93..111f3c3da8920 100644 --- a/DataFormats/Detectors/TPC/src/TrackCuts.cxx +++ b/DataFormats/Detectors/TPC/src/TrackCuts.cxx @@ -9,7 +9,7 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. -#include "FairLogger.h" +#include #include "DataFormatsTPC/TrackCuts.h" #include "DataFormatsTPC/TrackTPC.h" @@ -49,5 +49,8 @@ bool TrackCuts::goodTrack(o2::tpc::TrackTPC const& track) if (dEdx < mdEdxMin) { return false; } + if ((std::abs(track.getOuterParam().getZ()) - std::abs(track.getZ())) < -10) { + return false; + } return true; } diff --git a/DataFormats/Detectors/TRD/CMakeLists.txt b/DataFormats/Detectors/TRD/CMakeLists.txt index ebbefee810e15..35712caf5bfb9 100644 --- a/DataFormats/Detectors/TRD/CMakeLists.txt +++ b/DataFormats/Detectors/TRD/CMakeLists.txt @@ -13,6 +13,8 @@ o2_add_library(DataFormatsTRD SOURCES src/TriggerRecord.cxx src/LinkRecord.cxx src/AngularResidHistos.cxx + src/GainCalibHistos.cxx + src/T0FitHistos.cxx src/DcsCcdbObjects.cxx src/Tracklet64.cxx src/RawData.cxx @@ -35,14 +37,19 @@ o2_target_root_dictionary(DataFormatsTRD include/DataFormatsTRD/CalibratedTracklet.h include/DataFormatsTRD/DcsCcdbObjects.h include/DataFormatsTRD/AngularResidHistos.h + include/DataFormatsTRD/GainCalibHistos.h + include/DataFormatsTRD/T0FitHistos.h include/DataFormatsTRD/HelperMethods.h include/DataFormatsTRD/Hit.h include/DataFormatsTRD/Digit.h include/DataFormatsTRD/KrCluster.h include/DataFormatsTRD/KrClusterTriggerRecord.h include/DataFormatsTRD/NoiseCalibration.h + include/DataFormatsTRD/PHData.h + include/DataFormatsTRD/RawDataStats.h include/DataFormatsTRD/CTF.h include/DataFormatsTRD/CalVdriftExB.h + include/DataFormatsTRD/CalGain.h include/DataFormatsTRD/CalT0.h include/DataFormatsTRD/SignalArray.h include/DataFormatsTRD/CompressedDigit.h @@ -55,3 +62,11 @@ o2_add_test(Digit ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage LABELS trd ) + +o2_add_test(RawData + COMPONENT_NAME trd + PUBLIC_LINK_LIBRARIES O2::DataFormatsTRD + SOURCES test/testRawData.cxx + ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage + LABELS trd + ) diff --git a/DataFormats/Detectors/TRD/include/DataFormatsTRD/AngularResidHistos.h b/DataFormats/Detectors/TRD/include/DataFormatsTRD/AngularResidHistos.h index bf935866564c0..4380c79cc48e8 100644 --- a/DataFormats/Detectors/TRD/include/DataFormatsTRD/AngularResidHistos.h +++ b/DataFormats/Detectors/TRD/include/DataFormatsTRD/AngularResidHistos.h @@ -38,7 +38,6 @@ class AngularResidHistos size_t getNEntries() const { return mNEntriesTotal; } void fill(const AngularResidHistos& input); - void fill(const gsl::span input); // dummy! void merge(const AngularResidHistos* prev); void print(); diff --git a/DataFormats/Detectors/TRD/include/DataFormatsTRD/CalGain.h b/DataFormats/Detectors/TRD/include/DataFormatsTRD/CalGain.h new file mode 100644 index 0000000000000..b4e64db094a5c --- /dev/null +++ b/DataFormats/Detectors/TRD/include/DataFormatsTRD/CalGain.h @@ -0,0 +1,87 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file CalGain.h +/// \brief Object with MPV dEdx values per chamber to be written into the CCDB + +#ifndef ALICEO2_CALGAIN_H +#define ALICEO2_CALGAIN_H + +#include "DataFormatsTRD/Constants.h" +#include "Rtypes.h" +#include + +namespace o2 +{ +namespace trd +{ + +class CalGain +{ + public: + CalGain() = default; + CalGain(const CalGain&) = default; + ~CalGain() = default; + + void setMPVdEdx(int iDet, float mpv) { mMPVdEdx[iDet] = mpv; } + + float getMPVdEdx(int iDet, bool defaultAvg = true) const + { + // if defaultAvg = false, we take the value stored whatever it is + // if defaultAvg = true and we have default value or bad value stored, we take the average on all chambers instead + if (!defaultAvg || isGoodGain(iDet)) + return mMPVdEdx[iDet]; + else { + if (TMath::Abs(mMeanGain + 999.) < 1e-6) + mMeanGain = getAverageGain(); + return mMeanGain; + } + } + + float getAverageGain() const + { + float averageGain = 0.; + int ngood = 0; + + for (int iDet = 0; iDet < constants::MAXCHAMBER; iDet++) { + if (isGoodGain(iDet)) { + // The chamber has correct calibration + ngood++; + averageGain += mMPVdEdx[iDet]; + } + } + if (ngood == 0) { + // we should make sure it never happens + return constants::MPVDEDXDEFAULT; + } + averageGain /= ngood; + return averageGain; + } + + bool isGoodGain(int iDet) const + { + if (TMath::Abs(mMPVdEdx[iDet] - constants::MPVDEDXDEFAULT) > 1e-6) + return true; + else + return false; + } + + private: + std::array mMPVdEdx{}; ///< Most probable value of dEdx distribution per TRD chamber + mutable float mMeanGain{-999.}; ///! average gain, calculated only once + + ClassDefNV(CalGain, 2); +}; + +} // namespace trd +} // namespace o2 + +#endif // ALICEO2_CALGAIN_H diff --git a/DataFormats/Detectors/TRD/include/DataFormatsTRD/CalT0.h b/DataFormats/Detectors/TRD/include/DataFormatsTRD/CalT0.h index 30474888a7424..69fc23646c912 100644 --- a/DataFormats/Detectors/TRD/include/DataFormatsTRD/CalT0.h +++ b/DataFormats/Detectors/TRD/include/DataFormatsTRD/CalT0.h @@ -32,13 +32,38 @@ class CalT0 ~CalT0() = default; void setT0(int iDet, float t0) { mT0[iDet] = t0; } + void setT0av(float t0) { mT0av = t0; } float getT0(int iDet) const { return mT0[iDet]; } + // getT0av() returns the average T0 obtained by fitting the data from all chambers combined + float getT0av() const { return mT0av; } + // calcT0av() returns the average T0 from all individual chambers + float calcT0av() const + { + if (mT0.size() == 0) { + return -1; + } + float sum = 0; + int counts = 0; + for (int iDet = 0; iDet < constants::MAXCHAMBER; ++iDet) { + if (mT0[iDet] > -5) { + sum += mT0[iDet]; + ++counts; + } + } + + if (counts > 0) { + return sum / counts; + } else { + return -2; + } + } private: std::array mT0{}; ///< calibrated T0 per TRD chamber + float mT0av{-1}; ///< average T0 obtained from fitting the PH data from all chambers combined - ClassDefNV(CalT0, 1); + ClassDefNV(CalT0, 2); }; } // namespace trd diff --git a/DataFormats/Detectors/TRD/include/DataFormatsTRD/CalVdriftExB.h b/DataFormats/Detectors/TRD/include/DataFormatsTRD/CalVdriftExB.h index bad9dcfef4e37..65981e928fb39 100644 --- a/DataFormats/Detectors/TRD/include/DataFormatsTRD/CalVdriftExB.h +++ b/DataFormats/Detectors/TRD/include/DataFormatsTRD/CalVdriftExB.h @@ -34,14 +34,102 @@ class CalVdriftExB void setVdrift(int iDet, float vd) { mVdrift[iDet] = vd; } void setExB(int iDet, float exb) { mExB[iDet] = exb; } - float getVdrift(int iDet) const { return mVdrift[iDet]; } - float getExB(int iDet) const { return mExB[iDet]; } + float getVdrift(int iDet, bool defaultAvg = true) const + { + // if defaultAvg = false, we take the value stored whatever it is + // if defaultAvg = true and we have default value or bad value stored, we take the average on all chambers instead + if (!defaultAvg || (isGoodExB(iDet) && isGoodVdrift(iDet))) + return mVdrift[iDet]; + else { + if (TMath::Abs(mMeanVdrift + 999.) < 1e-6) + mMeanVdrift = getAverageVdrift(); + return mMeanVdrift; + } + } + float getExB(int iDet, bool defaultAvg = true) const + { + if (!defaultAvg || (isGoodExB(iDet) && isGoodVdrift(iDet))) + return mExB[iDet]; + else { + if (TMath::Abs(mMeanExB + 999.) < 1e-6) + mMeanExB = getAverageExB(); + return mMeanExB; + } + } + + float getAverageVdrift() const + { + float averageVdrift = 0.; + int ngood = 0; + + for (int iDet = 0; iDet < constants::MAXCHAMBER; iDet++) { + if (isGoodExB(iDet) && isGoodVdrift(iDet)) { + // Both values need to be correct to declare a chamber as well calibrated + ngood++; + averageVdrift += mVdrift[iDet]; + } + } + if (ngood == 0) { + // we should make sure it never happens + return constants::VDRIFTDEFAULT; + } + averageVdrift /= ngood; + return averageVdrift; + } + + float getAverageExB() const + { + float averageExB = 0.; + int ngood = 0; + + for (int iDet = 0; iDet < constants::MAXCHAMBER; iDet++) { + if (isGoodExB(iDet) && isGoodVdrift(iDet)) { + // Both values need to be correct to declare a chamber as well calibrated + ngood++; + averageExB += mExB[iDet]; + } + } + if (ngood == 0) { + // we should make sure it never happens + return constants::EXBDEFAULT; + } + averageExB /= ngood; + return averageExB; + } + + bool isGoodExB(int iDet) const + { + // check if value is well calibrated or not + // default calibration if not enough entries + // close to boundaries indicate a failed fit + if (TMath::Abs(mExB[iDet] - constants::EXBDEFAULT) > 1e-6 && + TMath::Abs(mExB[iDet] - constants::EXBMIN) > 0.01 && + TMath::Abs(mExB[iDet] - constants::EXBMAX) > 0.01) + return true; + else + return false; + } + + bool isGoodVdrift(int iDet) const + { + // check if value is well calibrated or not + // default calibration if not enough entries + // close to boundaries indicate a failed fit + if (TMath::Abs(mVdrift[iDet] - constants::VDRIFTDEFAULT) > 1e-6 && + TMath::Abs(mVdrift[iDet] - constants::VDRIFTMIN) > 0.1 && + TMath::Abs(mVdrift[iDet] - constants::VDRIFTMAX) > 0.1) + return true; + else + return false; + } private: std::array mVdrift{}; ///< calibrated drift velocity per TRD chamber std::array mExB{}; ///< calibrated Lorentz angle per TRD chamber + mutable float mMeanVdrift{-999.}; ///! average drift velocity, calculated only once + mutable float mMeanExB{-999.}; ///! average lorentz angle, calculated only once - ClassDefNV(CalVdriftExB, 1); + ClassDefNV(CalVdriftExB, 2); }; } // namespace trd diff --git a/DataFormats/Detectors/TRD/include/DataFormatsTRD/Constants.h b/DataFormats/Detectors/TRD/include/DataFormatsTRD/Constants.h index 08c8534a4858d..9a4da1024e251 100644 --- a/DataFormats/Detectors/TRD/include/DataFormatsTRD/Constants.h +++ b/DataFormats/Detectors/TRD/include/DataFormatsTRD/Constants.h @@ -22,74 +22,86 @@ namespace trd { namespace constants { -constexpr int NSECTOR = 18; // the number of sectors -constexpr int NSTACK = 5; // the number of stacks per sector -constexpr int NLAYER = 6; // the number of layers -constexpr int NCHAMBERPERSEC = 30; // the number of chambers per sector -constexpr int NHCPERSEC = 60; // the number of half-chambers per sector -constexpr int MAXCHAMBER = 540; // the maximum number of installed chambers -constexpr int MAXHALFCHAMBER = 1080; // the maximum number of installed half-chambers -constexpr int NCHAMBER = 521; // the number of chambers actually installed -constexpr int NHALFCRU = 72; // the number of half cru (link bundles) -constexpr int NLINKSPERHALFCRU = 15; // the number of links per half cru or cru end point. -constexpr int NLINKSPERCRU = 30; // the number of links per CRU (two CRUs serve one supermodule) -constexpr int NCRU = 36; // the number of CRU we have -constexpr int NFLP = 12; // the number of FLP we have. -constexpr int NCRUPERFLP = 3; // the number of CRU per FLP -constexpr int TRDLINKID = 15; // hard coded link id, specific to TRD +constexpr int NSECTOR = 18; ///< the number of sectors +constexpr int NSTACK = 5; ///< the number of stacks per sector +constexpr int NLAYER = 6; ///< the number of layers +constexpr int NCHAMBERPERSEC = 30; ///< the number of chambers per sector +constexpr int NHCPERSEC = 60; ///< the number of half-chambers per sector +constexpr int MAXCHAMBER = 540; ///< the maximum number of installed chambers +constexpr int MAXHALFCHAMBER = 1080; ///< the maximum number of installed half-chambers +constexpr int NCHAMBER = 521; ///< the number of chambers actually installed +constexpr int NHALFCRU = 72; ///< the number of half cru (link bundles) +constexpr int NLINKSPERHALFCRU = 15; ///< the number of links per half cru or cru end point. +constexpr int NLINKSPERCRU = 30; ///< the number of links per CRU (two CRUs serve one supermodule) +constexpr int NCRU = 36; ///< the number of CRU we have +constexpr int NFLP = 12; ///< the number of FLP we have. +constexpr int NCRUPERFLP = 3; ///< the number of CRU per FLP +constexpr int TRDLINKID = 15; ///< hard coded link id, specific to TRD -constexpr int NCOLUMN = 144; // the number of pad columns for each chamber -constexpr int NROWC0 = 12; // the number of pad rows for chambers of type C0 (installed stack 0,1,3 and 4) -constexpr int NROWC1 = 16; // the number of pad rows for chambers of type C1 (installed in stack 2) +constexpr int NCOLUMN = 144; ///< the number of pad columns for each chamber +constexpr int NROWC0 = 12; ///< the number of pad rows for chambers of type C0 (installed in stack 2) +constexpr int NROWC1 = 16; ///< the number of pad rows for chambers of type C1 (installed in stacks 0, 1, 3 and 4) +constexpr int FIRSTROW[NSTACK] = {0, 16, 32, 44, 60}; ///< first pad row for each stack -constexpr int NMCMROB = 16; // the number of MCMs per ROB -constexpr int NMCMHCMAX = 64; // the maximum number of MCMs for one half chamber (C1 type) -constexpr int NMCMROBINROW = 4; // the number of MCMs per ROB in row direction -constexpr int NMCMROBINCOL = 4; // the number of MCMs per ROB in column direction -constexpr int NROBC0 = 6; // the number of ROBs per C0 chamber -constexpr int NROBC1 = 8; // the number of ROBs per C1 chamber -constexpr int NADCMCM = 21; // the number of ADC channels per MCM -constexpr int NCOLMCM = 18; // the number of pads per MCM -constexpr int NCPU = 4; // the number of CPUs inside the TRAP chip -constexpr int NCHARGES = 3; // the number of charges per tracklet (Q0/1/2) +constexpr int NMCMROB = 16; ///< the number of MCMs per ROB +constexpr int NMCMHCMAX = 64; ///< the maximum number of MCMs for one half chamber (C1 type) +constexpr int NMCMROBINROW = 4; ///< the number of MCMs per ROB in row direction +constexpr int NMCMROBINCOL = 4; ///< the number of MCMs per ROB in column direction +constexpr int NROBC0 = 6; ///< the number of ROBs per C0 chamber +constexpr int NROBC1 = 8; ///< the number of ROBs per C1 chamber +constexpr int NADCMCM = 21; ///< the number of ADC channels per MCM +constexpr int NCOLMCM = 18; ///< the number of pads per MCM +constexpr int NCHANNELSPERROW = NMCMROBINCOL * 2 * NADCMCM; ///< the number of readout channels per pad row +constexpr int NCHANNELSC0 = NROWC0 * NCHANNELSPERROW; ///< the number of readout channels per C0 chamber +constexpr int NCHANNELSC1 = NROWC1 * NCHANNELSPERROW; ///< the number of readout channels per C1 chamber +constexpr int NCHANNELSTOTAL = NSECTOR * NLAYER * (NSTACK - 1) * NCHANNELSC1 + NSECTOR * NLAYER * NCHANNELSC0; ///< the total number of readout channels for TRD +constexpr int NCHANNELSPERSECTOR = NCHANNELSTOTAL / NSECTOR; ///< then number of readout channels per sector +constexpr int NCHANNELSPERLAYER = NCHANNELSPERSECTOR / NLAYER; ///< then number of readout channels per layer +constexpr int NCPU = 4; ///< the number of CPUs inside the TRAP chip +constexpr int NCHARGES = 3; ///< the number of charges per tracklet (Q0/1/2) // the values below should come out of the TRAP config in the future -constexpr int NBITSTRKLPOS = 11; // number of bits for position in tracklet64 word -constexpr int NBITSTRKLSLOPE = 8; // number of bits for slope in tracklet64 word -constexpr int ADDBITSHIFTSLOPE = 1 << 3; // in the TRAP the slope is shifted by 3 additional bits compared to the position -constexpr int PADGRANULARITYTRKLPOS = 40; // tracklet position is stored in units of 1/40 pad -constexpr int PADGRANULARITYTRKLSLOPE = 128; // tracklet deflection is stored in units of 1/128 pad per time bin -constexpr float GRANULARITYTRKLPOS = 1.f / PADGRANULARITYTRKLPOS; // granularity of position in tracklet64 word in pad-widths -constexpr float GRANULARITYTRKLSLOPE = 1.f / PADGRANULARITYTRKLSLOPE; // granularity of slope in tracklet64 word in pads/timebin -constexpr int ADCBASELINE = 10; // baseline in ADC units +constexpr int NBITSTRKLPOS = 11; ///< number of bits for position in tracklet64 word +constexpr int NBITSTRKLSLOPE = 8; ///< number of bits for slope in tracklet64 word +constexpr int ADDBITSHIFTSLOPE = 1 << 3; ///< in the TRAP the slope is shifted by 3 additional bits compared to the position +constexpr int PADGRANULARITYTRKLPOS = 40; ///< tracklet position is stored in units of 1/40 pad +constexpr int PADGRANULARITYTRKLSLOPE = 128; ///< tracklet deflection is stored in units of 1/128 pad per time bin +constexpr float GRANULARITYTRKLPOS = 1.f / PADGRANULARITYTRKLPOS; ///< granularity of position in tracklet64 word in pad-widths +constexpr float GRANULARITYTRKLSLOPE = 1.f / PADGRANULARITYTRKLSLOPE; ///< granularity of slope in tracklet64 word in pads/timebin +constexpr int ADCBASELINE = 10; ///< baseline in ADC units // OS: Should this not be flexible for example in case of Kr calib? -constexpr int TIMEBINS = 30; // the number of time bins -constexpr float MAXIMPACTANGLE = 25.f; // the maximum impact angle for tracks relative to the TRD detector plane to be considered for vDrift and ExB calibration -constexpr int NBINSANGLEDIFF = 25; // the number of bins for the track angle used for the vDrift and ExB calibration based on the tracking - -// Trigger parameters -constexpr double READOUT_TIME = 3000; // the time the readout takes, as 30 TB = 3 micro-s. -constexpr double DEAD_TIME = 200; // trigger deadtime, 2 micro-s -constexpr double BUSY_TIME = READOUT_TIME + DEAD_TIME; // the time for which no new trigger can be received in nanoseconds +constexpr int TIMEBINS = 30; ///< the number of time bins +constexpr float MAXIMPACTANGLE = 25.f; ///< the maximum impact angle for tracks relative to the TRD detector plane to be considered for vDrift and ExB calibration +constexpr int NBINSANGLEDIFF = 25; ///< the number of bins for the track angle used for the vDrift and ExB calibration based on the tracking +constexpr double VDRIFTDEFAULT = 1.546; ///< default value for vDrift +constexpr double VDRIFTMIN = 0.4; ///< min value for vDrift +constexpr double VDRIFTMAX = 2.0; ///< max value for vDrift +constexpr double EXBDEFAULT = 0.0; ///< default value for LorentzAngle +constexpr double EXBMIN = -0.4; ///< min value for LorentzAngle +constexpr double EXBMAX = 0.4; ///< max value for LorentzAngle +constexpr int NBINSGAINCALIB = 320; ///< number of bins in the charge (Q0+Q1+Q2) histogram for gain calibration +constexpr float MPVDEDXDEFAULT = 42.; ///< default Most Probable Value of TRD dEdx +constexpr float T0DEFAULT = 1.2; ///< default value for t0 // array size to store incoming half cru payload. -constexpr int HBFBUFFERMAX = 1048576; // max buffer size for data read from a half cru, (all events) -constexpr int CRUPADDING32 = 0xeeeeeeee; // padding word used in the cru. -constexpr int CHANNELNRNOTRKLT = 23; // this marks channels in the ADC mask which don't contribute to a tracklet -constexpr int NOTRACKLETFIT = 31; // this value is assigned to the fit pointer in case no tracklet is available -constexpr int TRACKLETENDMARKER = 0x10001000; // marker for the end of tracklets in raw data, 2 of these. -constexpr int PADDINGWORD = 0xeeeeeeee; // half-CRU links will be padded with this words to get an even number of 256bit words -constexpr int DIGITENDMARKER = 0x0; // marker for the end of digits in raw data, 2 of these -constexpr int MAXDATAPERLINK32 = 13824; // max number of 32 bit words per link ((21x12+2+4)*64) 64 mcm, 21 channels, 10 words per channel 2 header words(DigitMCMHeader DigitMCMADCmask) 4 words for tracklets. -constexpr int MAXDATAPERLINK256 = 1728; // max number of linkwords per cru link. (256bit words) -constexpr int MAXEVENTCOUNTERSEPERATION = 200; // how far apart can subsequent mcmheader event counters be before we flag for concern, used as a sanity check in rawreader. -constexpr int MAXMCMCOUNT = 69120; // at most mcm count maxchamber x nrobc1 nmcmrob -constexpr int MAXLINKERRORHISTOGRAMS = 10; // size of the array holding the link error plots from the raw reader -constexpr int MAXPARSEERRORHISTOGRAMS = 60; // size of the array holding the parsing error plots from the raw reader -constexpr int ETYPEPHYSICSTRIGGER = 0x2; // CRU Half Chamber header eventtype definition -constexpr int ETYPECALIBRATIONTRIGGER = 0x3; // CRU Half Chamber header eventtype definition -constexpr int MAXCRUERRORVALUE = 0x2; // Max possible value for a CRU Halfchamber link error. As of may 2022, can only be 0x0, 0x1, and 0x2, at least that is all so far(may2022). +constexpr int HBFBUFFERMAX = 1048576; ///< max buffer size for data read from a half cru, (all events) +constexpr unsigned int CRUPADDING32 = 0xeeeeeeee; ///< padding word used in the cru. +constexpr int CHANNELNRNOTRKLT = 23; ///< this marks channels in the ADC mask which don't contribute to a tracklet +constexpr int NOTRACKLETFIT = 31; ///< this value is assigned to the fit pointer in case no tracklet is available +constexpr int TRACKLETENDMARKER = 0x10001000; ///< marker for the end of tracklets in raw data, 2 of these. +constexpr int PADDINGWORD = 0xeeeeeeee; ///< half-CRU links will be padded with this words to get an even number of 256bit words +constexpr int DIGITENDMARKER = 0x0; ///< marker for the end of digits in raw data, 2 of these +constexpr int MAXDATAPERLINK32 = 13824; ///< max number of 32 bit words per link ((21x12+2+4)*64) 64 mcm, 21 channels, 10 words per channel 2 header words(DigitMCMHeader DigitMCMADCmask) 4 words for tracklets. +constexpr int MAXDATAPERLINK256 = 1728; ///< max number of linkwords per cru link. (256bit words) +constexpr int MAXEVENTCOUNTERSEPERATION = 200; ///< how far apart can subsequent mcmheader event counters be before we flag for concern, used as a sanity check in rawreader. +constexpr int MAXMCMCOUNT = 69120; ///< at most mcm count maxchamber x nrobc1 nmcmrob +constexpr int MAXLINKERRORHISTOGRAMS = 10; ///< size of the array holding the link error plots from the raw reader +constexpr int MAXPARSEERRORHISTOGRAMS = 60; ///< size of the array holding the parsing error plots from the raw reader +constexpr unsigned int ETYPEPHYSICSTRIGGER = 0x2; ///< CRU Half Chamber header eventtype definition +constexpr unsigned int ETYPECALIBRATIONTRIGGER = 0x3; ///< CRU Half Chamber header eventtype definition +constexpr int MAXCRUERRORVALUE = 0x2; ///< Max possible value for a CRU Halfchamber link error. As of may 2022, can only be 0x0, 0x1, and 0x2, at least that is all so far(may2022). +constexpr int INVALIDPRETRIGGERPHASE = 0xf; ///< Invalid value for phase, used to signify there is no hcheader. } // namespace constants } // namespace trd diff --git a/DataFormats/Detectors/TRD/include/DataFormatsTRD/Digit.h b/DataFormats/Detectors/TRD/include/DataFormatsTRD/Digit.h index 33f154254ca9a..9eba0318a5a13 100644 --- a/DataFormats/Detectors/TRD/include/DataFormatsTRD/Digit.h +++ b/DataFormats/Detectors/TRD/include/DataFormatsTRD/Digit.h @@ -38,16 +38,28 @@ using ArrayADC = std::array; // this negates the need for need alternate indexing strategies. // if you are trying to go from mcm/rob/adc to pad/row and back to mcm/rob/adc , // you may not end up in the same place, you need to remember to manually check for shared pads. +// Pre Trigger phase: +// LHC clock runs at 40.08MHz, ADC run at 1/4 or 10MHz and the trap runs at 120MHz or LHC*3. +// The trap clock can therefore be in 1 of 12 positions relative to the ADC clock. +// Only 4 of those positions are valid, dependent on the TRAP assembly program timing offset, as of 11/2022 pre-trigger phase: 0,3,6,9. +// vaguely graphically below: +// LHC ___---___---___---___---___---___---___---___---___---___---___---___---___---___---___---___---___---___---___---___---___---___---___ +// TRAP _-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_ +// ADC ___------------____________------------____________------------____________------------____________------------____________------------ +// _________------------____________------------____________------------____________------------____________------------____________------ +// ---____________------------____________------------____________------------____________------------____________------------____________ +// ---------____________------------____________------------____________------------____________------------____________------------______ +// PreTrig _________------________________________------________________________------________________________------________________________------ class Digit { public: Digit() = default; ~Digit() = default; - Digit(const int det, const int row, const int pad, const ArrayADC adc); - Digit(const int det, const int row, const int pad); // add adc data in a seperate step - Digit(const int det, const int rob, const int mcm, const int channel, const ArrayADC adc); - Digit(const int det, const int rob, const int mcm, const int channel); // add adc data in a seperate step + Digit(int det, int row, int pad, ArrayADC adc, int phase = 0); + Digit(int det, int row, int pad); // add adc data and pretrigger phase in a separate step + Digit(int det, int rob, int mcm, int channel, ArrayADC adc, int phase = 0); + Digit(int det, int rob, int mcm, int channel, int phase = 0); // add adc data // Copy Digit(const Digit&) = default; @@ -59,23 +71,30 @@ class Digit void setROB(int row, int col) { mROB = HelperMethods::getROBfromPad(row, col); } // set ROB from pad row, column void setMCM(int row, int col) { mMCM = HelperMethods::getMCMfromPad(row, col); } // set MCM from pad row, column void setChannel(int channel) { mChannel = channel; } - void setDetector(int det) { mDetector = det; } + void setDetector(int det) { mDetector = ((mDetector & 0xf000) | (det & 0xfff)); } void setADC(ArrayADC const& adc) { mADC = adc; } void setADC(const gsl::span& adc) { std::copy(adc.begin(), adc.end(), mADC.begin()); } + // set the trigger phase make sure it is mapped to 2 bits as it can only have 4 valid numbers shifted 0,3,6,9 or 1,4,7,10 etc. + void setPreTrigPhase(int phase); // Get methods - int getDetector() const { return mDetector; } - int getHCId() const { return mDetector * 2 + (mROB % 2); } + int getDetector() const { return mDetector & 0xfff; } + int getDetectorInFull() const { return mDetector; } // return the entire mDetector 16 bits, so far only for CTF encoding. + int getHCId() const { return (mDetector & 0xfff) * 2 + (mROB % 2); } int getPadRow() const { return HelperMethods::getPadRowFromMCM(mROB, mMCM); } int getPadCol() const { return HelperMethods::getPadColFromADC(mROB, mMCM, mChannel); } int getROB() const { return mROB; } int getMCM() const { return mMCM; } + int getMCMCol() const { return (getMCM() % constants::NMCMROBINCOL) + constants::NMCMROBINCOL * (getROB() % 2); } int getChannel() const { return mChannel; } + int getPreTrigPhase() const { return ((mDetector >> 12) & 0xf); } bool isSharedDigit() const; + bool isNeighbour(const Digit& other) const; ArrayADC const& getADC() const { return mADC; } ADC_t getADCsum() const { return std::accumulate(mADC.begin(), mADC.end(), (ADC_t)0); } // returns the max ADC value and sets idx to the time bin with the largest ADC value ADC_t getADCmax(int& idx) const; + ADC_t getADCval(int tb) const { return mADC[tb]; } bool operator==(const Digit& o) const { @@ -83,13 +102,18 @@ class Digit } private: - std::uint16_t mDetector{0}; // detector, the chamber [0-539] - std::uint8_t mROB{0}; // read out board within chamber [0-7] [0-5] depending on C0 or C1 - std::uint8_t mMCM{0}; // MCM chip this digit is attached [0-15] - std::uint8_t mChannel{0}; // channel of this chip the digit is attached to, see TDP chapter ?? TODO fill in later the figure number of ROB to MCM mapping picture - - ArrayADC mADC{}; // ADC vector (30 time-bins) - ClassDefNV(Digit, 3); + /// starting from ClassDef version 4, mDetector keeps both the chamber number and the trigger phase + /// + /// bits 0-11 contain the chamber number (valid range from 0-539) + /// bits 12-15 contain the trigger phase obtained from digit HC header. + /// |15|14|13|12|11|10|09|08|07|06|05|04|03|02|01|00| + /// | phase | chamber ID | + std::uint16_t mDetector{0}; + std::uint8_t mROB{0}; ///< read out board within chamber [0-7] [0-5] depending on C0 or C1 + std::uint8_t mMCM{0}; ///< MCM chip this digit is attached [0-15] + std::uint8_t mChannel{0}; ///< channel of this chip the digit is attached to, see TDP chapter ?? TODO fill in later the figure number of ROB to MCM mapping picture + ArrayADC mADC{}; ///< ADC values for 30 time bins (fixed size array) + ClassDefNV(Digit, 4); }; std::ostream& operator<<(std::ostream& stream, const Digit& d); diff --git a/DataFormats/Detectors/TRD/include/DataFormatsTRD/GainCalibHistos.h b/DataFormats/Detectors/TRD/include/DataFormatsTRD/GainCalibHistos.h new file mode 100644 index 0000000000000..6142646950969 --- /dev/null +++ b/DataFormats/Detectors/TRD/include/DataFormatsTRD/GainCalibHistos.h @@ -0,0 +1,53 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file GainCalibHistos.h +/// \brief Class to store the TRD dEdx distribution for each TRD chamber + +#ifndef ALICEO2_GAINCALIBHISTOS_H +#define ALICEO2_GAINCALIBHISTOS_H + +#include "DataFormatsTRD/Constants.h" +#include "Rtypes.h" +#include + +namespace o2 +{ +namespace trd +{ + +class GainCalibHistos +{ + public: + GainCalibHistos() = default; + GainCalibHistos(const GainCalibHistos&) = default; + ~GainCalibHistos() = default; + void reset(); + void init(); + auto getHistogramEntry(int index) const { return mdEdxEntries[index]; } + auto getNEntries() const { return mNEntriesTot; } + + void fill(const std::vector& input); + void merge(const GainCalibHistos* prev); + void print(); + + private: + std::vector mdEdxEntries{}; ///< dEdx histograms + size_t mNEntriesTot{0}; + bool mInitialized{false}; + + ClassDefNV(GainCalibHistos, 1); +}; + +} // namespace trd +} // namespace o2 + +#endif // ALICEO2_GAINCALIBHISTOS_H diff --git a/DataFormats/Detectors/TRD/include/DataFormatsTRD/HelperMethods.h b/DataFormats/Detectors/TRD/include/DataFormatsTRD/HelperMethods.h index 709c5978ab7d9..ecab7f49b92a2 100644 --- a/DataFormats/Detectors/TRD/include/DataFormatsTRD/HelperMethods.h +++ b/DataFormats/Detectors/TRD/include/DataFormatsTRD/HelperMethods.h @@ -14,6 +14,8 @@ #include "DataFormatsTRD/Constants.h" #include +#include +#include namespace o2 { @@ -54,6 +56,19 @@ struct HelperMethods { printf("%02i_%i_%i\n", det / constants::NCHAMBERPERSEC, (det % constants::NCHAMBERPERSEC) / constants::NLAYER, det % constants::NLAYER); } + static std::string getSectorStackLayerSide(int hcid) + { + int det = hcid / 2; + std::string side = (hcid % 2 == 0) ? "A" : "B"; + return fmt::format("{}_{}_{}{}", getSector(det), getStack(det), getLayer(det), side); + } + + static void printSectorStackLayerSide(int hcid) + { + // for a given half-chamber number prints SECTOR_STACK_LAYER_side + printf("%s\n", getSectorStackLayerSide(hcid).c_str()); + } + static int getPadColFromADC(int irob, int imcm, int iadc) { if (iadc < 0 || iadc > constants::NADCMCM) { @@ -149,20 +164,52 @@ struct HelperMethods { return getORIinSuperModule(hcid) + constants::NHCPERSEC * sector; } - inline static void swapByteOrder(unsigned int& word) + static int getChannelIndexInColumn(int rob, int mcm, int channel) { - word = (word >> 24) | - ((word << 8) & 0x00FF0000) | - ((word >> 8) & 0x0000FF00) | - (word << 24); + // the highest ADC channel number corresponds to the lowest pad column connected to given MCM + int mcmCol = (rob % 2) ? mcm % constants::NMCMROBINROW + constants::NMCMROBINCOL : mcm % constants::NMCMROBINROW; + return mcmCol * constants::NADCMCM + constants::NADCMCM - 1 - channel; } - inline static unsigned int swapByteOrderreturn(unsigned int word) + + static int getGlobalChannelIndex(int det, int rob, int mcm, int channel) { - // word = (word >> 24) | - // ((word << 8) & 0x00FF0000) | - // ((word >> 8) & 0x0000FF00) | - // (word << 24); - return word; + // return global readout channel index for given detector, ROB, MCM and channel index + // start with sector offset + int idx = getSector(det) * constants::NCHANNELSPERSECTOR; + // layer offset + idx += getLayer(det) * constants::NCHANNELSPERLAYER; + // stack offset + idx += (getStack(det) < 3) ? getStack(det) * constants::NCHANNELSC1 : (getStack(det) - 1) * constants::NCHANNELSC1 + constants::NCHANNELSC0; + // pad row offset + idx += getPadRowFromMCM(rob, mcm) * constants::NCHANNELSPERROW; + // position within pad column + idx += getChannelIndexInColumn(rob, mcm, channel); + + return idx; + } + + static void getPositionFromGlobalChannelIndex(int idx, int& det, int& rob, int& mcm, int& channel) + { + int sec = idx / constants::NCHANNELSPERSECTOR; + int layer = (idx % constants::NCHANNELSPERSECTOR) / constants::NCHANNELSPERLAYER; + int stackIndex = ((idx % constants::NCHANNELSPERSECTOR) % constants::NCHANNELSPERLAYER); + int stack, chamberIndex; + if (stackIndex >= 2 * constants::NCHANNELSC1 + constants::NCHANNELSC0) { + stack = 3 + (stackIndex - 2 * constants::NCHANNELSC1 - constants::NCHANNELSC0) / constants::NCHANNELSC1; + chamberIndex = (stackIndex - constants::NCHANNELSC0) % constants::NCHANNELSC1; + } else if (stackIndex > 2 * constants::NCHANNELSC1) { + stack = 2; + chamberIndex = stackIndex - 2 * constants::NCHANNELSC1; + } else { + stack = stackIndex / constants::NCHANNELSC1; + chamberIndex = stackIndex % constants::NCHANNELSC1; + } + int row = chamberIndex / constants::NCHANNELSPERROW; + int mcmCol = (chamberIndex % constants::NCHANNELSPERROW) / constants::NADCMCM; + det = getDetector(sec, stack, layer); + rob = (mcmCol >= constants::NMCMROBINCOL) ? (row / constants::NMCMROBINROW) * 2 + 1 : (row / constants::NMCMROBINROW) * 2; + mcm = (row % constants::NMCMROBINROW) * constants::NMCMROBINCOL + (mcmCol % constants::NMCMROBINCOL); + channel = constants::NADCMCM - 1 - ((chamberIndex % constants::NCHANNELSPERROW) % constants::NADCMCM); } }; diff --git a/DataFormats/Detectors/TRD/include/DataFormatsTRD/Hit.h b/DataFormats/Detectors/TRD/include/DataFormatsTRD/Hit.h index 1ae3a0dc6f8e3..b680df92c8444 100644 --- a/DataFormats/Detectors/TRD/include/DataFormatsTRD/Hit.h +++ b/DataFormats/Detectors/TRD/include/DataFormatsTRD/Hit.h @@ -13,7 +13,6 @@ #define ALICEO2_TRD_HIT_H_ #include -#include "DetectorsBase/Detector.h" #include "SimulationDataFormat/BaseHits.h" #include "CommonUtils/ShmAllocator.h" diff --git a/DataFormats/Detectors/TRD/include/DataFormatsTRD/NoiseCalibration.h b/DataFormats/Detectors/TRD/include/DataFormatsTRD/NoiseCalibration.h index 2a31bf7536dc2..f46fd8766745c 100644 --- a/DataFormats/Detectors/TRD/include/DataFormatsTRD/NoiseCalibration.h +++ b/DataFormats/Detectors/TRD/include/DataFormatsTRD/NoiseCalibration.h @@ -16,6 +16,7 @@ #include #include "DataFormatsTRD/Constants.h" #include "DataFormatsTRD/Tracklet64.h" +#include "DataFormatsTRD/Digit.h" namespace o2 { @@ -26,6 +27,41 @@ namespace trd /// \brief Simple noise status bit for each MCM of the TRD /// \author Ole Schmidt +class ChannelInfo +{ + public: + ChannelInfo() = default; + + bool isDummy() const { return mNEntries == 0; } + float getMean() const { return mMean; } + float getRMS() const { return mRMS; } + uint32_t getEntries() const { return mNEntries; } + + void setMean(float mean) { mMean = mean; } + void setRMS(float rms) { mRMS = rms; } + void setNentries(uint32_t n) { mNEntries = n; } + + private: + float mMean{0.f}; + float mRMS{0.f}; + uint32_t mNEntries{0}; + ClassDefNV(ChannelInfo, 1); +}; + +class ChannelInfoContainer +{ + public: + ChannelInfoContainer() { mData.resize(constants::NCHANNELSTOTAL); } + ChannelInfo& getChannel(int index) { return mData[index]; } + ChannelInfo getChannel(int index) const { return mData[index]; } + + const std::vector& getData() const { return mData; } + + private: + std::vector mData{}; + ClassDefNV(ChannelInfoContainer, 1); +}; + class NoiseStatusMCM { @@ -52,6 +88,7 @@ class NoiseStatusMCM bool getIsNoisy(int mcmIdxGlb) const { return mNoiseFlag.test(mcmIdxGlb); } auto getNumberOfNoisyMCMs() const { return mNoiseFlag.count(); } bool isTrackletFromNoisyMCM(const Tracklet64& trklt) const { return getIsNoisy(trklt.getHCID(), trklt.getROB(), trklt.getMCM()); } + bool isDigitFromNoisyMCM(const Digit& d) const { return getIsNoisy(d.getHCId(), d.getROB(), d.getMCM()); } private: std::bitset mNoiseFlag{}; diff --git a/DataFormats/Detectors/TRD/include/DataFormatsTRD/PHData.h b/DataFormats/Detectors/TRD/include/DataFormatsTRD/PHData.h new file mode 100644 index 0000000000000..fc46ca0207993 --- /dev/null +++ b/DataFormats/Detectors/TRD/include/DataFormatsTRD/PHData.h @@ -0,0 +1,126 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef ALICEO2_TRD_PHDATA_H_ +#define ALICEO2_TRD_PHDATA_H_ + +#include +#include "Rtypes.h" + +namespace o2::trd +{ + +/* + This data type is used to send around the information required to fill PH plots per chamber + + |31|30|29|28|27|26|25|24|23|22|21|20|19|18|17|16|15|14|13|12|11|10|09|08|07|06|05|04|03|02|01|00| + ------------------------------------------------------------------------------------------------- + |type |nNeighb | time bin | detector number | ADC sum for all neigbours | + ------------------------------------------------------------------------------------------------- +*/ + +class PHData +{ + public: + enum Origin : uint8_t { + ITSTPCTRD, + TPCTRD, + TRACKLET, + OTHER + }; + + PHData() = default; + PHData(int adc, int det, int tb, int nb, int type) { set(adc, det, tb, nb, type); } + + void set(int adc, int det, int tb, int nb, int type) + { + mData = ((type & 0x3) << 30) | ((nb & 0x7) << 27) | ((tb & 0x1f) << 22) | ((det & 0x3ff) << 12) | (adc & 0xfff); + } + + // the ADC sum for given time bin for up to three neighbours + int getADC() const { return mData & 0xfff; } + // the TRD detector number + int getDetector() const { return (mData >> 12) & 0x3ff; } + // the given time bin + int getTimebin() const { return (mData >> 22) & 0x1f; } + // number of neighbouring digits for which the ADC is accumulated + int getNneighbours() const { return (mData >> 27) & 0x7; } + // the origin of this point: digit on ITS-TPC-TRD track, ... (see enum Origin above) + int getType() const { return (mData >> 30) & 0x3; } + + private: + uint32_t mData{0}; // see comment above for data content + + ClassDefNV(PHData, 1); +}; + +/* + This data type is used to send around the information required to fill PH plots per chamber + + |19|18|17|16|15|14|13|12|11|10|09|08|07|06|05|04|03|02|01|00| + ------------------------------------------------------------- + |type |nNeighb | time bin | detector number | + ------------------------------------------------------------- +*/ +/* + This data type is used to send around the information required to fill PH plots per chamber + + |15|14|13|12|11|10|09|08|07|06|05|04|03|02|01|00| + ------------------------------------------------ + | ADC sum for all neigbours | + ------------------------------------------------ +*/ + +class PHDataHD +{ + public: + enum Origin : uint8_t { + ITSTPCTRD, + TPCTRD, + TRACKLET, + OTHER + }; + + PHDataHD() = default; + PHDataHD(int adc, int det, int tb, int nb, int type) { set(adc, det, tb, nb, type); } + + void set(int adc, int det, int tb, int nb, int type) + { + mDetector = det; + mTimeBin = tb; + mType = type; + mNNeighbours = nb; + mADC = adc; + } + + // the ADC sum for given time bin for up to three neighbours + int getADC() const { return mADC; } + // the TRD detector number + int getDetector() const { return mDetector; } + // the given time bin + int getTimebin() const { return mTimeBin; } + // number of neighbouring digits for which the ADC is accumulated + int getNNeighbours() const { return mNNeighbours; } + // the origin of this point: digit on ITS-TPC-TRD track, ... (see enum Origin above) + int getType() const { return mType; } + + private: + uint16_t mDetector{0}; + uint8_t mTimeBin{0}; + uint8_t mType{0}; + uint8_t mNNeighbours{0}; + uint16_t mADC{0}; + + ClassDefNV(PHDataHD, 1); +}; +} // namespace o2::trd + +#endif // ALICEO2_TRD_PHDATA_H_ diff --git a/DataFormats/Detectors/TRD/include/DataFormatsTRD/PID.h b/DataFormats/Detectors/TRD/include/DataFormatsTRD/PID.h new file mode 100644 index 0000000000000..d99a5be14db4a --- /dev/null +++ b/DataFormats/Detectors/TRD/include/DataFormatsTRD/PID.h @@ -0,0 +1,100 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// @file PID.h +/// @author Felix Schlepper + +#ifndef TRD_PID_H +#define TRD_PID_H + +#include +#include +#include +#include + +namespace o2 +{ +namespace trd +{ + +/// Option for available PID policies. +enum class PIDPolicy : unsigned int { + // Classical Algorithms + LQ1D = 0, ///< 1-Dimensional Likelihood model + LQ2D, ///< 2-Dimensional Likelihood model + LQ3D, ///< 3-Dimensional Likelihood model + +#ifdef TRDPID_WITH_ONNX + // ML models + XGB, ///< XGBOOST + PY, ///< Pytorch +#endif + + // Do not add anything after this! + NMODELS, ///< Count of all models + Dummy, ///< Dummy object outputting -1.f + DEFAULT = Dummy, ///< The default option +}; + +inline std::ostream& operator<<(std::ostream& os, const PIDPolicy& policy) +{ + std::string name; + switch (policy) { + case PIDPolicy::LQ1D: + name = "LQ1D"; + break; + case PIDPolicy::LQ2D: + name = "LQ2D"; + break; + case PIDPolicy::LQ3D: + name = "LQ3D"; + break; +#ifdef TRDPID_WITH_ONNX + case PIDPolicy::XGB: + name = "XGBoost"; + break; + case PIDPolicy::PY: + name = "PyTorch"; + break; +#endif + case PIDPolicy::Dummy: + name = "Dummy"; + break; + default: + name = "Default"; + } + os << name; + return os; +} + +/// Transform PID policy from string to enum. +static const std::unordered_map PIDPolicyString{ + // Classical Algorithms + {"LQ1D", PIDPolicy::LQ1D}, + {"LQ2D", PIDPolicy::LQ2D}, + {"LQ3D", PIDPolicy::LQ3D}, + +#ifdef TRDPID_WITH_ONNX + // ML models + {"XGB", PIDPolicy::XGB}, + {"PY", PIDPolicy::PY}, +#endif + + // General + {"DUMMY", PIDPolicy::Dummy}, + // Default + {"default", PIDPolicy::DEFAULT}, +}; + +} // namespace trd +} // namespace o2 + +#endif diff --git a/DataFormats/Detectors/TRD/include/DataFormatsTRD/RawData.h b/DataFormats/Detectors/TRD/include/DataFormatsTRD/RawData.h index 827b58de67f56..f7f5f96208191 100644 --- a/DataFormats/Detectors/TRD/include/DataFormatsTRD/RawData.h +++ b/DataFormats/Detectors/TRD/include/DataFormatsTRD/RawData.h @@ -17,8 +17,11 @@ // this is the header added by the CRU #include +#include #include #include +#include "DataFormatsTRD/Constants.h" +#include "Rtypes.h" namespace o2 { @@ -139,8 +142,8 @@ struct TrackletHCHeader { uint32_t word; struct { uint32_t side : 1; // side of chamber - uint32_t layer : 3; uint32_t stack : 3; + uint32_t layer : 3; uint32_t supermodule : 5; uint32_t one : 1; //always 1 uint32_t MCLK : 15; // MCM clock counter 120MHz ... for simulation -- incrementing, and uniform across an event @@ -162,9 +165,9 @@ struct TrackletMCMHeader { // uint32_t: 33222222222211111111110000000000 // 1zzzz pppppppp pppppppp1 // || yy| pppppppp | |--- 0 1 check bits - // || | | | ----------- 1-8 pid for tracklet 3 second part - // || | | -------------------- 9-16 pid for tracklet 2 second part - // || | ---------------------------- 17-24 pid for tracklet 1 second part + // || | | | ----------- 1-8 pid for cpu0 second part + // || | | -------------------- 9-16 pid for cpu1 second part + // || | ---------------------------- 17-24 pid for cpu2 second part // || ------------------------------ 25-26 col // |---------------------------------- 27-30 padrow // ----------------------------------- 31 1 @@ -173,9 +176,9 @@ struct TrackletMCMHeader { uint32_t word; struct { uint32_t oneb : 1; // - uint32_t pid0 : 8; // part of pid for tracklet 0 // 6 bits of Q2 and 2 bits of Q1 - uint32_t pid1 : 8; // part of pid for tracklet 1 - uint32_t pid2 : 8; // part of pid for tracklet 2 + uint32_t pid0 : 8; // part of pid calculated in cpu0 // 6 bits of Q2 and 2 bits of Q1 + uint32_t pid1 : 8; // part of pid calculated in cpu1 + uint32_t pid2 : 8; // part of pid calculated in cpu2 uint32_t col : 2; // 2 bits for position in pad direction. uint32_t padrow : 4; // padrow,z coordinate for chip. uint32_t onea : 1; // @@ -255,8 +258,8 @@ struct DigitHCHeader1 { uint32_t word; struct { uint32_t res : 2; - uint32_t ptrigcount : 4; uint32_t ptrigphase : 4; + uint32_t ptrigcount : 4; uint32_t bunchcrossing : 16; uint32_t numtimebins : 6; } __attribute__((__packed__)); @@ -398,20 +401,12 @@ struct trdTestPattern6 { struct DigitMCMData { // 10987654321098765432109876543210 // uint32_t: 00000000000000000000000000000000 - /* union { - uint32_t word0; - struct { - uint32_t a : 2; - uint32_t b : 5; - uint32_t adc : 21; //adc bit patternpad plane - } __attribute__((__packed__)); - };*/ union { // 10987654321098765432109876543210 // uint32_t: 00000000000000000000000000000000 uint32_t word; struct { - uint32_t c : 2; // c is wrong I cant remember name, but not a concern at the moment. + uint32_t f : 2; uint32_t z : 10; uint32_t y : 10; uint32_t x : 10; @@ -419,29 +414,28 @@ struct DigitMCMData { }; }; +struct LinkToHCIDMapping { + // for simplicity we store two maps to have one for each direction + // link ID -> half-chamber ID + // half-chamber ID -> link ID + + bool isOK() const; + int getHCID(int link) const { return linkIDToHCID.at(link); } + int getLink(int hcid) const { return hcIDToLinkID.at(hcid); } + void swapLinks(int linkA, int linkB); + + std::map linkIDToHCID; + std::map hcIDToLinkID; + ClassDefNV(LinkToHCIDMapping, 1); +}; + uint32_t setHalfCRUHeader(HalfCRUHeader& cruhead, int crurdhversion, int bunchcrossing, int stopbits, int endpoint, int eventtype, int feeid, int cruid); uint32_t setHalfCRUHeaderLinkData(HalfCRUHeader& cruhead, int link, int size, int errors); uint32_t getlinkerrorflag(const HalfCRUHeader& cruhead, const uint32_t link); uint32_t getlinkdatasize(const HalfCRUHeader& cruhead, const uint32_t link); uint32_t getlinkerrorflags(const HalfCRUHeader& cruheader, std::array& linkerrorflags); uint32_t getlinkdatasizes(const HalfCRUHeader& cruheader, std::array& linksizes); -uint32_t getQFromRaw(const o2::trd::TrackletMCMHeader* header, const o2::trd::TrackletMCMData* data, int pidindex, int trackletindex); -std::ostream& operator<<(std::ostream& stream, const TrackletHCHeader& halfchamberheader); -std::ostream& operator<<(std::ostream& stream, const TrackletMCMHeader& mcmhead); -std::ostream& operator<<(std::ostream& stream, const TrackletMCMData& tracklet); -void printTrackletMCMData(o2::trd::TrackletMCMData& tracklet); -void printTrackletMCMHeader(o2::trd::TrackletMCMHeader& mcmhead); -void printHalfChamber(o2::trd::TrackletHCHeader& halfchamber); -void dumpHalfChamber(o2::trd::TrackletHCHeader& halfchamber); -std::ostream& operator<<(std::ostream& stream, const HalfCRUHeader& halfcru); -bool trackletMCMHeaderSanityCheck(o2::trd::TrackletMCMHeader& header); -bool trackletHCHeaderSanityCheck(o2::trd::TrackletHCHeader& header); -bool digitMCMHeaderSanityCheck(o2::trd::DigitMCMHeader* header); -bool digitMCMADCMaskSanityCheck(o2::trd::DigitMCMADCMask& mask, int numberofbitsset); -bool digitMCMWordSanityCheck(o2::trd::DigitMCMData* word, int adcchannel); -bool halfCRUHeaderSanityCheck(o2::trd::HalfCRUHeader& header, std::array& lengths, std::array& eflags); -void printDigitMCMHeader(o2::trd::DigitMCMHeader& header); -int getDigitHCHeaderWordType(uint32_t word); +bool halfCRUHeaderSanityCheck(const o2::trd::HalfCRUHeader& header); void printDigitHCHeader(o2::trd::DigitHCHeader& header, uint32_t headers[3]); //functions updated/checked/new for new raw reader. @@ -453,12 +447,10 @@ void setHalfCRUHeaderLinkSizeAndFlags(HalfCRUHeader& cruhead, int link, int size DigitMCMADCMask constructBlankADCMask(); uint32_t getHalfCRULinkInfo(const HalfCRUHeader& cruhead, const uint32_t link, const bool data); -uint32_t getHalfCRULinkErrorFlag(const HalfCRUHeader& cruhead, const uint32_t link); -uint32_t getHalfCRULinkDataSize(const HalfCRUHeader& cruhead, const uint32_t link); -void getHalfCRULinkErrorFlags(const HalfCRUHeader& cruheader, std::array& linkerrorflags); -void getHalfCRULinkDataSizes(const HalfCRUHeader& cruheader, std::array& linksizes); -uint32_t getChargeFromRawHeaders(const o2::trd::TrackletHCHeader& hcheader, const o2::trd::TrackletMCMHeader* header, const std::array& data, int pidindex, int trackletindex); -uint32_t getHCIDFromTrackletHCHeader(const TrackletHCHeader& header); +uint8_t getHalfCRULinkErrorFlag(const HalfCRUHeader& cruhead, const uint32_t link); +uint16_t getHalfCRULinkDataSize(const HalfCRUHeader& cruhead, const uint32_t link); +void getHalfCRULinkErrorFlags(const HalfCRUHeader& cruheader, std::array& linkerrorflags); +void getHalfCRULinkDataSizes(const HalfCRUHeader& cruheader, std::array& linksizes); std::ostream& operator<<(std::ostream& stream, const TrackletHCHeader& halfchamberheader); std::ostream& operator<<(std::ostream& stream, const TrackletMCMHeader& tracklmcmhead); std::ostream& operator<<(std::ostream& stream, const TrackletMCMData& trackletmcmdata); @@ -468,34 +460,24 @@ std::ostream& operator<<(std::ostream& stream, const DigitMCMData& digitmcmdata) std::ostream& operator<<(std::ostream& stream, const DigitMCMADCMask& adcmask); std::ostream& operator<<(std::ostream& stream, const HalfCRUHeader& halfcru); -void printTrackletHCHeader(o2::trd::TrackletHCHeader& tracklet); -void printTrackletMCMData(o2::trd::TrackletMCMData& tracklet); -void printTrackletMCMHeader(o2::trd::TrackletMCMHeader& mcmhead); - -void printHalfChamber(o2::trd::TrackletHCHeader& digithcheader); +void printTrackletHCHeader(const o2::trd::TrackletHCHeader& tracklet); +void printTrackletMCMData(const o2::trd::TrackletMCMData& tracklet); +void printTrackletMCMHeader(const o2::trd::TrackletMCMHeader& mcmhead); -void printDigitHCHeader(o2::trd::DigitHCHeader& digitmcmheader); -void printDigitMCMData(o2::trd::DigitMCMData& digitmcmdata); -void printDigitMCMHeader(o2::trd::DigitMCMHeader& digitmcmhead); +void printDigitMCMData(const o2::trd::DigitMCMData& digitmcmdata); +void printDigitMCMHeader(const o2::trd::DigitMCMHeader& digitmcmhead); +void printDigitMCMADCMask(const o2::trd::DigitMCMADCMask& digitmcmadcmask); -void printHalfCRUHeader(o2::trd::HalfCRUHeader& halfcru); -void dumpHalfCRUHeader(o2::trd::HalfCRUHeader& halfcru); +void printHalfCRUHeader(const o2::trd::HalfCRUHeader& halfcru); void clearHalfCRUHeader(o2::trd::HalfCRUHeader& halfcru); -bool sanityCheckTrackletMCMHeader(o2::trd::TrackletMCMHeader* header); -bool sanityCheckTrackletHCHeader(o2::trd::TrackletHCHeader& header, bool verbose = false); -bool sanityCheckDigitMCMHeader(o2::trd::DigitMCMHeader* header); -bool sanityCheckDigitMCMADCMask(o2::trd::DigitMCMADCMask& mask, int numberofbitsset); -bool sanityCheckDigitMCMWord(o2::trd::DigitMCMData* word, int adcchannel); +bool sanityCheckTrackletHCHeader(const o2::trd::TrackletHCHeader& header); +bool sanityCheckTrackletMCMHeader(const o2::trd::TrackletMCMHeader& header); +bool sanityCheckDigitMCMHeader(const o2::trd::DigitMCMHeader& header); +bool sanityCheckDigitMCMADCMask(const o2::trd::DigitMCMADCMask& mask); void incrementADCMask(DigitMCMADCMask& mask, int channel); -void printDigitMCMHeader(o2::trd::DigitMCMHeader& header); int getDigitHCHeaderWordType(uint32_t word); void printDigitHCHeaders(o2::trd::DigitHCHeader& header, uint32_t headers[3], int index, int offset, bool good); void printDigitHCHeader(o2::trd::DigitHCHeader& header, uint32_t headers[3]); -int getNumberOfTrackletsFromHeader(o2::trd::TrackletMCMHeader* header, bool verbose = false); -int getNextMCMADCfromBP(uint32_t& bp, int channel); - -inline bool isTrackletHCHeader(uint32_t& header) { return (((header >> 12) & 0x1) == 0x1); } -inline bool isTrackletMCMHeader(uint32_t& header) { return ((header & 0x80000001) == 0x80000001); } } } #endif diff --git a/DataFormats/Detectors/TRD/include/DataFormatsTRD/RawDataStats.h b/DataFormats/Detectors/TRD/include/DataFormatsTRD/RawDataStats.h index c1e03257573ce..04ebd9c7eea3a 100644 --- a/DataFormats/Detectors/TRD/include/DataFormatsTRD/RawDataStats.h +++ b/DataFormats/Detectors/TRD/include/DataFormatsTRD/RawDataStats.h @@ -16,118 +16,117 @@ #ifndef O2_TRD_RAWDATASTATS #define O2_TRD_RAWDATASTATS -#include "TObject.h" -#include +#include "Rtypes.h" #include #include #include -#include -#include -#include -#include -#include +#include #include "DataFormatsTRD/Constants.h" +#include "CommonDataFormat/TFIDInfo.h" namespace o2::trd { -enum ParsingErrors { TRDParsingNoError, - TRDParsingUnrecognisedVersion, - TRDParsingBadDigt, - TRDParsingBadTracklet, - TRDParsingDigitEndMarkerWrongState, // read a end marker but we were expecting something else due to - TRDParsingDigitMCMHeaderSanityCheckFailure, //essentially we did not see an MCM header see RawData.h for requirement - TRDParsingDigitROBDecreasing, // sequential headers must have the same or increasing rob number - TRDParsingDigitMCMNotIncreasing, // sequential headers must have increasing mcm number - TRDParsingDigitADCMaskMismatch, // mask adc count does not match # of 1s in bitpattern - TRDParsingDigitADCMaskAdvanceToEnd, // in advancing to adcmask we have reached the end of the buffer - TRDParsingDigitMCMHeaderBypassButStateMCMHeader, // we are reading mcmadc data but the state is mcmheader - TRDParsingDigitEndMarkerStateButReadingMCMADCData, // read the endmarker while expecting to read the mcmadcdata - TRDParsingDigitADCChannel21, // ADCMask is zero but we are still on a digit. - TRDParsingDigitADCChannelGT22, // error allocating digit, so digit channel has error value - TRDParsingDigitGT10ADCs, // more than 10 adc data words seen - TRDParsingDigitSanityCheck, // adc failed sanity check see RawData.cxx for faiulre reasons - TRDParsingDigitExcessTimeBins, // ADC has more than 30 timebins (10 adc words) - TRDParsingDigitParsingExitInWrongState, // exiting parsing in the wrong state ... got to the end of the buffer in wrong state. - TRDParsingDigitStackMismatch, // mismatch between rdh and hcheader stack calculation/value - TRDParsingDigitLayerMismatch, // mismatch between rdh and hcheader stack calculation/value - TRDParsingDigitSectorMismatch, // mismatch between rdh and hcheader stack calculation/value - TRDParsingTrackletCRUPaddingWhileParsingTracklets, // reading a padding word while expecting tracklet data - TRDParsingTrackletBit11NotSetInTrackletHCHeader, // bit 11 not set in hc header for tracklets. - TRDParsingTrackletHCHeaderSanityCheckFailure, // HCHeader sanity check failure, see RawData.cxx for reasons. - TRDParsingTrackletMCMHeaderSanityCheckFailure, // MCMHeader sanity check failure, see RawData.cxx for reasons. - TRDParsingTrackletMCMHeaderButParsingMCMData, // state is still MCMHeader but we are parsing MCMData - TRDParsingTrackletStateMCMHeaderButParsingMCMData, - TRDParsingTrackletTrackletCountGTThatDeclaredInMCMHeader, //mcmheader tracklet count does not match that in we have parsed. - TRDParsingTrackletInvalidTrackletCount, // invalid tracklet count in header vs data - TRDParsingTrackletPadRowIncreaseError, // subsequent padrow can not be less than previous one. - TRDParsingTrackletColIncreaseError, // subsequent col can not be less than previous one - TRDParsingTrackletNoTrackletEndMarker, // got to the end of the buffer with out finding a tracklet end marker. - TRDParsingTrackletExitingNoTrackletEndMarker, // got to the end of the buffer exiting tracklet parsing with no tracklet end marker - TRDParsingDigitHeaderCountGT3, // digital half chamber header had more than 3 additional words expected by header. most likely corruption above somewhere. - TRDParsingDigitHeaderWrong1, // expected header word1 but wrong ending marker - TRDParsingDigitHeaderWrong2, // expected header word2 but wrong ending marker - TRDParsingDigitHeaderWrong3, // expected header word3 but wrong ending marker - TRDParsingDigitHeaderWrong4, // expected header word but have no idea what we are looking at default of switch statement - TRDParsingDigitDataStillOnLink, // got to the end of digit parsing and there is still data on link, normally not advancing far enough when dumping data. - TRDParsingTrackletIgnoringDataTillEndMarker, // for some reason we are bouncing to the end word by word, this counts those words - TRDParsingGarbageDataAtEndOfHalfCRU, // if the first word of the halfcru is wrong i.e. side, eventype, the half cru header is so wrong its not corrupt, its other garbage - TRDParsingHalfCRUSumLength, // if the HalfCRU headers summed lengths wont fit into the buffer, implies corruption, its a faster check than the next one. - TRDParsingBadRDHFEEID, // RDH parsing failure for reasons in the word - TRDParsingBadRDHEndPoint, // RDH parsing failure for reasons in the word - TRDParsingBadRDHOrbit, // RDH parsing failure for reasons in the word - TRDParsingBadRDHCRUID, // RDH parsing failure for reasons in the word - TRDParsingBadRDHPacketCounter, // RDH parsing failure for reasons in the word - TRDParsingHalfCRUCorrupt, // if the HalfCRU headers has values out of range, corruption is assumed. - TRDParsingDigitHCHeader1, // multiple instances of Digit HC Header 1 - TRDParsingDigitHCHeader2, // multiple instances of Digit HC Header 2 - TRDParsingDigitHCHeader3, // multiple instances of Digit HC Header 3 - TRDProcessingBadPayloadOrOffset, // if something is off with the HBFPayload array or its offset into it. - TRDParsingDigitHCHeaderSVNMismatch, // svn version information has changed in the DigitHCHeader3. - TRDParsingBadLinkstartend, // end - start of tracklet is greater than the maximal length stored in the cru half chamber header field. - TRDParsingTrackletsReturnedMinusOne, // trackletparsing returned -1, data was dumped; - TRDFEEIDIsFFFF, // RDH is in error, the FEEID is 0xffff - TRDFEEIDBadSector, // RDH is in error, the FEEID.supermodule is not a valid value. - TRDParsingDigitHCHeaderPreTriggerPhaseOOB, // pretrigger phase in Digit HC header has to be less than 12, it is not. - TRDParsingHalfCRUBadBC, // saw a bc below the L0 trigger - TRDLastParsingError +enum ParsingErrors { + NoError, + DigitEndMarkerWrongState, // read a end marker but we were expecting something else + DigitMCMHeaderSanityCheckFailure, // the checked bits in the DigitMCMHeader were not correctly set + DigitMCMNotIncreasing, // sequential headers must have increasing mcm number + DigitMCMDuplicate, // we saw two DigitMCMHeaders for the same MCM in one trigger + DigitADCMaskInvalid, // mask adc count does not match # of 1s in bitpattern or the check bits are wrongly set + DigitSanityCheck, // adc failed sanity check based on current channel (odd/even) and check bits DigitMCMData.f + DigitParsingExitInWrongState, // exiting parsing in the wrong state ... got to the end of the buffer in wrong state. + DigitParsingNoSecondEndmarker, // we found a single digit end marker not followed by a second one + DigitHCHeaderMismatch, // the half-chamber ID from the digit HC header is not consistent with the one expected from the link ID + TrackletHCHeaderFailure, // either reserved bit not set or HCID is not what was expected from RDH + TrackletMCMHeaderSanityCheckFailure, // MCMHeader sanity check failure, LSB or MSB not set + TrackletDataWrongOrdering, // the tracklet data is not arriving in increasing MCM order + TrackletDataDuplicateMCM, // we see more than one TrackletMCMHeader for the same MCM + TrackletNoTrackletEndMarker, // got to the end of the buffer with out finding a tracklet end marker. + TrackletNoSecondEndMarker, // we expected to see a second tracklet end marker, but found something else instead + TrackletMCMDataFailure, // invalid word for TrackletMCMData detected + TrackletDataMissing, // we expected tracklet data but got an endmarker instead + TrackletExitingNoTrackletEndMarker, // got to the end of the buffer exiting tracklet parsing with no tracklet end marker + UnparsedTrackletDataRemaining, // the tracklet parsing has finished correctly, but there is still data left on the link (CRU puts incorrect link size or corrupt data?) + UnparsedDigitDataRemaining, // the digit parsing has finished correctly, but there is still data left on the link (CRU puts incorrect link size or corrupt data? RDH > 8kByte before?) + DigitHeaderCountGT3, // digital half chamber header had more than 3 additional words expected by header. most likely corruption above somewhere. + DigitHeaderWrongType, // expected digit header, but could not determine type + HalfCRUSumLength, // if the HalfCRU headers summed lengths wont fit into the buffer, implies corruption, its a faster check than the next one. + BadRDHMemSize, // RDH memory size is supposedly zero + BadRDHFEEID, // RDH parsing failure for reasons in the word + BadRDHEndPoint, // RDH parsing failure for reasons in the word + BadRDHOrbit, // RDH parsing failure for reasons in the word + BadRDHCRUID, // RDH parsing failure for reasons in the word + BadRDHPacketCounter, // RDH packet counter not incrementing + HalfCRUCorrupt, // if the HalfCRU headers has values out of range, corruption is assumed. + DigitHCHeader1Problem, // multiple instances of Digit HC Header 1 + DigitHCHeader2Problem, // multiple instances of Digit HC Header 2 + DigitHCHeader3Problem, // multiple instances of Digit HC Header 3 + DigitHCHeaderSVNMismatch, // svn version information has changed in the DigitHCHeader3. + TrackletsReturnedMinusOne, // trackletparsing returned -1, data was dumped; + FEEIDIsFFFF, // RDH is in error, the FEEID is 0xffff + FEEIDBadSector, // RDH is in error, the FEEID.supermodule is not a valid value. + HalfCRUBadBC, // the BC in the half-CRU header is so low that the BC shift would make it negative + TRDLastParsingError // This is to keep QC happy until we can change it there as well. }; -extern std::vector ParsingErrorsString; +static const std::unordered_map ParsingErrorsString = { + {NoError, "NoError"}, + {DigitEndMarkerWrongState, "DigitEndMarkerWrongState"}, + {DigitMCMHeaderSanityCheckFailure, "DigitMCMHeaderSanityCheckFailure"}, + {DigitMCMNotIncreasing, "DigitMCMNotIncreasing"}, + {DigitMCMDuplicate, "DigitMCMDuplicate"}, + {DigitADCMaskInvalid, "DigitADCMaskInvalid"}, + {DigitSanityCheck, "DigitSanityCheck"}, + {DigitParsingExitInWrongState, "DigitParsingExitInWrongState"}, + {DigitParsingNoSecondEndmarker, "DigitParsingNoSecondEndmarker"}, + {DigitHCHeaderMismatch, "DigitHCHeaderMismatch"}, + {TrackletHCHeaderFailure, "TrackletHCHeaderFailure"}, + {TrackletMCMHeaderSanityCheckFailure, "TrackletMCMHeaderSanityCheckFailure"}, + {TrackletDataWrongOrdering, "TrackletDataWrongOrdering"}, + {TrackletDataDuplicateMCM, "TrackletDataDuplicateMCM"}, + {TrackletNoTrackletEndMarker, "TrackletNoTrackletEndMarker"}, + {TrackletNoSecondEndMarker, "TrackletNoSecondEndMarker"}, + {TrackletMCMDataFailure, "TrackletMCMDataFailure"}, + {TrackletDataMissing, "TrackletDataMissing"}, + {TrackletExitingNoTrackletEndMarker, "TrackletExitingNoTrackletEndMarker"}, + {UnparsedTrackletDataRemaining, "UnparsedTrackletDataRemaining"}, + {UnparsedDigitDataRemaining, "UnparsedDigitDataRemaining"}, + {DigitHeaderCountGT3, "DigitHeaderCountGT3"}, + {DigitHeaderWrongType, "DigitHeaderWrongType"}, + {HalfCRUSumLength, "HalfCRUSumLength"}, + {BadRDHMemSize, "BadRDHMemSize"}, + {BadRDHFEEID, "BadRDHFEEID"}, + {BadRDHEndPoint, "BadRDHEndPoint"}, + {BadRDHOrbit, "BadRDHOrbit"}, + {BadRDHCRUID, "BadRDHCRUID"}, + {BadRDHPacketCounter, "BadRDHPacketCounter"}, + {HalfCRUCorrupt, "HalfCRUCorrupt"}, + {DigitHCHeader1Problem, "DigitHCHeader1Problem"}, + {DigitHCHeader2Problem, "DigitHCHeader2Problem"}, + {DigitHCHeader3Problem, "DigitHCHeader3Problem"}, + {DigitHCHeaderSVNMismatch, "DigitHCHeaderSVNMismatch"}, + {TrackletsReturnedMinusOne, "TrackletsReturnedMinusOne"}, + {FEEIDIsFFFF, "FEEIDIsFFFF"}, + {FEEIDBadSector, "FEEIDBadSector"}, + {HalfCRUBadBC, "HalfCRUBadBC"}, + {TRDLastParsingError, "TRDLastParsingError"}}; //enumerations for the options, saves on having a long parameter list. enum OptionBits { - TRDByteSwapBit, TRDVerboseBit, - TRDHeaderVerboseBit, - TRDDataVerboseBit, - TRDCompressedDataBit, - TRDFixDigitCorruptionBit, - TRDEnableTimeInfoBit, - TRDEnableStatsBit, - TRDIgnoreDigitHCHeaderBit, - TRDIgnoreTrackletHCHeaderBit, - TRDEnableRootOutputBit, + TRDVerboseErrorsBit, TRDIgnore2StageTrigger, TRDGenerateStats, - TRDM1Debug -}; + TRDOnlyCalibrationTriggerBit, + TRDSortDigits, + TRDLinkStats +}; // this is currently 16 options, the array is 16, if you add here you need to change the 16; -//Data to be stored and accumulated on an event basis. -//events are spread out with in the data coming in with a halfcruheader per event, per ... half cru. -//this is looked up via the interaction record (orbit and bunchcrossing). -//this permits averaging in the data that gets senton per timeframe -class TRDDataCountersPerEvent -{ - public: - //TODO this should go into a dpl message for catching by qc ?? I think. - double mTimeTaken; // time take to process an event (summed trackletparsing and digitparsing) parts not accounted for. - double mTimeTakenForDigits; // time take to process tracklet data blocks [us]. - double mTimeTakenForTracklets; // time take to process digit data blocks [us]. - uint64_t mWordsRead; // words read in - uint64_t mWordsRejected; // words skipped for various reasons. - uint16_t mTrackletsFound; // tracklets found in the event - uint16_t mDigitsFound; // digits found in the event +struct DataCountersPerTrigger { + std::array mLinkWords{}; + std::array mLinkErrorFlag{}; + ClassDefNV(DataCountersPerTrigger, 1); }; //Data to be stored on a timeframe basis to then be sent as a message to be ultimately picked up by qc. @@ -135,47 +134,42 @@ class TRDDataCountersPerEvent class TRDDataCountersPerTimeFrame { public: - std::array mLinkErrorFlag{}; // status of the error flags for this timeframe, 8bit values from cru halfchamber header. - std::array mLinkNoData; // Link had no data or was not present. - std::array mLinkWords{}; // units of 256bits, read from the cru half chamber header - std::array mLinkWordsRead{}; // units of 32 bits the data words read before dumping or finishing - std::array mLinkWordsRejected{}; // units of 32 bits the data dumped due to some or other error + std::array mLinkErrorFlag{}; // status of the error flags for this timeframe, 8bit values from cru halfchamber header. + std::array mLinkNoData{}; // Link had no data or was not present. + std::array mLinkWords{}; // units of 256bits, read from the cru half chamber header + std::array mLinkWordsRead{}; // units of 32 bits the data words read before dumping or finishing + std::array mLinkWordsRejected{}; // units of 32 bits the data dumped due to some or other error + std::array mParsingOK{}; // count how often given link could be parsed without any errors std::array mParsingErrors{}; // errors in parsing, indexed by enum above of ParsingErrors - std::array mParsingErrorsByLink{}; // errors in parsing, indexed by enum above of ParsingErrors - uint16_t mDigitsPerEvent; // average digits found per event in this timeframe, ignoring the no digit events where there is no calibration trigger. - uint16_t mTrackletsPerEvent; // average tracklets found per event in this timeframe - double mTimeTaken; // time taken to process the entire timeframe [ms]. - double mTimeTakenForDigits; // time take to process tracklet data blocks [ms]. - double mTimeTakenForTracklets; // time take to process digit data blocks [ms]. + std::vector mParsingErrorsByLink{}; // each entry is for a single parsing error on a given link (HCID * number of Errors + error index) + float mTimeTaken; // time taken to process all half-CRU data blocks combined [us]. + float mTimeTakenForDigits; // time take to process tracklet data blocks [us]. + float mTimeTakenForTracklets; // time take to process digit data blocks [us]. uint32_t mDigitsFound; // digits found in the time frame. uint32_t mTrackletsFound; // tracklets found in the time frame. - std::array mDataFormatRead{}; // We just keep the major version number + uint16_t mNTriggersCalib; // number of triggers with digit readout + uint16_t mNTriggersTotal; // total number of triggers + std::array mDataFormatRead{}; // We just keep the major version number + o2::dataformats::TFIDInfo mTFIDInfo; // keep track of TF ID void clear() { mLinkNoData.fill(0); mLinkWords.fill(0); mLinkWordsRead.fill(0); mLinkWordsRejected.fill(0); + mParsingOK.fill(0); mParsingErrors.fill(0); - mParsingErrorsByLink.fill(0); - mDigitsPerEvent = 0; - mTrackletsPerEvent = 0; + mParsingErrorsByLink.clear(); mTimeTaken = 0; mTimeTakenForDigits = 0; mTimeTakenForTracklets = 0; mDigitsFound = 0; - mTrackletsFound = 0; //tracklets found in timeframe. + mTrackletsFound = 0; + mNTriggersCalib = 0; + mNTriggersTotal = 0; mDataFormatRead.fill(0); }; - ClassDefNV(TRDDataCountersPerTimeFrame, 1); // primarily for serialisation so we can send this as a message in o2 -}; - -//TODO not sure this class is needed -class TRDDataCountersRunning -{ //those counters that keep counting - std::array mLinkFreq{}; //units of 256bits "cru word" - std::array mLinkEmpty{}; // Link only has padding words only, probably not serious. - std::array mDataFormatRead{}; // 7bits.7bits major.minor version read from HCHeader. + ClassDefNV(TRDDataCountersPerTimeFrame, 3); // primarily for serialisation so we can send this as a message in o2 }; } // namespace o2::trd diff --git a/DataFormats/Detectors/TRD/include/DataFormatsTRD/RecoInputContainer.h b/DataFormats/Detectors/TRD/include/DataFormatsTRD/RecoInputContainer.h index 21a079ec0df77..032dd4162a785 100644 --- a/DataFormats/Detectors/TRD/include/DataFormatsTRD/RecoInputContainer.h +++ b/DataFormats/Detectors/TRD/include/DataFormatsTRD/RecoInputContainer.h @@ -27,7 +27,7 @@ #include "Framework/InputRecord.h" #include "SimulationDataFormat/MCTruthContainer.h" -#include "GPUDataTypes.h" +#include "GPUDataTypesIO.h" #include #include @@ -88,7 +88,7 @@ inline void RecoInputContainer::fillGPUIOPtr(o2::gpu::GPUTrackingInOutPointers* ptrs->nTRDTriggerRecords = mNTriggerRecords; ptrs->trdTriggerTimes = &(trdTriggerTimes[0]); ptrs->trdTrackletIdxFirst = &(trdTriggerIndices[0]); - ptrs->trdTrigRecMask = reinterpret_cast(mTrigRecMask.data()); + ptrs->trdTrigRecMask = reinterpret_cast(mTrigRecMask.data()); ptrs->nTRDTracklets = mNTracklets; ptrs->trdTracklets = reinterpret_cast(mTracklets.data()); ptrs->trdSpacePoints = reinterpret_cast(mSpacePoints.data()); diff --git a/DataFormats/Detectors/TRD/include/DataFormatsTRD/T0FitHistos.h b/DataFormats/Detectors/TRD/include/DataFormatsTRD/T0FitHistos.h new file mode 100644 index 0000000000000..93448696025c9 --- /dev/null +++ b/DataFormats/Detectors/TRD/include/DataFormatsTRD/T0FitHistos.h @@ -0,0 +1,56 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file T0FitHistos.h +/// \brief Class to store the TRD PH values for each TRD chamber + +#ifndef ALICEO2_T0FITHISTOS_H +#define ALICEO2_T0FITHISTOS_H + +#include "DataFormatsTRD/Constants.h" +#include "DataFormatsTRD/PHData.h" +#include "Rtypes.h" +#include +#include + +namespace o2 +{ +namespace trd +{ + +class T0FitHistos +{ + public: + T0FitHistos() = default; + T0FitHistos(const T0FitHistos&) = default; + ~T0FitHistos() = default; + auto getDetector(int index) const { return mDet[index]; } + auto getTimeBin(int index) const { return mTB[index]; } + auto getADC(int index) const { return mADC[index]; } + auto getNEntries() const { return mNEntriesTot; } + + void fill(const std::vector& data); + void merge(const T0FitHistos* prev); + void print(); + + private: + std::vector mDet{}; + std::vector mTB{}; + std::vector mADC{}; + size_t mNEntriesTot{0}; + + ClassDefNV(T0FitHistos, 1); +}; + +} // namespace trd +} // namespace o2 + +#endif // ALICEO2_T0FITHISTOS_H diff --git a/DataFormats/Detectors/TRD/include/DataFormatsTRD/Tracklet64.h b/DataFormats/Detectors/TRD/include/DataFormatsTRD/Tracklet64.h index 8d734640f576a..e63d8fbb5f277 100644 --- a/DataFormats/Detectors/TRD/include/DataFormatsTRD/Tracklet64.h +++ b/DataFormats/Detectors/TRD/include/DataFormatsTRD/Tracklet64.h @@ -9,9 +9,9 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. -//#include "TRDBase/GeometryBase.h" -//#include "DetectorsCommonDataFormats/DetMatrixCache.h" -//#include "DetectorsCommonDataFormats/DetID.h" +// #include "TRDBase/GeometryBase.h" +// #include "DetectorsCommonDataFormats/DetMatrixCache.h" +// #include "DetectorsCommonDataFormats/DetID.h" #ifndef O2_TRD_TRACKLET64_H #define O2_TRD_TRACKLET64_H @@ -27,8 +27,12 @@ // //////////////////////////////////////////////////////////////////////////// +#ifndef GPUCA_GPUCODE_DEVICE +#include +#endif #include "GPUCommonDef.h" #include "GPUCommonRtypes.h" +#include "GPUCommonMath.h" #include "DataFormatsTRD/Constants.h" namespace o2 @@ -84,15 +88,73 @@ class Tracklet64 GPUdDefault() ~Tracklet64() = default; GPUdDefault() Tracklet64& operator=(const Tracklet64& rhs) = default; + GPUd() bool operator==(const Tracklet64& o) const { return mtrackletWord == o.mtrackletWord; } + + // ----- Getters ----- + GPUd() uint64_t getTrackletWord() const { return mtrackletWord; } + + // position / direction information + + // half-chamber number [0..1079] + GPUd() int getHCID() const { return ((mtrackletWord & hcidmask) >> hcidbs); }; + + // chamber number [0..539] + GPUd() int getDetector() const { return getHCID() / 2; } + + // pad row number on ROB [0..15] + GPUd() int getPadRow() const { return ((mtrackletWord & padrowmask) >> padrowbs); }; + + // MCM position in column direction on ROB [0..3] + GPUd() int getColumn() const { return ((mtrackletWord & colmask) >> colbs); }; + + // in units of 1/40 pads, 11 bit granularity (MSP is used to indicate sign) + GPUd() int getPosition() const { return ((mtrackletWord & posmask) >> posbs); }; + + // applies two's complement to getPosition() to obtain signed value + GPUd() int getPositionBinSigned() const; - // ----- Getters for contents of tracklet word ----- - GPUd() int getFormat() const { return ((mtrackletWord & formatmask) >> formatbs); }; // no units 0..15 - GPUd() int getHCID() const { return ((mtrackletWord & hcidmask) >> hcidbs); }; // no units 0..1079 - GPUd() int getPadRow() const { return ((mtrackletWord & padrowmask) >> padrowbs); }; // pad row number [0..15] - GPUd() int getColumn() const { return ((mtrackletWord & colmask) >> colbs); }; // column refers to MCM position in column direction on readout board [0..3] - GPUd() int getPosition() const { return ((mtrackletWord & posmask) >> posbs); }; // in units of 1/40 pads, 11 bit granularity - GPUd() int getSlope() const { return ((mtrackletWord & slopemask) >> slopebs); }; // in units of 1/1000 pads/timebin, 8 bit granularity - GPUd() int getPID() const { return ((mtrackletWord & PIDmask)); }; // no unit, all 3 charge windows combined + // the position as float in unit of pads relative to MCM center + GPUd() float getPositionFloat() const { return getPositionBinSigned() * constants::GRANULARITYTRKLPOS; } + + // in units of 1/128 pads/timebin, 8 bit granularity (MSP is used to indicate sign) + GPUd() int getSlope() const { return ((mtrackletWord & slopemask) >> slopebs); }; + + // applies two's complement to getSlope() to obtain signed value + GPUd() int getSlopeBinSigned() const; + + // as float in units of pads per time bin + GPUd() float getSlopeFloat() const { return getSlopeBinSigned() * constants::GRANULARITYTRKLSLOPE / constants::ADDBITSHIFTSLOPE; } + + // MCM position on ROB [0..15] + GPUd() int getMCM() const { return constants::NMCMROBINROW * (getPadRow() % constants::NMCMROBINCOL) + getColumn(); } + + // ROB number [0..5] for C0 chamber and [0..7] for C1 chamber + GPUd() int getROB() const { return (getHCID() % 2) ? (getPadRow() / constants::NMCMROBINROW) * 2 + 1 : (getPadRow() / constants::NMCMROBINROW) * 2; } + + // MCM number in pad column direction [0..7] + GPUd() int getMCMCol() const { return (getMCM() % constants::NMCMROBINCOL) + constants::NMCMROBINCOL * (getROB() % 2); } + + // pad width in cm required for transformation from pad coordinates to tracking coordinates + GPUd() float getPadWidth() const { return 0.635f + 0.03f * (getDetector() % constants::NLAYER); } + + // pad column number inside pad row as float + // FIXME: understand why the offset seems to be 8 pads and not nChannels / 2 = 10.5 + // Due to wrong pad shift included in alignment we need to optionally shift the tracklets by one pad + // in case we are not using the ideal alignment + GPUd() float getPadColFloat(bool applyShift) const { return getPositionFloat() + getMCMCol() * constants::NCOLMCM + 8.f + (applyShift ? 1.f : 0.f); } + + // pad column number inside pad row as int can be off by +-1 pad (same function name as for TRD digit) + GPUd() int getPadCol(bool applyShift = false) const { return o2::gpu::CAMath::Float2IntRn(getPadColFloat(applyShift)); } + + // translate local position into global y (in cm) not taking into account calibrations (ExB, vDrift, t0) + GPUd() float getUncalibratedY(bool applyShift = false) const { return (getPadColFloat(applyShift) - (constants::NCOLUMN / 2.f)) * getPadWidth(); } + + // translate local slope into dy/dx with dx=3m (drift length) and default drift time in time bins (19.4 timebins / 3cm) + GPUd() float getUncalibratedDy(float nTbDrift = 19.4f) const { return getSlopeFloat() * getPadWidth() * nTbDrift; } + + // PID related getters + GPUd() int getFormat() const { return ((mtrackletWord & formatmask) >> formatbs); }; + GPUd() int getPID() const { return ((mtrackletWord & PIDmask)); }; GPUd() int getDynamicCharge(unsigned int charge) const { int shift = (charge >> 6) & 0x3; @@ -129,54 +191,14 @@ class Tracklet64 } }; // no unit + // ----- Setters for tracklet word manipulation ----- GPUd() void setTrackletWord(uint64_t trackletword) { mtrackletWord = trackletword; } - - // ----- Getters for tracklet information ----- - GPUd() int getMCM() const { return 4 * (getPadRow() % 4) + getColumn(); } // returns MCM position on ROB [0..15] - GPUd() int getROB() const { return (getHCID() % 2) ? (getPadRow() / 4) * 2 + 1 : (getPadRow() / 4) * 2; } // returns ROB number [0..5] for C0 chamber and [0..7] for C1 chamber - GPUd() int getPositionBinSigned() const; - GPUd() int getSlopeBinSigned() const; - GPUd() float getUncalibratedY() const; // translate local position into global y (in cm) not taking into account calibrations (ExB, vDrift, t0) - GPUd() float getUncalibratedDy(float nTbDrift = 19.4f) const; // translate local slope into dy/dx with dx=3m (drift length) and default drift time in time bins (19.4 timebins / 3cm) - - // ----- Getters for offline corresponding values ----- - GPUd() int getDetector() const { return getHCID() / 2; } - - GPUd() uint64_t getTrackletWord() const { return mtrackletWord; } - - GPUd() void setQ0(int charge) - { - mtrackletWord &= ~Q0mask; - mtrackletWord |= ((charge << Q0bs) & Q0mask); - } - GPUd() void setQ1(int charge) - { - mtrackletWord &= ~Q1mask; - mtrackletWord |= ((charge << Q1bs) & Q1mask); - } - GPUd() void setQ2(int charge) - { - mtrackletWord &= ~Q2mask; - mtrackletWord |= ((charge << Q2bs) & Q2mask); - } - GPUd() void setPID(uint64_t pid) - { - // set the entire pid area of the trackletword, all the 3 Q's - mtrackletWord &= ~PIDmask; - mtrackletWord |= ((pid << PIDbs) & PIDmask); - } - GPUd() void setPosition(uint64_t position) - { - mtrackletWord &= ~posmask; - mtrackletWord |= ((position << posbs) & posmask); - } - GPUd() void setSlope(uint64_t slope) - { - mtrackletWord &= ~slopemask; - mtrackletWord |= ((slope << slopebs) & slopemask); - } - - GPUd() bool operator==(const Tracklet64& o) const { return mtrackletWord == o.mtrackletWord; } + GPUd() void setQ0(int charge) { mtrackletWord = (mtrackletWord & ~Q0mask) | ((charge & Q0mask) << Q0bs); } + GPUd() void setQ1(int charge) { mtrackletWord = (mtrackletWord & ~Q1mask) | ((charge & Q1mask) << Q1bs); } + GPUd() void setQ2(int charge) { mtrackletWord = (mtrackletWord & ~Q2mask) | ((charge & Q2mask) << Q2bs); } + GPUd() void setPID(int pid) { mtrackletWord = (mtrackletWord & ~PIDmask) | ((pid & PIDmask) << PIDbs); } + GPUd() void setPosition(int position) { mtrackletWord = (mtrackletWord & ~posmask) | ((position & posmask) << posbs); } + GPUd() void setSlope(int slope) { mtrackletWord = (mtrackletWord & ~slopemask) | ((slope & slopemask) << slopebs); } GPUd() void print() const; #ifndef GPUCA_GPUCODE_DEVICE @@ -194,7 +216,7 @@ class Tracklet64 static constexpr uint64_t Q1mask = 0x000000000000ff00; static constexpr uint64_t Q0mask = 0x00000000000000ff; static constexpr uint64_t PIDmask = 0x0000000000ffffff; - //bit shifts for the above raw data + // bit shifts for the above raw data static constexpr uint64_t formatbs = 60; static constexpr uint64_t hcidbs = 49; static constexpr uint64_t padrowbs = 45; @@ -224,17 +246,6 @@ GPUdi() int Tracklet64::getPositionBinSigned() const return padLocal; } -GPUdi() float Tracklet64::getUncalibratedY() const -{ - int padLocal = getPositionBinSigned(); - int mcmCol = (getMCM() % constants::NMCMROBINCOL) + constants::NMCMROBINCOL * (getROB() % 2); - // one pad column has 144 pads, the offset of -63 is the center of the first MCM in that column - // which is connected to the pads -63 - 9 = -72 to -63 + 9 = -54 - float offset = -63.f + ((float)constants::NCOLMCM) * mcmCol; - float padWidth = 0.635f + 0.03f * (getDetector() % constants::NLAYER); - return (offset + padLocal * constants::GRANULARITYTRKLPOS) * padWidth; -} - GPUdi() int Tracklet64::getSlopeBinSigned() const { int slopeBin = getSlope(); @@ -247,18 +258,11 @@ GPUdi() int Tracklet64::getSlopeBinSigned() const return -slope; } -GPUdi() float Tracklet64::getUncalibratedDy(float nTbDrift) const -{ - int slope = getSlopeBinSigned(); - float padWidth = 0.635f + 0.03f * (getDetector() % constants::NLAYER); - return slope * constants::GRANULARITYTRKLSLOPE * padWidth * nTbDrift / constants::ADDBITSHIFTSLOPE; -} - #ifndef GPUCA_GPUCODE_DEVICE std::ostream& operator<<(std::ostream& stream, const Tracklet64& trg); #endif // GPUCA_GPUCODE_DEVICE -} //namespace trd -} //namespace o2 +} // namespace trd +} // namespace o2 #endif diff --git a/DataFormats/Detectors/TRD/src/AngularResidHistos.cxx b/DataFormats/Detectors/TRD/src/AngularResidHistos.cxx index 706f7cecaf412..dfee36f944723 100644 --- a/DataFormats/Detectors/TRD/src/AngularResidHistos.cxx +++ b/DataFormats/Detectors/TRD/src/AngularResidHistos.cxx @@ -31,14 +31,14 @@ bool AngularResidHistos::addEntry(float deltaAlpha, float impactAngle, int chamb // add entry for given angular residual // returns 0 in case of success (impact angle is in valid range) int chamberOffset = chamberId * NBINSANGLEDIFF; - if (std::fabs(impactAngle) >= MAXIMPACTANGLE) { - LOG(debug) << "Under-/overflow entry detected for impact angle " << impactAngle; - return 1; - } else { + if (std::fabs(impactAngle) < MAXIMPACTANGLE) { int iBin = (impactAngle + MAXIMPACTANGLE) * INVBINWIDTH; mHistogramEntries[chamberOffset + iBin] += deltaAlpha; ++mNEntriesPerBin[chamberOffset + iBin]; ++mNEntriesTotal; + } else { + LOG(debug) << "Under-/overflow entry detected for impact angle " << impactAngle; + return 1; } return 0; } @@ -52,12 +52,6 @@ void AngularResidHistos::fill(const AngularResidHistos& input) } } -void AngularResidHistos::fill(const gsl::span input) -{ - (void)input; - LOG(fatal) << "This function must not be called. But it must be available for the compilation to work"; -} - void AngularResidHistos::merge(const AngularResidHistos* prev) { for (int i = 0; i < MAXCHAMBER * NBINSANGLEDIFF; ++i) { diff --git a/DataFormats/Detectors/TRD/src/DataFormatsTRDLinkDef.h b/DataFormats/Detectors/TRD/src/DataFormatsTRDLinkDef.h index 919f8f94cd2af..c6d36a7aee495 100644 --- a/DataFormats/Detectors/TRD/src/DataFormatsTRDLinkDef.h +++ b/DataFormats/Detectors/TRD/src/DataFormatsTRDLinkDef.h @@ -23,6 +23,7 @@ #pragma link C++ struct o2::trd::TrackletHCHeader + ; #pragma link C++ struct o2::trd::TrackletMCMHeader + ; #pragma link C++ struct o2::trd::TrackletMCMData + ; +// TODO add also digit data types and ClassDefNV etc... #pragma link C++ class o2::trd::Tracklet64 + ; #pragma link C++ class o2::trd::CalibratedTracklet + ; #pragma link C++ class o2::trd::Hit + ; @@ -31,10 +32,20 @@ #pragma link C++ class o2::trd::KrClusterTriggerRecord + ; #pragma link C++ class o2::trd::NoiseStatusMCM + ; #pragma link C++ class o2::trd::AngularResidHistos + ; +#pragma link C++ class o2::trd::GainCalibHistos + ; +#pragma link C++ class o2::trd::T0FitHistos + ; #pragma link C++ class o2::trd::CalVdriftExB + ; +#pragma link C++ class o2::trd::CalGain + ; #pragma link C++ class o2::trd::CalT0 + ; #pragma link C++ class o2::trd::CompressedDigit + ; #pragma link C++ class o2::trd::HelperMethods + ; +#pragma link C++ class o2::trd::LinkToHCIDMapping + ; +#pragma link C++ class o2::trd::ChannelInfo + ; +#pragma link C++ class o2::trd::ChannelInfoContainer + ; +#pragma link C++ struct o2::trd::PHData + ; +#pragma link C++ struct o2::trd::PHDataHD + ; +#pragma link C++ class o2::trd::TRDDataCountersPerTimeFrame + ; +#pragma link C++ class o2::trd::DataCountersPerTrigger + ; #pragma link C++ class std::vector < o2::trd::Tracklet64> + ; #pragma link C++ class std::vector < o2::trd::CalibratedTracklet> + ; #pragma link C++ class std::vector < o2::trd::TrackTriggerRecord> + ; @@ -43,8 +54,13 @@ #pragma link C++ class std::vector < o2::trd::Hit > +; #pragma link C++ class std::vector < o2::trd::Digit> + ; #pragma link C++ class std::vector < o2::trd::AngularResidHistos> + ; +#pragma link C++ class std::vector < o2::trd::GainCalibHistos> + ; +#pragma link C++ class std::vector < o2::trd::T0FitHistos> + ; +#pragma link C++ class std::vector < o2::trd::PHData> + ; +#pragma link C++ class std::vector < o2::trd::PHDataHD> + ; #pragma link C++ class std::vector < o2::trd::KrCluster> + ; #pragma link C++ class std::vector < o2::trd::KrClusterTriggerRecord> + ; +#pragma link C++ class std::vector < o2::trd::DataCountersPerTrigger> + ; #pragma link C++ struct o2::trd::CTFHeader + ; #pragma link C++ struct o2::trd::CTF + ; diff --git a/DataFormats/Detectors/TRD/src/Digit.cxx b/DataFormats/Detectors/TRD/src/Digit.cxx index 11133c78ddc64..37d6638ac0996 100644 --- a/DataFormats/Detectors/TRD/src/Digit.cxx +++ b/DataFormats/Detectors/TRD/src/Digit.cxx @@ -12,22 +12,24 @@ #include "DataFormatsTRD/Digit.h" #include #include +#include "fairlogger/Logger.h" namespace o2::trd { using namespace constants; -Digit::Digit(const int det, const int row, const int pad, const ArrayADC adc) +Digit::Digit(int det, int row, int pad, ArrayADC adc, int pretrigphase) { setDetector(det); setROB(row, pad); setMCM(row, pad); setADC(adc); setChannel(NADCMCM - 2 - (pad % NCOLMCM)); + setPreTrigPhase(pretrigphase); } -Digit::Digit(const int det, const int row, const int pad) // add adc data in a seperate step +Digit::Digit(int det, int row, int pad) // add adc data in a seperate step { setDetector(det); setROB(row, pad); @@ -35,21 +37,28 @@ Digit::Digit(const int det, const int row, const int pad) // add adc data in a s setChannel(NADCMCM - 2 - (pad % NCOLMCM)); } -Digit::Digit(const int det, const int rob, const int mcm, const int channel, const ArrayADC adc) +Digit::Digit(int det, int rob, int mcm, int channel, ArrayADC adc, int pretrigphase) { setDetector(det); setROB(rob); setMCM(mcm); setChannel(channel); setADC(adc); + setPreTrigPhase(pretrigphase); } -Digit::Digit(const int det, const int rob, const int mcm, const int channel) // add adc data in a seperate step +Digit::Digit(int det, int rob, int mcm, int channel, int pretrigphase) // add adc data in a seperate step { setDetector(det); setROB(rob); setMCM(mcm); setChannel(channel); + setPreTrigPhase(pretrigphase); +} + +void Digit::setPreTrigPhase(int phase) +{ + mDetector = ((((phase) & 0x3) << 12) | (mDetector & 0xfff)); } bool Digit::isSharedDigit() const @@ -61,6 +70,11 @@ bool Digit::isSharedDigit() const } } +bool Digit::isNeighbour(const Digit& other) const +{ + return (getDetector() == other.getDetector() && getROB() == other.getROB() && getMCM() == other.getMCM() && std::abs(getChannel() - other.getChannel()) == 1); +} + ADC_t Digit::getADCmax(int& idx) const { auto itMax = std::max_element(mADC.begin(), mADC.end()); diff --git a/DataFormats/Detectors/TRD/src/GainCalibHistos.cxx b/DataFormats/Detectors/TRD/src/GainCalibHistos.cxx new file mode 100644 index 0000000000000..5580c03d40c5c --- /dev/null +++ b/DataFormats/Detectors/TRD/src/GainCalibHistos.cxx @@ -0,0 +1,62 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file GainCalibHistos.cxx +/// \brief Class to store the output of the global tracking based TRD calibration + +#include "DataFormatsTRD/GainCalibHistos.h" +#include +#include + +using namespace o2::trd; +using namespace o2::trd::constants; + +void GainCalibHistos::init() +{ + mdEdxEntries.resize(constants::MAXCHAMBER * constants::NBINSGAINCALIB, 0); + mInitialized = true; +} + +void GainCalibHistos::reset() +{ + if (!mInitialized) { + init(); + } + std::fill(mdEdxEntries.begin(), mdEdxEntries.end(), 0); + mNEntriesTot = 0; +} + +void GainCalibHistos::fill(const std::vector& input) +{ + if (!mInitialized) { + init(); + } + for (auto elem : input) { + ++mdEdxEntries[elem]; + } + mNEntriesTot += input.size(); +} + +void GainCalibHistos::merge(const GainCalibHistos* prev) +{ + if (!mInitialized) { + init(); + } + for (int i = 0; i < MAXCHAMBER * NBINSGAINCALIB; ++i) { + mdEdxEntries[i] += prev->getHistogramEntry(i); + mNEntriesTot += prev->getHistogramEntry(i); + } +} + +void GainCalibHistos::print() +{ + LOG(info) << "There are " << mNEntriesTot << " entries in the container"; +} diff --git a/DataFormats/Detectors/TRD/src/RawData.cxx b/DataFormats/Detectors/TRD/src/RawData.cxx index 644044f1f46d8..8453cd55e4dc4 100644 --- a/DataFormats/Detectors/TRD/src/RawData.cxx +++ b/DataFormats/Detectors/TRD/src/RawData.cxx @@ -11,11 +11,14 @@ #include #include +#include #include "fairlogger/Logger.h" #include "DataFormatsTRD/RawData.h" #include "DataFormatsTRD/LinkRecord.h" -#include "DataFormatsTRD/Constants.h" #include "DataFormatsTRD/Tracklet64.h" +#include "DataFormatsTRD/HelperMethods.h" + +using namespace o2::trd::constants; namespace o2 { @@ -23,6 +26,32 @@ namespace o2 namespace trd { +bool LinkToHCIDMapping::isOK() const +{ + for (int linkIn = 0; linkIn < MAXHALFCHAMBER; ++linkIn) { + int hcid = getHCID(linkIn); + if (linkIn != getLink(hcid)) { + return false; + } + } + return true; +} + +// linkA and linkB refer to the global ORI index and not to the half-chamber ID +void LinkToHCIDMapping::swapLinks(int linkA, int linkB) +{ + int hcidA = HelperMethods::getHCIDFromLinkID(linkA); + int hcidB = HelperMethods::getHCIDFromLinkID(linkB); + linkIDToHCID.erase(linkA); + linkIDToHCID.insert({linkA, hcidB}); + linkIDToHCID.erase(linkB); + linkIDToHCID.insert({linkB, hcidA}); + hcIDToLinkID.erase(hcidA); + hcIDToLinkID.insert({hcidA, linkB}); + hcIDToLinkID.erase(hcidB); + hcIDToLinkID.insert({hcidB, linkA}); +} + // // Printing methods to dump and display the various structures above in pretty format or hexdump // printNameOfStruct(const NameOfStruct& nameofstruct); @@ -58,17 +87,6 @@ std::ostream& operator<<(std::ostream& stream, const TrackletMCMHeader& mcmhead) return stream; } -void dumpHalfCRUHeader(o2::trd::HalfCRUHeader& halfcru) -{ - std::array raw{}; - memcpy(&raw[0], &halfcru, sizeof(halfcru)); - for (int i = 0; i < 2; i++) { - int index = 4 * i; - LOGF(info, "[1/2CRUHeader %d] 0x%08x 0x%08x 0x%08x 0x%08x", i, raw[index + 3], raw[index + 2], - raw[index + 1], raw[index + 0]); - } -} - //functions updated/checked/new for new raw reader. above methods left for cross checking what changes have occured. // construct a tracklet half chamber header according to the tdp and assembler found in @@ -94,23 +112,23 @@ void setHalfCRUHeaderLinkSizeAndFlags(HalfCRUHeader& cruhead, int link, int size cruhead.errorflags[link].errorflag = errors; } -uint32_t getHalfCRULinkErrorFlag(const HalfCRUHeader& cruhead, const uint32_t link) +uint8_t getHalfCRULinkErrorFlag(const HalfCRUHeader& cruhead, const uint32_t link) { // link is the link you are requesting information on, 0-14 - uint32_t errorflag = 0; + uint8_t errorflag = 0; //dealing with word0-2 errorflag = cruhead.errorflags[link].errorflag; return errorflag; } -uint32_t getHalfCRULinkDataSize(const HalfCRUHeader& cruhead, const uint32_t link) +uint16_t getHalfCRULinkDataSize(const HalfCRUHeader& cruhead, const uint32_t link) { // link is the link you are requesting information on, 0-14 //return number 32 byte blocks for the link 3x64bit ints. return cruhead.datasizes[link].size; } -void getHalfCRULinkErrorFlags(const HalfCRUHeader& cruheader, std::array& linkerrorflags) +void getHalfCRULinkErrorFlags(const HalfCRUHeader& cruheader, std::array& linkerrorflags) { // retrieve all the link error flags for this half cru for (uint32_t link = 0; link < 15; link++) { @@ -118,7 +136,7 @@ void getHalfCRULinkErrorFlags(const HalfCRUHeader& cruheader, std::array& linksizes) +void getHalfCRULinkDataSizes(const HalfCRUHeader& cruheader, std::array& linksizes) { // retrieve all the link error flags for this half cru for (uint32_t link = 0; link < 15; link++) { @@ -152,9 +170,9 @@ std::ostream& operator<<(std::ostream& stream, const HalfCRUHeader& halfcru) void constructTrackletHCHeader(TrackletHCHeader& header, int hcid, int chipclock, int format) { int detector = hcid / 2; - int sector = (detector % (constants::NLAYER * constants::NSTACK)); - int stack = (detector % constants::NLAYER); - int layer = ((detector % (constants::NLAYER * constants::NSTACK)) / constants::NLAYER); + int sector = HelperMethods::getSector(detector); + int stack = HelperMethods::getStack(detector); + int layer = HelperMethods::getLayer(detector); int side = hcid % 2; header.word = 0; header.format = format; @@ -166,90 +184,6 @@ void constructTrackletHCHeader(TrackletHCHeader& header, int hcid, int chipclock header.one = 1; } -uint32_t getHCIDFromTrackletHCHeader(const TrackletHCHeader& header) -{ - return header.layer * 2 + header.stack * constants::NLAYER * 2 + header.supermodule * constants::NLAYER * constants::NSTACK * 2 + header.side; -} - -uint32_t getChargeFromRawHeaders(const o2::trd::TrackletHCHeader& hcheader, const o2::trd::TrackletMCMHeader* header, const std::array& data, int pidindex, int trackletindex) -{ - uint32_t pid = 0; - uint32_t highPID = 0; // highPID holds the 8 bits from the mcmheader - uint32_t lowPID = 0; // lowPID holds the 12 bits from mcmdata - uint32_t datatype = (hcheader.format) >> 2; - switch (datatype) { - case 0: //Cosmic - // LOG(warn) << "This is a problem cosmic format tracklets "; - //break; - case 1: //TPT - //LOG(warn) << "This is a problem TPT format tracklets "; - //break; - case 2: //DIS - //LOG(warn) << "This is a problem DIS format tracklets "; - //break; - case 3: - //PID VERSION 1 - //PID is 20 bits, 8 bits in mcmheader and 12 bits in mcmdata word - //frist part of pid (highPID) is in the TrackletMCMHeader - //highPID is 7 bits Q2, 1 bit Q1 OR ... 2 bit offset, 6 bits Q2. - switch (trackletindex) { - case 0: - highPID = header->pid0; - break; - case 1: - highPID = header->pid1; - break; - case 2: - highPID = header->pid2; - break; - default: - LOG(warn) << " unknown trackletindex of " << trackletindex << " to " << __func__ << " : " << pidindex; - break; - } - lowPID = data[trackletindex].pid; - //lowPID is 6 bits Q0 and 6 bits of Q1 - uint32_t pidword = (highPID << 12) | lowPID; // the entire original 20 bit pid in the trap chips - int dynamicq = hcheader.format & 0x1; // last bit of format (lsb) defines the version of tracklet charge calculation - uint32_t pidoffset = ((pidword >> 18) & 0x3); //<<6; // used for dynamic ranged charge windows, may or may not be used below. - //pidword is here to make this code more readible and less error prone. - switch (pidindex) { - case 2: //Q2 - if (!dynamicq) { - pid = (pidword >> 14) & 0x3f; // 6 bits at the top of all of pid (MSB) - } else { - pid = (pidword >> 12) & 0x3f; // 6 bits of Q2 and a shift - pid |= pidoffset << 6; - // LOG(info) << "Q2 pid : " << std::hex << pid << " pidoffset: " << pidoffset; - } - break; - case 1: //Q1 - if (!dynamicq) { - pid = (pidword >> 7) & 0x7f; // 7 bits Q1 above the 7 bits of Q0 - } else { - pid = (pidword >> 6) & 0x3f; // 6 bits of Q1 and a shift - pid |= pidoffset << 6; - //LOG(info) << "Q1 pid : " << std::hex << pid << " pidoffset: " << pidoffset;; - } - break; - case 0: //Q0 - if (!dynamicq) { - pid = pidword & 0x7f; // 7 least significant bits - } else { - pid = pidword & 0x3f; // 6 bits of Q0 - pid |= pidoffset << 6; - // LOG(info) << "Q0 pid : " << std::hex << pid << " pidoffset: " << pidoffset; - } - break; - default: - LOG(warn) << " unknown pid index of : " << pidindex; - break; - } - } // end of case of various formats. - return pid; -} - -//Tracklet MCM Header - uint16_t constructTRDFeeID(int supermodule, int side, int endpoint) { TRDFeeID feeid; @@ -275,36 +209,34 @@ DigitMCMADCMask constructBlankADCMask() return mask; } -void printTrackletHCHeader(o2::trd::TrackletHCHeader& halfchamber) +void printTrackletHCHeader(const o2::trd::TrackletHCHeader& halfchamber) { LOGF(info, "TrackletHCHeader: Raw:0x%08x SM : %d stack %d layer %d side : %d MCLK: 0x%0x Format: 0x%0x Always1:0x%0x", halfchamber.word, (int)(~halfchamber.supermodule) & 0x1f, (int)(~halfchamber.stack) & 0x7, (int)(~halfchamber.layer) & 0x7, (int)(~halfchamber.side) & 0x1, (int)halfchamber.MCLK, (int)halfchamber.format, (int)halfchamber.one); } -void printTrackletMCMData(o2::trd::TrackletMCMData& tracklet) +void printTrackletMCMData(const o2::trd::TrackletMCMData& tracklet) { LOGF(info, "TrackletMCMData: Raw:0x%08x pos:%d slope:%d pid:0x%03x checkbit:0x%02x", tracklet.word, tracklet.pos, tracklet.slope, tracklet.pid, tracklet.checkbit); } -void printTrackletMCMHeader(o2::trd::TrackletMCMHeader& mcmhead) +void printTrackletMCMHeader(const o2::trd::TrackletMCMHeader& mcmhead) { - LOG(info) << " about to print mcm raw header"; LOGF(info, "MCMRawHeader: Raw:0x%08x 1:%d padrow: 0x%02x col: 0x%01x pid2 0x%02x pid1: 0x%02x pid0: 0x%02x 1:%d", mcmhead.word, mcmhead.onea, mcmhead.padrow, mcmhead.col, mcmhead.pid2, mcmhead.pid1, mcmhead.pid0, mcmhead.oneb); - LOG(info) << " printed mcm raw header"; } -void printHalfCRUHeader(o2::trd::HalfCRUHeader& halfcru) +void printHalfCRUHeader(const o2::trd::HalfCRUHeader& halfcru) { - std::array sizes; - std::array errorflags; + std::array sizes; + std::array errorflags; getHalfCRULinkDataSizes(halfcru, sizes); getHalfCRULinkErrorFlags(halfcru, errorflags); LOGF(info, "V:%d BC:%d SB:%d EType:%d", halfcru.HeaderVersion, halfcru.BunchCrossing, halfcru.StopBit, halfcru.EventType); for (int i = 0; i < 15; i++) { - LOGF(info, "Link %d size: %ul eflag: 0x%02x", i, sizes[i], errorflags[i]); + LOGF(info, "Link %d size: %lu eflag: 0x%02x", i, sizes[i], errorflags[i]); } LOG(info) << "Raw: " << std::hex << halfcru.word0 << " " << halfcru.word12[0] << " " << halfcru.word12[1] << " " << halfcru.word3 << " " << halfcru.word47[0] << " " << halfcru.word47[1] << " " << halfcru.word47[2] << " " << halfcru.word47[3]; } @@ -314,132 +246,78 @@ void clearHalfCRUHeader(o2::trd::HalfCRUHeader& halfcru) std::memset(&halfcru, 0, sizeof(o2::trd::HalfCRUHeader)); } -bool sanityCheckTrackletMCMData(o2::trd::TrackletMCMData& data) -{ - bool gooddata = true; - if (data.checkbit == 0) { - gooddata = false; - } - /* - if(data.slope< o2::trd::constants::MinTrackletSlope || data.slope> o2::trd::MaxTrackletSlope){ - gooddata = false; - } - if(data.pos< o2::trd::constants::MinTrackletPos|| data.pos> o2::trd::constants::MinTrackletPos){ - gooddata = false; - } - if(data.pid){ - - } - */ - return gooddata; -} - -bool halfCRUHeaderSanityCheck(o2::trd::HalfCRUHeader& header, std::array& lengths, std::array& eflags) +bool halfCRUHeaderSanityCheck(const o2::trd::HalfCRUHeader& header) { // check the sizes for less than max value // check the errors for either < 0x3, for now (may 2022) there is only no error, 1, or 2. // - bool goodheader = true; - for (int lengthindex = 0; lengthindex < 15; ++lengthindex) { - if (lengths[lengthindex] > o2::trd::constants::MAXDATAPERLINK256) { - // something has gone insane. - goodheader = false; - } - } - for (int eflagindex = 0; eflagindex < 15; ++eflagindex) { - if (eflags[eflagindex] > o2::trd::constants::MAXCRUERRORVALUE) { - // something has gone insane. - goodheader = false; + for (int link = 0; link < 15; ++link) { + if (header.datasizes[link].size > MAXDATAPERLINK256) { + return false; } - if (header.EndPoint > 1) { - // end point can only be zero or 1, for ach of the 2 pci end points in the cru - goodheader = false; + if (header.errorflags[link].errorflag > MAXCRUERRORVALUE) { + return false; } } - - return goodheader; + if (header.EndPoint > 1) { + // end point can only be zero or 1, for each of the 2 pci end points in the cru + return false; + } + return true; } -bool sanityCheckTrackletMCMHeader(o2::trd::TrackletMCMHeader* header) +bool sanityCheckTrackletHCHeader(const o2::trd::TrackletHCHeader& header) { - // a bit limited to what we can check. - bool goodheader = true; - if (header->onea != 1) { - goodheader = false; + if (header.one != 1) { + return false; } - if (header->oneb != 1) { - goodheader = false; + if (((~header.supermodule) & 0x1f) >= NSECTOR) { + return false; } - // if we have 3rd tracklet (pid2!=0) then we must have all the others as well. - if ((header->pid2 != 0xff) && (header->pid1 == 0xff || header->pid0 == 0xff)) { - goodheader = false; + if (((~header.stack) & 0x7) >= NSTACK) { + return false; } - // sim for 2 tracklets. - if ((header->pid1 != 0xff) && (header->pid0 == 0xff)) { - goodheader = false; + if (((~header.layer) & 0x7) >= NLAYER) { + return false; } - return goodheader; + return true; } -bool sanityCheckTrackletHCHeader(o2::trd::TrackletHCHeader& header, bool verbose) +bool sanityCheckTrackletMCMHeader(const o2::trd::TrackletMCMHeader& header) { - bool goodheader = true; - if ((~header.supermodule) > 17) { - if (verbose) { - LOG(info) << " TrackletHCHeader : 0x" << std::hex << header.word << " failure header.supermodule=" << ~header.supermodule; - } - goodheader = false; - } - if ((~header.layer) > 6) { - if (verbose) { - LOG(info) << " TrackletHCHeader : 0x" << std::hex << header.word << " failure header.layer=" << ~header.layer; - } - goodheader = false; - } - if ((~header.stack) > 5) { - if (verbose) { - LOG(info) << " TrackletHCHeader : 0x" << std::hex << header.word << " failure header.stack=" << ~header.stack; - } - goodheader = false; + // a bit limited to what we can check. + if (header.onea != 1) { + return false; } - if (header.one != 1) { - if (verbose) { - LOG(info) << " TrackletHCHeader : 0x" << std::hex << header.word << " failure header.one=" << header.one; - } - goodheader = false; + if (header.oneb != 1) { + return false; } - return goodheader; + return true; } -bool sanityCheckDigitMCMHeader(o2::trd::DigitMCMHeader* header) +bool sanityCheckDigitMCMHeader(const o2::trd::DigitMCMHeader& header) { // a bit limited to what we can check. - bool goodheader = true; - if (header->res != 0xc) { - goodheader = false; + if (header.res != 0xc) { + return false; } - if (header->yearflag == 0) { //we only have data after 2007 now in run3. - goodheader = false; + if (header.yearflag == 0) { // we only have data after 2007 now in run3. + return false; } - return goodheader; + return true; } -bool sanityCheckDigitMCMADCMask(o2::trd::DigitMCMADCMask& mask, int numberofbitsset) +bool sanityCheckDigitMCMADCMask(const o2::trd::DigitMCMADCMask& mask) { - bool goodadcmask = true; - uint32_t count = (unsigned int)mask.c; - count = (~count) & 0x1f; - if (count != numberofbitsset) { - goodadcmask=false; - LOG(info) << "***DigitMCMADCMask bad bit count maskcount:" << ((~mask.c) & 0x1f) << " bitscounting:" << numberofbitsset << " bp: 0x" << std::hex << mask.adcmask; - } if (mask.n != 0x1) { - goodadcmask = false; + return false; } if (mask.j != 0xc) { - goodadcmask = false; + return false; } - return goodadcmask; + unsigned int counter = (~mask.c) & 0x1f; + std::bitset headerMask(mask.adcmask); + return (counter == headerMask.count()); } void incrementADCMask(DigitMCMADCMask& mask, int channel) @@ -450,62 +328,37 @@ void incrementADCMask(DigitMCMADCMask& mask, int channel) mask.c = ~((bitcount)&0x1f); } -bool sanityCheckDigitMCMWord(o2::trd::DigitMCMData* word, int adcchannel) -{ - bool gooddata = true; - // DigitMCMWord0x3 is odd 10 for odd adc channels and 11 for even, counted as the first of the 3. - switch (word->c) { - case 3: // even adc channnel - if (adcchannel % 2 == 0) { - gooddata = true; - } else { - gooddata = false; - } - break; - case 2: // odd adc channel - if (adcchannel % 2 == 1) { - gooddata = true; - } else { - gooddata = false; - } - break; - case 1: // error - gooddata = false; - break; - case 0: // error - gooddata = false; - break; - // no default all cases taken care of - } - return gooddata; -} - -void printDigitMCMHeader(o2::trd::DigitMCMHeader& digitmcmhead) +void printDigitMCMHeader(const o2::trd::DigitMCMHeader& digitmcmhead) { - LOGF(info, "DigitMCMRawHeader: Raw:0x%08x res(0xc):0x%02x mcm: 0x%03x rob: 0x%03x eventcount 0x%05x year(>2007?): 0x%02x ", + LOGF(info, "DigitMCMHeader: Raw:0x%08x, res: 0x%02x mcm: 0x%x rob: 0x%x eventcount 0x%05x year(>2007?): 0x%x ", digitmcmhead.word, digitmcmhead.res, digitmcmhead.mcm, digitmcmhead.rob, digitmcmhead.eventcount, digitmcmhead.yearflag); } -void printDigitMCMData(o2::trd::DigitMCMData& digitmcmdata) +void printDigitMCMData(const o2::trd::DigitMCMData& digitmcmdata) +{ + LOGF(info, "DigitMCMRawData: Raw:0x%08x res:0x%x x: 0x%03x y: 0x%03x z 0x%03x ", + digitmcmdata.word, digitmcmdata.f, digitmcmdata.x, digitmcmdata.y, digitmcmdata.z); +} +void printDigitMCMADCMask(const o2::trd::DigitMCMADCMask& digitmcmadcmask) { - LOGF(info, "DigitMCMRawData: Raw:0x%08x res(0xc):0x%02x x: 0x%04x y: 0x%04x z 0x%04x ", - digitmcmdata.word, digitmcmdata.c, digitmcmdata.x, digitmcmdata.y, digitmcmdata.z); + LOGF(info, "DigitMCMADCMask: Raw:0x%08x j(0xc):0x%01x mask: 0x%05x count: 0x%02x n(0x1) 0x%01x ", + digitmcmadcmask.word, digitmcmadcmask.j, digitmcmadcmask.adcmask, digitmcmadcmask.c, digitmcmadcmask.n); } int getDigitHCHeaderWordType(uint32_t word) { - // LOG(info) << "getDigitHCHeaderwordtype : " << std::hex << word; + // all digit HC headers end with the bit pattern 01 + // the bits 5..2 are used to distinguish the different header types + // the bit patterns 0b11001 and 0b110101 are outside the valid range + // for the header type 1 if ((word & 0x3f) == 0b110001) { - // LOG(info) << "getDigitHCHeaderwordtype 2 : " << std::hex << word << " returning 2 for :" << std::hex << (word&0x3f); return 2; } if ((word & 0x3f) == 0b110101) { - // LOG(info) << "getDigitHCHeaderwordtype 3 : " << std::hex << word << " returning 3 for :" << std::hex << (word&0x3f); return 3; } if ((word & 0x3) == 0b01) { - // LOG(info) << "getDigitHCHeaderwordtype 1 : " << std::hex << word << " returning 1 for :" << std::hex << (word&0x3f); return 1; } return -1; @@ -540,9 +393,8 @@ void printDigitHCHeaders(o2::trd::DigitHCHeader& header, uint32_t headers[3], in void printDigitHCHeader(o2::trd::DigitHCHeader& header, uint32_t headers[3]) { printDigitHCHeaders(header, headers, -1, 0, true); - int countheaderwords = header.numberHCW; int index; - //for the currently 3 implemeented other header words, they can come in any order, and are identified by their reserved portion + //for the currently 3 implemented other header words, they can come in any order, and are identified by their reserved portion for (int countheaderwords = 0; countheaderwords < header.numberHCW; ++countheaderwords) { switch (getDigitHCHeaderWordType(headers[countheaderwords])) { case 1: @@ -579,63 +431,5 @@ void printDigitHCHeader(o2::trd::DigitHCHeader& header, uint32_t headers[3]) } } -int getNumberOfTrackletsFromHeader(o2::trd::TrackletMCMHeader* header, bool verbose) -{ - int headertrackletcount = 0; - if (header->pid0 == 0xff) { - headertrackletcount = 0; - } else { - if (header->pid2 != 0xff) { - // 3 tracklets - headertrackletcount = 3; - if (header->pid1 == 0xff || header->pid0 == 0xff) { - if (verbose) { - LOG(warn) << __func__ << " 3 tracklets but first 2 are empty ??" << header; - } - } - } else { - if (header->pid1 != 0xff) { - // 2 tracklets - headertrackletcount = 2; - if (header->pid0 == 0xff) { - if (verbose) { - LOG(warn) << __func__ << " 2 tracklets but first is empty ??" << header; - } - } - } else { - if (header->pid0 != 0xff) { - // 1 tracklet - headertrackletcount = 1; - } else { - if (verbose) { - LOG(warn) << __func__ << " 1 tracklet but first is empty ?? so should be no tracklets " << header; - } - } - } - } - } - return headertrackletcount; -} - -int getNextMCMADCfromBP(uint32_t& bp, int channel) -{ - //given a bitpattern (adcmask) find next channel with in the mask starting from the current channel. - //channels are read from right to left, lsb to msb. channel zero is position 0 in the bit pattern. - if (bp == 0) { - return 22; - } - int position = channel; - int m = 1 << channel; - while (!(bp & m)) { - m = m << 1; - position++; - if (position > 21) { - break; - } - } - bp &= ~(1UL << (position)); - return position; -} - } // namespace trd } // namespace o2 diff --git a/DataFormats/Detectors/TRD/src/RawDataStats.cxx b/DataFormats/Detectors/TRD/src/RawDataStats.cxx index 4766481ce6997..09e383e74a79b 100644 --- a/DataFormats/Detectors/TRD/src/RawDataStats.cxx +++ b/DataFormats/Detectors/TRD/src/RawDataStats.cxx @@ -21,36 +21,3 @@ // Primarily to QC and debug graphs built into the readers. // This file is primarily for the strings that appear at titles in the graphs. -std::vector ParsingErrorsString{"TRDParsingNoError", - "TRDParsingUnrecognisedVersion", - "TRDParsingBadDigt", - "TRDParsingBadTracklet", - "TRDParsingDigitEndMarkerWrongState", - "TRDParsingDigitMCMHeaderSanityCheckFailure", - "TRDParsingDigitROBDecreasing", - "TRDParsingDigitMCMNotIncreasing", - "TRDParsingDigitADCMaskMismatch", - "TRDParsingDigitADCMaskAdvanceToEnd", - "TRDParsingDigitMCMHeaderBypassButStateMCMHeader", - "TRDParsingDigitEndMarkerStateButReadingMCMADCData", - "TRDParsingDigitADCChannel21", - "TRDParsingDigitADCChannelGT22", - "TRDParsingDigitGT10ADCs", - "TRDParsingDigitSanityCheck", - "TRDParsingDigitExcessTimeBins", - "TRDParsingDigitParsingExitInWrongState", - "TRDParsingDigitStackMisMatch", - "TRDParsingDigitLayerMisMatch", - "TRDParsingDigitSectorMisMatch", - "TRDParsingTrackletCRUPaddingWhileParsingTracklets", - "TRDParsingTrackletBit11NotSetInTrackletHCHeader", - "TRDParsingTrackletHCHeaderSanityCheckFailure", - "TRDParsingTrackletMCMHeaderSanityCheckFailure", - "TRDParsingTrackletMCMHeaderButParsingMCMData", - "TRDParsingTrackletStateMCMHeaderButParsingMCMData", - "TRDParsingTrackletTrackletCountGTThatDeclaredInMCMHeader", - "TRDParsingTrackletInvalidTrackletCount", - "TRDParsingTrackletPadRowIncreaseError", - "TRDParsingTrackletColIncreaseError", - "TRDParsingTrackletNoTrackletEndMarker", - "TRDParsingTrackletExitingNoTrackletEndMarker"}; diff --git a/DataFormats/Detectors/TRD/src/T0FitHistos.cxx b/DataFormats/Detectors/TRD/src/T0FitHistos.cxx new file mode 100644 index 0000000000000..623f61b761188 --- /dev/null +++ b/DataFormats/Detectors/TRD/src/T0FitHistos.cxx @@ -0,0 +1,54 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file T0FitHistos.cxx +/// \brief Class to store the TRD PH values for each chamber + +#include "DataFormatsTRD/T0FitHistos.h" +#include +#include + +using namespace o2::trd; +using namespace o2::trd::constants; + +void T0FitHistos::fill(const std::vector& data) +{ + for (const auto& ph : data) { + int det = ph.getDetector(); + int tb = ph.getTimebin(); + int adc = ph.getADC(); + + if (ph.getNneighbours() != 2) { + continue; + } + + mDet.push_back(det); + mTB.push_back(tb); + mADC.push_back(adc); + ++mNEntriesTot; + } +} + +void T0FitHistos::merge(const T0FitHistos* prev) +{ + auto sizePrev = (int)prev->getNEntries(); + + for (int i = 0; i < sizePrev; ++i) { + mDet.push_back(prev->getDetector(i)); + mTB.push_back(prev->getTimeBin(i)); + mADC.push_back(prev->getADC(i)); + } +} + +void T0FitHistos::print() +{ + LOG(info) << "There are " << mNEntriesTot << " entries in the container"; +} diff --git a/DataFormats/Detectors/TRD/src/Tracklet64.cxx b/DataFormats/Detectors/TRD/src/Tracklet64.cxx index 0e940b4f0a1d1..d7b63cae45354 100644 --- a/DataFormats/Detectors/TRD/src/Tracklet64.cxx +++ b/DataFormats/Detectors/TRD/src/Tracklet64.cxx @@ -10,24 +10,19 @@ // or submit itself to any jurisdiction. #include "DataFormatsTRD/Tracklet64.h" -#include "DataFormatsTRD/Constants.h" #include "DataFormatsTRD/HelperMethods.h" - #include "fairlogger/Logger.h" #include namespace o2 { - namespace trd { -using namespace constants; - void Tracklet64::print() const { - LOGF(info, "%02i_%i_%i, row(%i), col(%i), position(%i), slope(%i), pid(%i), q0(%i), q1(%i), q2(%i)", - HelperMethods::getSector(getDetector()), HelperMethods::getStack(getDetector()), HelperMethods::getLayer(getDetector()), getPadRow(), getColumn(), getPosition(), getSlope(), getPID(), getQ0(), getQ1(), getQ2()); + LOGF(info, "%02i_%i_%i, ROB(%i), MCM(%i), row(%i), col(%i), position(%i), slope(%i), pid(%i), q0(%i), q1(%i), q2(%i). Format(%i)", + HelperMethods::getSector(getDetector()), HelperMethods::getStack(getDetector()), HelperMethods::getLayer(getDetector()), getROB(), getMCM(), getPadRow(), getPadCol(), getPosition(), getSlope(), getPID(), getQ0(), getQ1(), getQ2(), getFormat()); } #ifndef GPUCA_GPUCODE_DEVICE @@ -45,6 +40,16 @@ std::ostream& operator<<(std::ostream& stream, const Tracklet64& trg) trg.printStream(stream); return stream; } + +bool operator<(const Tracklet64& lhs, const Tracklet64& rhs) +{ + return (lhs.getDetector() < rhs.getDetector()) || + (lhs.getDetector() == rhs.getDetector() && lhs.getROB() < rhs.getROB()) || + (lhs.getDetector() == rhs.getDetector() && lhs.getROB() == rhs.getROB() && lhs.getMCM() < rhs.getMCM()) || + (lhs.getDetector() == rhs.getDetector() && lhs.getROB() == rhs.getROB() && lhs.getMCM() == rhs.getMCM() && lhs.getPadRow() < rhs.getPadRow()) || + (lhs.getDetector() == rhs.getDetector() && lhs.getROB() == rhs.getROB() && lhs.getMCM() == rhs.getMCM() && lhs.getPadRow() == rhs.getPadRow() && lhs.getPadCol() < rhs.getPadCol()); +} + #endif // GPUCA_GPUCODE_DEVICE } // namespace trd diff --git a/DataFormats/Detectors/TRD/test/testDigit.cxx b/DataFormats/Detectors/TRD/test/testDigit.cxx index 1406cc8671e0c..93bd0994c3990 100644 --- a/DataFormats/Detectors/TRD/test/testDigit.cxx +++ b/DataFormats/Detectors/TRD/test/testDigit.cxx @@ -176,6 +176,27 @@ BOOST_AUTO_TEST_CASE(TRDDigit_test) testDigitDetRowPad(za, 10, 12, NCOLMCM - 1); BOOST_CHECK(za.getADC()[14] == 56); // 14th time bin should be 56; BOOST_CHECK(za.getADC()[16] == 58); // 16th time bin should be 58; + + // check the setting and reading of the phase for various combinations of setting the detector and phase. + Digit withphase(10, 6, 0, 2, data, 3); + BOOST_CHECK(withphase.getPreTrigPhase() == 0x3); + withphase.setPreTrigPhase(1); + BOOST_CHECK(withphase.getPreTrigPhase() == 0x1); + withphase.setDetector(12); + BOOST_CHECK(withphase.getPreTrigPhase() == 0x1); + BOOST_CHECK(withphase.getDetector() == 0xc); + withphase.setDetector(539); + BOOST_CHECK(withphase.getPreTrigPhase() == 0x1); + BOOST_CHECK(withphase.getDetector() == 539); + withphase.setDetector(12); + BOOST_CHECK(withphase.getPreTrigPhase() == 0x1); + BOOST_CHECK(withphase.getDetector() == 12); + withphase.setDetector(0); + BOOST_CHECK(withphase.getPreTrigPhase() == 0x1); + BOOST_CHECK(withphase.getDetector() == 0); + withphase.setPreTrigPhase(0); + BOOST_CHECK(withphase.getPreTrigPhase() == 0x0); + BOOST_CHECK(withphase.getDetector() == 0); } } // namespace trd diff --git a/Detectors/TRD/base/test/testRawData.cxx b/DataFormats/Detectors/TRD/test/testRawData.cxx similarity index 75% rename from Detectors/TRD/base/test/testRawData.cxx rename to DataFormats/Detectors/TRD/test/testRawData.cxx index 00c0047a61231..2e8f300e4b42b 100644 --- a/Detectors/TRD/base/test/testRawData.cxx +++ b/DataFormats/Detectors/TRD/test/testRawData.cxx @@ -84,6 +84,40 @@ BOOST_AUTO_TEST_CASE(TRDRawDataHeaderInternals) //check tracklet tracklet.word = 0xffe00000; BOOST_CHECK_EQUAL((uint32_t)tracklet.pos, 0x7ff); + + // This will get expanded, for now just check the current changes to charges. + // build a tracklet package of ff00ff + TrackletHCHeader hcheader; + hcheader.word = 0; + constructTrackletHCHeader(hcheader, 42, 42, 3); //only thing important is the format of 3 + // hcid=42, clock=42 + TrackletMCMHeader header; + header.word = 0; + header.onea = 1; + header.oneb = 1; + header.padrow = 2; + header.col = 1; + header.pid0 = 0xff; + header.pid1 = 0x02; + header.pid2 = 0xff; + std::array tracklets; + tracklets[0].word = 0; + tracklets[0].pos = 42; + tracklets[0].slope = 24; + tracklets[0].checkbit = 1; + tracklets[0].pid = 0xefe; + tracklets[1].word = 0; + tracklets[2].word = 0; + std::array charges; + // auto invalidheader = getChargesFromRawHeaders(hcheader, &header, tracklets, charges, 0); + // auto trackletcount = getNumberOfTrackletsFromHeader(&header); + + // BOOST_CHECK(trackletcount == 1); + // BOOST_CHECK(invalidheader == 0); + // std::cout << std::hex << " charges[0]:0x"<<(int)charges[0]; + // std::cout << std::hex << " charges[1]:0x"<<(int)charges[1]; + // std::cout << std::hex << " charges[2]:0x"<<(int)charges[2]; + // BOOST_CHECK((int)charges[2] == 0x02); } } // namespace trd } // namespace o2 diff --git a/DataFormats/Detectors/Upgrades/ALICE3/CMakeLists.txt b/DataFormats/Detectors/Upgrades/ALICE3/CMakeLists.txt new file mode 100644 index 0000000000000..b3944c2e502d8 --- /dev/null +++ b/DataFormats/Detectors/Upgrades/ALICE3/CMakeLists.txt @@ -0,0 +1,12 @@ +# Copyright 2019-2025 CERN and copyright holders of ALICE O2. +# See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +# All rights not expressly granted are reserved. +# +# This software is distributed under the terms of the GNU General Public +# License v3 (GPL Version 3), copied verbatim in the file "COPYING". +# +# In applying this license CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization +# or submit itself to any jurisdiction. + +add_subdirectory(FD3) diff --git a/DataFormats/Detectors/Upgrades/ALICE3/FD3/CMakeLists.txt b/DataFormats/Detectors/Upgrades/ALICE3/FD3/CMakeLists.txt new file mode 100644 index 0000000000000..e2219bb893612 --- /dev/null +++ b/DataFormats/Detectors/Upgrades/ALICE3/FD3/CMakeLists.txt @@ -0,0 +1,23 @@ +# Copyright 2019-2020 CERN and copyright holders of ALICE O2. +# See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +# All rights not expressly granted are reserved. +# +# This software is distributed under the terms of the GNU General Public +# License v3 (GPL Version 3), copied verbatim in the file "COPYING". +# +# In applying this license CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization +# or submit itself to any jurisdiction. + +o2_add_library(DataFormatsFD3 + SOURCES src/Hit.cxx + PUBLIC_LINK_LIBRARIES O2::FD3Base + O2::SimulationDataFormat + O2::CommonDataFormat + Microsoft.GSL::GSL + O2::DetectorsCommonDataFormats +) + +o2_target_root_dictionary(DataFormatsFD3 + HEADERS include/DataFormatsFD3/Hit.h +) diff --git a/DataFormats/Detectors/Upgrades/ALICE3/FD3/include/DataFormatsFD3/Hit.h b/DataFormats/Detectors/Upgrades/ALICE3/FD3/include/DataFormatsFD3/Hit.h new file mode 100644 index 0000000000000..4fde2f6cde6b4 --- /dev/null +++ b/DataFormats/Detectors/Upgrades/ALICE3/FD3/include/DataFormatsFD3/Hit.h @@ -0,0 +1,125 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file Hit.h +/// \brief Definition of the FD3 Hit class (based on ITSMFT and FV0) + +#ifndef ALICEO2_FVD_HIT_H_ +#define ALICEO2_FVD_HIT_H_ + +#include +#include "SimulationDataFormat/BaseHits.h" // for BasicXYZEHit +#include "Rtypes.h" // for Bool_t, Double_t, int, Double32_t, etc +#include "TVector3.h" // for TVector3 +#include "CommonUtils/ShmAllocator.h" + +namespace o2 +{ +namespace fd3 +{ + +class Hit : public o2::BasicXYZEHit +{ + public: + /// Default constructor + Hit() = default; + + /// Class Constructor + /// \param trackID Index of MCTrack + /// \param cellID Cell ID + /// \param startPos Coordinates at entrance to active volume [cm] + /// \param endPos Coordinates to active volume [cm] + /// \param startMom Momentum of track at entrance [GeV] + /// \param startE Energy of track at entrance [GeV] + /// \param endTime Final time [ns] + /// \param eLoss Energy deposit [GeV] + /// \param particlePdg PDG code of the partcile associated with the track + inline Hit(int trackID, + int cellID, + const math_utils::Point3D& startPos, + const math_utils::Point3D& endPos, + const math_utils::Vector3D& startMom, + double startE, + double endTime, + double eLoss, + int particlePdg); + + // Entrance position getters + math_utils::Point3D const& GetPosStart() const { return mPositionStart; } + float GetStartX() const { return mPositionStart.X(); } + float GetStartY() const { return mPositionStart.Y(); } + float GetStartZ() const { return mPositionStart.Z(); } + template + void GetStartPosition(F& x, F& y, F& z) const + { + x = GetStartX(); + y = GetStartY(); + z = GetStartZ(); + } + + // Momentum getters + math_utils::Vector3D const& GetMomentum() const { return mMomentumStart; } + math_utils::Vector3D& GetMomentum() { return mMomentumStart; } + float GetPx() const { return mMomentumStart.X(); } + float GetPy() const { return mMomentumStart.Y(); } + float GetPz() const { return mMomentumStart.Z(); } + float GetE() const { return mEnergyStart; } + float GetTotalEnergyAtEntrance() const { return GetE(); } + int GetParticlePdg() const { return mParticlePdg; } + + void Print(const Option_t* opt) const; + + private: + math_utils::Vector3D mMomentumStart; ///< momentum at entrance + math_utils::Point3D mPositionStart; ///< position at entrance (base mPos give position on exit) + float mEnergyStart; ///< total energy at entrance + int mParticlePdg; ///< PDG code of the particle associated with this track + + ClassDefNV(Hit, 1); +}; + +Hit::Hit(int trackID, + int detID, + const math_utils::Point3D& startPos, + const math_utils::Point3D& endPos, + const math_utils::Vector3D& startMom, + double startE, + double endTime, + double eLoss, + int particlePdg) + : BasicXYZEHit(endPos.X(), + endPos.Y(), + endPos.Z(), + endTime, + eLoss, + trackID, + detID), + mMomentumStart(startMom.X(), startMom.Y(), startMom.Z()), + mPositionStart(startPos.X(), startPos.Y(), startPos.Z()), + mEnergyStart(startE), + mParticlePdg(particlePdg) +{ +} + +} // namespace fd3 +} // namespace o2 + +#ifdef USESHM +namespace std +{ +template <> +class allocator : public o2::utils::ShmAllocator +{ +}; + +} // namespace std +#endif /* USESHM */ +#endif /* ALICEO2_FD3_HIT_H_ */ diff --git a/DataFormats/Detectors/Upgrades/ALICE3/FD3/src/DataFormatsFD3LinkDef.h b/DataFormats/Detectors/Upgrades/ALICE3/FD3/src/DataFormatsFD3LinkDef.h new file mode 100644 index 0000000000000..1014b3d8c704e --- /dev/null +++ b/DataFormats/Detectors/Upgrades/ALICE3/FD3/src/DataFormatsFD3LinkDef.h @@ -0,0 +1,21 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifdef __CLING__ + +#pragma link off all globals; +#pragma link off all classes; +#pragma link off all functions; + +#pragma link C++ class o2::fd3::Hit + ; +#pragma link C++ class vector < o2::fd3::Hit> + ; + +#endif diff --git a/DataFormats/Detectors/Upgrades/ALICE3/FD3/src/Hit.cxx b/DataFormats/Detectors/Upgrades/ALICE3/FD3/src/Hit.cxx new file mode 100644 index 0000000000000..403a3402bd30c --- /dev/null +++ b/DataFormats/Detectors/Upgrades/ALICE3/FD3/src/Hit.cxx @@ -0,0 +1,35 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file Hit.cxx +/// \brief Implementation of the Hit class + +#include "DataFormatsFD3/Hit.h" +#include + +ClassImp(o2::fd3::Hit); + +namespace o2 +{ +namespace fd3 +{ + +void Hit::Print(const Option_t* opt) const +{ + printf( + "Det: %5d Track: %6d E.loss: %.3e P: %+.3e %+.3e %+.3e\n" + "PosIn: %+.3e %+.3e %+.3e PosOut: %+.3e %+.3e %+.3e\n", + GetDetectorID(), GetTrackID(), GetEnergyLoss(), GetPx(), GetPy(), GetPz(), + GetStartX(), GetStartY(), GetStartZ(), GetX(), GetY(), GetZ()); +} + +} // namespace fd3 +} // namespace o2 diff --git a/DataFormats/Detectors/Upgrades/CMakeLists.txt b/DataFormats/Detectors/Upgrades/CMakeLists.txt index a2d470b8ff6d5..0dfe07dc2827d 100644 --- a/DataFormats/Detectors/Upgrades/CMakeLists.txt +++ b/DataFormats/Detectors/Upgrades/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright 2019-2020 CERN and copyright holders of ALICE O2. +# Copyright 2019-2025 CERN and copyright holders of ALICE O2. # See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. # All rights not expressly granted are reserved. # @@ -10,3 +10,4 @@ # or submit itself to any jurisdiction. message(STATUS "Building dataformats for upgrades") +add_subdirectory(ALICE3) diff --git a/DataFormats/Detectors/ZDC/include/DataFormatsZDC/BCData.h b/DataFormats/Detectors/ZDC/include/DataFormatsZDC/BCData.h index cdbf813db318d..916181030c583 100644 --- a/DataFormats/Detectors/ZDC/include/DataFormatsZDC/BCData.h +++ b/DataFormats/Detectors/ZDC/include/DataFormatsZDC/BCData.h @@ -38,7 +38,9 @@ struct __attribute__((__packed__)) ModuleTriggerMap { unsigned Auto_1 : 1; unsigned Auto_2 : 1; unsigned Auto_3 : 1; - unsigned empty : 7; + unsigned AliceErr : 1; + unsigned AutoErr : 1; + unsigned empty : 5; }; union ModuleTriggerMapData { @@ -49,11 +51,13 @@ union ModuleTriggerMapData { struct BCData { /// we are going to refer to at most 26 channels, so 5 bits for the NChannels and 27 for the reference - o2::dataformats::RangeRefComp<5> ref; + /// 220803 we are transmitting 32 ch: need to increase to 6 bit + o2::dataformats::RangeRefComp<6> ref; o2::InteractionRecord ir; std::array moduleTriggers{}; + // N.B. channels and triggers have geographical addressing (0x1 << (NChPerModule * im + ic) uint32_t channels = 0; // pattern of channels it refers to - uint32_t triggers = 0; // pattern of triggered channels (not necessarily stored) in this BC + uint32_t triggers = 0; // pattern of triggered channels (not necessarily stored) in this BC (i.e. with Hit bit on) uint8_t ext_triggers = 0; // pattern of ALICE triggers BCData() = default; @@ -71,7 +75,7 @@ struct BCData { gsl::span getBunchChannelData(const gsl::span tfdata) const; void print(uint32_t triggerMask = 0, int diff = 0) const; - ClassDefNV(BCData, 2); + ClassDefNV(BCData, 3); }; } // namespace zdc } // namespace o2 diff --git a/DataFormats/Detectors/ZDC/include/DataFormatsZDC/Hit.h b/DataFormats/Detectors/ZDC/include/DataFormatsZDC/Hit.h index 6e0b99dca6761..3ce90a95f7248 100644 --- a/DataFormats/Detectors/ZDC/include/DataFormatsZDC/Hit.h +++ b/DataFormats/Detectors/ZDC/include/DataFormatsZDC/Hit.h @@ -58,6 +58,7 @@ class Hit : public o2::BasicXYZEHit float getPMCLightYield() const { return mNphePMC; } float getPMQLightYield() const { return mNphePMQ; } int getNumContributingSteps() const { return mNoContributingSteps; } + bool getSecFlag() const { return mSecFlag; } private: Int_t mParentID; diff --git a/DataFormats/Detectors/ZDC/include/DataFormatsZDC/RawEventData.h b/DataFormats/Detectors/ZDC/include/DataFormatsZDC/RawEventData.h index 5d856f55c1d65..622f3116b7868 100644 --- a/DataFormats/Detectors/ZDC/include/DataFormatsZDC/RawEventData.h +++ b/DataFormats/Detectors/ZDC/include/DataFormatsZDC/RawEventData.h @@ -33,7 +33,9 @@ constexpr unsigned short Id_w0 = 0x0; constexpr unsigned short Id_w1 = 0x1; constexpr unsigned short Id_w2 = 0x2; constexpr unsigned short Id_wn = 0x3; -constexpr int NWPerGBTW = 4; +constexpr int NWPerGBTW = 4; // 4*32bit=128 bit per GBTW +constexpr int NBPerGBTW = 4 * NWPerGBTW; // 16B=128 bit per GBTW +constexpr int PayloadPerGBTW = 10; // 80 bit per GBTW struct __attribute__((__packed__)) ChannelDataV0 { // First GBT word diff --git a/DataFormats/Detectors/ZDC/include/DataFormatsZDC/RecEventFlat.h b/DataFormats/Detectors/ZDC/include/DataFormatsZDC/RecEventFlat.h index bc5bb8fd7628e..ea5fd10f92dcc 100644 --- a/DataFormats/Detectors/ZDC/include/DataFormatsZDC/RecEventFlat.h +++ b/DataFormats/Detectors/ZDC/include/DataFormatsZDC/RecEventFlat.h @@ -24,6 +24,7 @@ #include #include #include +#include /// \file RecEventFlat.h /// \brief Class to decode the reconstructed ZDC event (single BC with signal in one of detectors) @@ -36,6 +37,21 @@ namespace zdc using FirstEntry = int; using NElem = int; +struct RecEventScale { + static std::array fe; + static std::array fa; + static void reset(); + static void setGeV(); + static void setGeVZN(); + static void setGeVZP(); + static void setTeV(); + static void setTeVZN(); + static void setTeVZP(); + static void setNucleonEnergy(float energy); + static void setNucleonEnergyZN(float energy); + static void setNucleonEnergyZP(float energy); +}; + struct RecEventFlat { // NOLINT: false positive in clang-tidy !! o2::InteractionRecord ir; uint32_t channels = 0; /// pattern of channels acquired @@ -65,6 +81,7 @@ struct RecEventFlat { // NOLINT: false positive in clang-tidy !! std::array isEnd{}; //! End of sequence BCRecData mCurB; //! Current BC std::vector inter[NChannels]; //! Interpolated samples + std::array mComputed{}; //! Centroid computed // Reconstruction messages std::array genericE{}; /// 0 Generic error @@ -170,7 +187,7 @@ struct RecEventFlat { // NOLINT: false positive in clang-tidy !! { if (ich < NTDCChannels) { if (ipos < TDCAmp[ich].size()) { - return FTDCAmp * TDCAmp[ich][ipos]; + return TDCAmp[ich][ipos]; } } return -std::numeric_limits::infinity(); @@ -235,11 +252,30 @@ struct RecEventFlat { // NOLINT: false positive in clang-tidy !! float EZPC4() const { return EZDC(IdZPC4); } float EZPCSum() const { return EZDC(IdZPCSum); } + // Centroid cartesian reference frame as seen by projectile fragments + // Side A: X direction towards outside of the ring + // Side C: X direction towards inside of the ring + // Y direction up + // Unit: cm + void centroidZNA(float& x, float& y); + void centroidZNC(float& x, float& y); + void centroidZPA(float& x, float& rms); + void centroidZPC(float& x, float& rms); + float xZNA(); + float yZNA(); + float xZNC(); + float yZNC(); + float xZPA(); // Positive + float xZPC(); // Negative + float rmsZPA(); + float rmsZPC(); + void decodeInfo(uint8_t ch, uint16_t code); void decodeMapInfo(uint32_t ch, uint16_t code); void print() const; void printDecodedMessages() const; + void printEvent() const; ClassDefNV(RecEventFlat, 1); }; diff --git a/DataFormats/Detectors/ZDC/include/DataFormatsZDC/ZDCTDCData.h b/DataFormats/Detectors/ZDC/include/DataFormatsZDC/ZDCTDCData.h index b159ccdb1a3a2..d4d6bcdf75be0 100644 --- a/DataFormats/Detectors/ZDC/include/DataFormatsZDC/ZDCTDCData.h +++ b/DataFormats/Detectors/ZDC/include/DataFormatsZDC/ZDCTDCData.h @@ -27,49 +27,88 @@ namespace o2 namespace zdc { +struct ZDCTDCDataErr { + + static uint32_t mErrVal[NTDCChannels]; // Errors in encoding TDC values + static uint32_t mErrId; // Errors with TDC Id + + static void print() + { + if (mErrId > 0) { + LOG(error) << "TDCId was out of range #times = " << mErrId; + } + for (int itdc = 0; itdc < NTDCChannels; itdc++) { + if (mErrVal[itdc] > 0) { + LOG(error) << "TDCVal itdc=" << itdc << " " << ChannelNames[TDCSignal[itdc]] << " was out of range #times = " << mErrVal[itdc]; + } + } + } +}; + struct ZDCTDCData { uint8_t id = 0xff; // channel ID int16_t val = 0; // tdc value - int16_t amp = 0; // tdc amplitude + float amp = 0; // tdc amplitude ZDCTDCData() = default; - ZDCTDCData(uint8_t ida, int16_t vala, int16_t ampa, bool isbeg = false, bool isend = false) + + ZDCTDCData(uint8_t ida, int16_t vala, float ampa, bool isbeg = false, bool isend = false) { // TDC value and amplitude are encoded externally - id = ida & 0x0f; + id = ida < NTDCChannels ? ida : 0xf; id = id | (isbeg ? 0x80 : 0x00); id = id | (isend ? 0x40 : 0x00); - val = vala; - amp = ampa; + + if (ida < NTDCChannels) { + val = vala; + amp = ampa; + } else { + val = kMaxShort; + amp = FInfty; +#ifdef O2_ZDC_DEBUG + LOG(error) << __func__ << "TDC Id = " << int(ida) << " is out of range"; +#endif + ZDCTDCDataErr::mErrId++; + } } ZDCTDCData(uint8_t ida, float vala, float ampa, bool isbeg = false, bool isend = false) { - // TDC value and amplitude are encoded externally - id = ida & 0x0f; + // TDC value and amplitude are encoded externally but argument is float + id = ida < NTDCChannels ? ida : 0xf; id = id | (isbeg ? 0x80 : 0x00); id = id | (isend ? 0x40 : 0x00); + if (ida >= NTDCChannels) { + val = kMaxShort; + amp = FInfty; +#ifdef O2_ZDC_DEBUG + LOG(error) << __func__ << "TDC Id = " << int(ida) << " is out of range"; +#endif + ZDCTDCDataErr::mErrId++; + return; + } + auto TDCVal = std::nearbyint(vala); - auto TDCAmp = std::nearbyint(ampa); if (TDCVal < kMinShort) { - LOG(error) << __func__ << " TDCVal " << int(ida) << " " << ChannelNames[ida] << " = " << TDCVal << " is out of range"; + int itdc = int(id); +#ifdef O2_ZDC_DEBUG + LOG(error) << __func__ << "TDCVal itdc=" << itdc << " " << ChannelNames[TDCSignal[itdc]] << " = " << TDCVal << " is out of range"; +#endif + ZDCTDCDataErr::mErrVal[itdc]++; TDCVal = kMinShort; } + if (TDCVal > kMaxShort) { - LOG(error) << __func__ << " TDCVal " << int(ida) << " " << ChannelNames[ida] << " = " << TDCVal << " is out of range"; + int itdc = int(ida); +#ifdef O2_ZDC_DEBUG + LOG(error) << __func__ << "TDCVal itdc=" << itdc << " " << ChannelNames[TDCSignal[itdc]] << " = " << TDCVal << " is out of range"; +#endif + ZDCTDCDataErr::mErrVal[itdc]++; TDCVal = kMaxShort; } - if (TDCAmp < kMinShort) { - LOG(error) << __func__ << " TDCAmp " << int(ida) << " " << ChannelNames[ida] << " = " << TDCAmp << " is out of range"; - TDCAmp = kMinShort; - } - if (TDCAmp > kMaxShort) { - LOG(error) << __func__ << " TDCAmp " << int(ida) << " " << ChannelNames[ida] << " = " << TDCAmp << " is out of range"; - TDCAmp = kMaxShort; - } val = TDCVal; amp = ampa; @@ -77,22 +116,25 @@ struct ZDCTDCData { inline float amplitude() const { - // Return decoded value - return FTDCAmp * amp; + return amp; } + inline float value() const { // Return decoded value (ns) return FTDCVal * val; } + inline int ch() const { return (id & 0x0f); } + inline bool isBeg() const { return id & 0x80 ? true : false; } + inline bool isEnd() const { return id & 0x40 ? true : false; @@ -100,7 +142,7 @@ struct ZDCTDCData { void print() const; - ClassDefNV(ZDCTDCData, 1); + ClassDefNV(ZDCTDCData, 2); }; } // namespace zdc } // namespace o2 diff --git a/DataFormats/Detectors/ZDC/src/DataFormatsZDCLinkDef.h b/DataFormats/Detectors/ZDC/src/DataFormatsZDCLinkDef.h index 6e272dbc8a118..0fbc7308f39e3 100644 --- a/DataFormats/Detectors/ZDC/src/DataFormatsZDCLinkDef.h +++ b/DataFormats/Detectors/ZDC/src/DataFormatsZDCLinkDef.h @@ -51,4 +51,6 @@ #pragma link C++ struct o2::zdc::CTF + ; #pragma link C++ class o2::ctf::EncodedBlocks < o2::zdc::CTFHeader, 12, uint32_t> + ; +#pragma read sourceClass = "ZDCTDCData" targetClass = "ZDCTDCData" source = "int16_t amp" version = "[-1]" target = "amp" targetType = "float" code = "{ amp=onfile.amp / 8.; }" + #endif diff --git a/DataFormats/Detectors/ZDC/src/RecEventFlat.cxx b/DataFormats/Detectors/ZDC/src/RecEventFlat.cxx index 7c9d50c3d61eb..cb2e298eee317 100644 --- a/DataFormats/Detectors/ZDC/src/RecEventFlat.cxx +++ b/DataFormats/Detectors/ZDC/src/RecEventFlat.cxx @@ -15,6 +15,133 @@ using namespace o2::zdc; +std::array RecEventScale::fe = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; +std::array RecEventScale::fa = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; + +void RecEventScale::reset() +{ + for (int ich = 0; ich < NChannels; ich++) { + fe[ich] = 1; + } + for (int itdc = 0; itdc < NTDCChannels; itdc++) { + fa[itdc] = 1; + } +} + +void RecEventScale::setGeV() +{ + for (int ich = 0; ich < NChannels; ich++) { + fe[ich] = 1; + } + for (int itdc = 0; itdc < NTDCChannels; itdc++) { + fa[itdc] = 1000.; + } +} + +void RecEventScale::setGeVZN() +{ + for (int ich = IdZNAC; ich <= IdZNASum; ich++) { + fe[ich] = 1; + } + for (int ich = IdZNCC; ich <= IdZNCSum; ich++) { + fe[ich] = 1; + } + fa[TDCZNAC] = 1000.; + fa[TDCZNAS] = 1000.; + fa[TDCZNCC] = 1000.; + fa[TDCZNCS] = 1000.; +} + +void RecEventScale::setGeVZP() +{ + for (int ich = IdZPAC; ich <= IdZPASum; ich++) { + fe[ich] = 1; + } + for (int ich = IdZPCC; ich <= IdZPCSum; ich++) { + fe[ich] = 1; + } + fa[TDCZPAC] = 1000.; + fa[TDCZPAS] = 1000.; + fa[TDCZPCC] = 1000.; + fa[TDCZPCS] = 1000.; +} + +void RecEventScale::setTeV() +{ + for (int ich = 0; ich < NChannels; ich++) { + fe[ich] = 0.001; + } + for (int itdc = 0; itdc < NTDCChannels; itdc++) { + fa[itdc] = 1.; + } +}; + +void RecEventScale::setTeVZN() +{ + for (int ich = IdZNAC; ich <= IdZNASum; ich++) { + fe[ich] = 0.001; + } + for (int ich = IdZNCC; ich <= IdZNCSum; ich++) { + fe[ich] = 0.001; + } + fa[TDCZNAC] = 1.; + fa[TDCZNAS] = 1.; + fa[TDCZNCC] = 1.; + fa[TDCZNCS] = 1.; +}; + +void RecEventScale::setTeVZP() +{ + for (int ich = IdZPAC; ich <= IdZPASum; ich++) { + fe[ich] = 0.001; + } + for (int ich = IdZPCC; ich <= IdZPCSum; ich++) { + fe[ich] = 0.001; + } + fa[TDCZPAC] = 1.; + fa[TDCZPAS] = 1.; + fa[TDCZPCC] = 1.; + fa[TDCZPCS] = 1.; +}; + +void RecEventScale::setNucleonEnergy(float energy) +{ // In GeV + for (int ich = 0; ich < NChannels; ich++) { + fe[ich] = 1. / energy; + } + for (int itdc = 0; itdc < NTDCChannels; itdc++) { + fa[itdc] = 1000. / energy; + } +}; + +void RecEventScale::setNucleonEnergyZN(float energy) +{ // In GeV + for (int ich = IdZNAC; ich <= IdZNASum; ich++) { + fe[ich] = 1. / energy; + } + for (int ich = IdZNCC; ich <= IdZNCSum; ich++) { + fe[ich] = 1. / energy; + } + fa[TDCZNAC] = 1000. / energy; + fa[TDCZNAS] = 1000. / energy; + fa[TDCZNCC] = 1000. / energy; + fa[TDCZNCS] = 1000. / energy; +}; + +void RecEventScale::setNucleonEnergyZP(float energy) +{ // In GeV + for (int ich = IdZPAC; ich <= IdZPASum; ich++) { + fe[ich] = 1. / energy; + } + for (int ich = IdZPCC; ich <= IdZPCSum; ich++) { + fe[ich] = 1. / energy; + } + fa[TDCZPAC] = 1000. / energy; + fa[TDCZPAS] = 1000. / energy; + fa[TDCZPCC] = 1000. / energy; + fa[TDCZPCS] = 1000. / energy; +}; + void RecEventFlat::init(const std::vector* RecBC, const std::vector* Energy, const std::vector* TDCData, const std::vector* Info) { mRecBC = *RecBC; @@ -37,6 +164,7 @@ void RecEventFlat::init(const gsl::span RecBC, const g void RecEventFlat::clearBitmaps() { + genericE.fill(false); tdcPedEv.fill(false); tdcPedOr.fill(false); tdcPedQC.fill(false); @@ -61,6 +189,7 @@ void RecEventFlat::clearBitmaps() // Other bitmaps isBeg.fill(false); isEnd.fill(false); + mComputed.fill(false); } int RecEventFlat::next() @@ -141,7 +270,7 @@ int RecEventFlat::at(int ientry) for (int i = mFirstE; i < mStopE; i++) { auto myenergy = mEnergy[i]; auto ch = myenergy.ch(); - ezdc[ch] = myenergy.energy(); + ezdc[ch] = myenergy.energy() * RecEventScale::fe[ch]; // Assign implicit event info if (adcPedOr[ch] == false && adcPedQC[ch] == false && adcPedMissing[ch] == false) { adcPedEv[ch] = true; @@ -161,7 +290,7 @@ int RecEventFlat::at(int ientry) isEnd[ch] = true; } TDCVal[ch].push_back(mytdc.val); - TDCAmp[ch].push_back(mytdc.amp); + TDCAmp[ch].push_back(mytdc.amp * RecEventScale::fa[ch]); // Assign implicit event info if (tdcPedQC[ch] == false && tdcPedMissing[ch] == false) { tdcPedOr[ch] = true; @@ -258,6 +387,276 @@ void RecEventFlat::decodeInfo(uint8_t ch, uint16_t code) mDecodedInfo.emplace_back((code & 0x03ff) | ((ch & 0x1f) << 10)); } +void RecEventFlat::centroidZNA(float& x, float& y) +{ + // Coordinates of tower centers, looking from IP + const static float xc[4] = {-1.75, 1.75, -1.75, 1.75}; + const static float yc[4] = {-1.75, -1.75, 1.75, 1.75}; + static float c[2] = {0}; + if (mComputed[0]) { + x = c[0]; + y = c[1]; + return; + } + mComputed[0] = true; + float e[4], w[4]; + e[0] = EZDC(IdZNA1); + e[1] = EZDC(IdZNA2); + e[2] = EZDC(IdZNA3); + e[3] = EZDC(IdZNA4); + if (e[0] < -1000000 || e[1] < -1000000 || e[2] < -1000000 || e[3] < -1000000) { + c[0] = -std::numeric_limits::infinity(); + c[1] = -std::numeric_limits::infinity(); + x = c[0]; + y = c[1]; + return; + } + float sumw = 0; + c[0] = 0; + c[1] = 0; + for (int i = 0; i < 4; i++) { + if (e[i] < 0) { + e[i] = 0; + } + w[i] = std::sqrt(e[i]); + sumw += w[i]; + c[0] += w[i] * xc[i]; + c[1] += w[i] * yc[i]; + } + if (sumw <= 0) { + c[0] = -std::numeric_limits::infinity(); + c[1] = -std::numeric_limits::infinity(); + } else { + c[0] /= sumw; + c[1] /= sumw; + } + x = c[0]; + y = c[1]; + return; +} + +void RecEventFlat::centroidZNC(float& x, float& y) +{ + // Coordinates of tower centers, looking from IP + const static float xc[4] = {-1.75, 1.75, -1.75, 1.75}; + const static float yc[4] = {-1.75, -1.75, 1.75, 1.75}; + static float c[2] = {0}; + if (mComputed[1]) { + x = c[0]; + y = c[1]; + return; + } + mComputed[1] = true; + float e[4], w[4]; + e[0] = EZDC(IdZNC1); + e[1] = EZDC(IdZNC2); + e[2] = EZDC(IdZNC3); + e[3] = EZDC(IdZNC4); + if (e[0] < -1000000 || e[1] < -1000000 || e[2] < -1000000 || e[3] < -1000000) { + c[0] = -std::numeric_limits::infinity(); + c[1] = -std::numeric_limits::infinity(); + x = c[0]; + y = c[1]; + return; + } + float sumw = 0; + c[0] = 0; + c[1] = 0; + for (int i = 0; i < 4; i++) { + if (e[i] < 0) { + e[i] = 0; + } + w[i] = std::sqrt(e[i]); + sumw += w[i]; + c[0] += w[i] * xc[i]; + c[1] += w[i] * yc[i]; + } + if (sumw <= 0) { + c[0] = -std::numeric_limits::infinity(); + c[1] = -std::numeric_limits::infinity(); + } else { + c[0] /= sumw; + c[1] /= sumw; + } + x = c[0]; + y = c[1]; + return; +} + +float RecEventFlat::xZNA() +{ + float x, y; + centroidZNA(x, y); + return x; +} + +float RecEventFlat::yZNA() +{ + float x, y; + centroidZNA(x, y); + return y; +} + +float RecEventFlat::xZNC() +{ + float x, y; + centroidZNC(x, y); + return x; +} + +float RecEventFlat::yZNC() +{ + float x, y; + centroidZNC(x, y); + return x; +} + +float RecEventFlat::xZPA() +{ + float x, rms; + centroidZPA(x, rms); + return x; +} + +float RecEventFlat::xZPC() +{ + float x, rms; + centroidZPC(x, rms); + return x; +} + +float RecEventFlat::rmsZPA() +{ + float x, rms; + centroidZPA(x, rms); + return rms; +} + +float RecEventFlat::rmsZPC() +{ + float x, rms; + centroidZPC(x, rms); + return rms; +} + +void RecEventFlat::centroidZPA(float& x, float& rms) +{ + // x coordinates of tower centers + // Negative because ZPA calorimeter is on the left of ZNA when looking from IP + // rms can result from 0 to sqrt((2.8^2+19.6^2)/2-((2.6+19.4)/2)^2) = 8.66 + const static float xc[4] = {-19.6, -14.0, -8.4, -2.8}; + const static float xq[4] = {19.6 * 19.6, 14.0 * 14.0, 8.4 * 8.4, 2.8 * 2.8}; + static float c = 0; + static float d = 0; + if (mComputed[2] == true) { + x = c; + rms = d; + } + mComputed[2] = true; + float e[4], w[4]; + e[0] = EZDC(IdZPA1); + e[1] = EZDC(IdZPA2); + e[2] = EZDC(IdZPA3); + e[3] = EZDC(IdZPA4); + if (e[0] < -1000000 || e[1] < -1000000 || e[2] < -1000000 || e[3] < -1000000) { + c = -std::numeric_limits::infinity(); + d = -std::numeric_limits::infinity(); + x = c; + rms = d; + return; + } + float sumw = 0; + c = 0; + d = 0; + for (int i = 0; i < 4; i++) { + if (e[i] < 0) { + w[i] = 0; + } else { + // w[i] = std::sqrt(e[i]); + w[i] = e[i]; + } + sumw += w[i]; + c = c + w[i] * xc[i]; + d = d + w[i] * xq[i]; + } + if (sumw <= 0) { + c = -std::numeric_limits::infinity(); + d = -std::numeric_limits::infinity(); + } else { + c = c / sumw; + d = d / sumw; + d = d - c * c; // variance + if (d >= 0) { + d = TMath::Sqrt(d); + } else { + LOGF(warn, "%s FOP exception @ %d", __FILE__, __LINE__); + d = -std::numeric_limits::infinity(); + } + } + x = c; + rms = d; + return; +} + +void RecEventFlat::centroidZPC(float& x, float& rms) +{ + // x coordinates of tower centers + // Positive because ZPC calorimeter is on the right of ZNC when looking from IP + // x can result in a value from 2.8 to 19.6 + const static float xc[4] = {2.8, 8.4, 14.0, 19.6}; + const static float xq[4] = {2.8 * 2.8, 8.4 * 8.4, 14.0 * 14.0, 19.6 * 19.6}; + static float c = 0; + static float d = 0; + if (mComputed[3]) { + x = c; + rms = d; + } + mComputed[3] = true; + float e[4], w[4]; + e[0] = EZDC(IdZPC1); + e[1] = EZDC(IdZPC2); + e[2] = EZDC(IdZPC3); + e[3] = EZDC(IdZPC4); + if (e[0] < -1000000 || e[1] < -1000000 || e[2] < -1000000 || e[3] < -1000000) { + c = -std::numeric_limits::infinity(); + d = -std::numeric_limits::infinity(); + x = c; + rms = d; + return; + } + float sumw = 0; + c = 0; + d = 0; + for (int i = 0; i < 4; i++) { + if (e[i] < 0) { + w[i] = 0; + } else { + // w[i] = std::sqrt(e[i]); + w[i] = e[i]; + } + sumw += w[i]; + c += w[i] * xc[i]; + d += w[i] * xq[i]; + } + if (sumw <= 0) { + c = -std::numeric_limits::infinity(); + d = -std::numeric_limits::infinity(); + } else { + c /= sumw; + d /= sumw; + d = d - c * c; // variance + if (d >= 0) { + d = TMath::Sqrt(d); + } else { + LOGF(warn, "%s FOP exception @ %d", __FILE__, __LINE__); + d = -std::numeric_limits::infinity(); + } + } + x = c; + rms = d; + return; +} + void RecEventFlat::print() const { printf("%9u.%04u ", ir.orbit, ir.bc); @@ -351,3 +750,30 @@ void RecEventFlat::printDecodedMessages() const } } } + +void RecEventFlat::printEvent() const +{ + print(); + if (getNInfo() > 0) { + printDecodedMessages(); + } + if (getNEnergy() > 0 && mCurB.triggers == 0) { + printf("%9u.%04u Untriggered bunch\n", mCurB.ir.orbit, mCurB.ir.bc); + } + // TDC + for (int32_t itdc = 0; itdc < o2::zdc::NTDCChannels; itdc++) { + int ich = o2::zdc::TDCSignal[itdc]; + int nhit = NtdcV(itdc); + if (NtdcA(itdc) != nhit) { + fprintf(stderr, "Mismatch in TDC %d data length Val=%d Amp=%d values\n", itdc, NtdcV(itdc), NtdcA(ich)); + continue; + } + for (int32_t ipos = 0; ipos < nhit; ipos++) { + printf("%9u.%04u T %2d %s = %g @ %g \n", mCurB.ir.orbit, mCurB.ir.bc, ich, ChannelNames[ich].data(), TDCAmp[itdc][ipos], FTDCVal * TDCVal[itdc][ipos]); + } + } + // Energy + for (const auto& [ich, value] : ezdc) { + printf("%9u.%04u E %2d %s = %g\n", mCurB.ir.orbit, mCurB.ir.bc, ich, ChannelNames[ich].data(), value); + } +} diff --git a/DataFormats/Detectors/ZDC/src/ZDCTDCData.cxx b/DataFormats/Detectors/ZDC/src/ZDCTDCData.cxx index f78db83a1c983..5c55eddee1525 100644 --- a/DataFormats/Detectors/ZDC/src/ZDCTDCData.cxx +++ b/DataFormats/Detectors/ZDC/src/ZDCTDCData.cxx @@ -13,6 +13,9 @@ using namespace o2::zdc; +uint32_t ZDCTDCDataErr::mErrVal[NTDCChannels] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; +uint32_t ZDCTDCDataErr::mErrId = 0; + void o2::zdc::ZDCTDCData::print() const { int itdc = id & 0x0f; @@ -20,5 +23,5 @@ void o2::zdc::ZDCTDCData::print() const if (id != 0xff && itdc >= 0 && itdc < NTDCChannels) { isig = TDCSignal[itdc]; } - printf("%2d (%s) %d = %8.3f @ %d = %6.3f%s%s\n", isig, channelName(isig), amp, amplitude(), val, value(), isBeg() ? " B" : "", isEnd() ? " E" : ""); + printf("%2d (%s) %8.3f @ %d = %6.3f%s%s\n", isig, channelName(isig), amp, val, value(), isBeg() ? " B" : "", isEnd() ? " E" : ""); } diff --git a/DataFormats/Headers/include/Headers/DataHeader.h b/DataFormats/Headers/include/Headers/DataHeader.h index 710ea9d5bea6a..b44f41c5d3cb3 100644 --- a/DataFormats/Headers/include/Headers/DataHeader.h +++ b/DataFormats/Headers/include/Headers/DataHeader.h @@ -224,14 +224,21 @@ struct Descriptor { // Note: don't need to define operator=(ItgType v) because the compiler // can use Descriptor(ItgType initializer) for conversion + using ImplicitConversion = std::conditional_t<(size <= 8), ItgType, std::string_view>; // type cast operator for simplified usage of the descriptor's integer member - // TODO: this is sort of a hack, takes the first element. - // we should rethink these implicit conversions - operator ItgType() const + // in case it does not fit into the descriptor, the string representation is returned + operator ImplicitConversion() const { - static_assert(arraySize == 1, "casting Descriptor to ItgType only allowed for N<=8"); - return itg[0]; + if constexpr (std::is_same_v) { + return itg[0]; + } else { + size_t len = size; + while (len > 1 && str[len - 1] == 0) { + --len; + } + return std::string_view(str, len); + } } /// constructor from a compile-time string @@ -240,7 +247,7 @@ struct Descriptor { { static_assert(L <= N + 1, "initializer string must not exceed descriptor size"); unsigned i = 0; - for (; in[i] && i < (N < L ? N : L); ++i) { + for (; i < (N < L ? N : L) && in[i]; ++i) { str[i] = in[i]; } } @@ -279,6 +286,8 @@ struct Descriptor { bool operator<(const Descriptor& other) const { return std::memcmp(this->str, other.str, N) < 0; } bool operator!=(const Descriptor& other) const { return not this->operator==(other); } + // Convesion operators for comparison with their implicitly convertible types + friend bool operator==(const Descriptor& lhs, ImplicitConversion rhs) { return static_cast(lhs) == rhs; } // explicitly forbid comparison with e.g. const char* strings // use: value == Descriptor("DESC") for the appropriate // template instantiation instead @@ -364,7 +373,8 @@ struct BaseHeader { uint32_t flags; struct { uint32_t flagsNextHeader : 1, // do we have a next header after this one? - flagsUnused : 31; // currently unused + flagsReserved : 15, // reserved for future use + flagsDerivedHeader : 16; // reserved for usage by the derived header }; }; @@ -570,9 +580,17 @@ constexpr o2::header::DataOrigin gDataOriginTST{"TST"}; constexpr o2::header::DataOrigin gDataOriginACO{"ACO"}; // for bwd compatibility with DD constexpr o2::header::DataOrigin gDataOriginIT3{"IT3"}; +constexpr o2::header::DataOrigin gDataOriginFOC{"FOC"}; constexpr o2::header::DataOrigin gDataOriginTRK{"TRK"}; constexpr o2::header::DataOrigin gDataOriginFT3{"FT3"}; constexpr o2::header::DataOrigin gDataOriginFCT{"FCT"}; +constexpr o2::header::DataOrigin gDataOriginTF3{"TF3"}; +constexpr o2::header::DataOrigin gDataOriginRCH{"RCH"}; +constexpr o2::header::DataOrigin gDataOriginMI3{"MI3"}; +constexpr o2::header::DataOrigin gDataOriginECL{"ECL"}; // upgrades +constexpr o2::header::DataOrigin gDataOriginFD3{"FD3"}; // upgrades + +constexpr o2::header::DataOrigin gDataOriginGPU{"GPU"}; // possible data types constexpr o2::header::DataDescription gDataDescriptionAny{"***************"}; diff --git a/DataFormats/Headers/include/Headers/DataHeaderHelpers.h b/DataFormats/Headers/include/Headers/DataHeaderHelpers.h index 41d59090e0274..4f7e49acb4d98 100644 --- a/DataFormats/Headers/include/Headers/DataHeaderHelpers.h +++ b/DataFormats/Headers/include/Headers/DataHeaderHelpers.h @@ -38,23 +38,23 @@ struct fmt::formatter::value, c } template - auto format(const T& p, FormatContext& ctx) + auto format(const T& p, FormatContext& ctx) const { - return format_to(ctx.out(), "{}", p.template as()); + return fmt::format_to(ctx.out(), "{}", p.template as()); } }; template <> struct fmt::formatter { - // Presentation format: 'f' - fixed, 'e' - exponential. char presentation = 's'; - // Parses format specifications of the form ['f' | 'e']. constexpr auto parse(format_parse_context& ctx) { auto it = ctx.begin(), end = ctx.end(); if (it != end && (*it == 's')) { presentation = *it++; + } else if (it != end && (*it == 'x')) { + presentation = *it++; } // Check if reached the end of the range: @@ -67,19 +67,28 @@ struct fmt::formatter { } template - auto format(const o2::header::DataHeader& h, FormatContext& ctx) + auto format(const o2::header::DataHeader& h, FormatContext& ctx) const { - auto res = fmt::format("Data header version {}, flags: {}\n", h.headerVersion, h.flags) + - fmt::format(" origin : {}\n", h.dataOrigin.str) + - fmt::format(" serialization: {}\n", h.payloadSerializationMethod.str) + - fmt::format(" description : {}\n", h.dataDescription.str) + - fmt::format(" sub spec. : {}\n", (long long unsigned int)h.subSpecification) + - fmt::format(" header size : {}\n", h.headerSize) + - fmt::format(" payloadSize : {}\n", (long long unsigned int)h.payloadSize) + - fmt::format(" firstTForbit : {}\n", h.firstTForbit) + - fmt::format(" tfCounter : {}\n", h.tfCounter) + - fmt::format(" runNumber : {}\n", h.runNumber); - return format_to(ctx.out(), "{}", res); + if (presentation == 's') { + auto res = fmt::format("Data header version {}, flags: {}\n", h.headerVersion, h.flags) + + fmt::format(" origin : {}\n", h.dataOrigin.str) + + fmt::format(" serialization: {}\n", h.payloadSerializationMethod.str) + + fmt::format(" description : {}\n", h.dataDescription.str) + + fmt::format(" sub spec. : {}\n", (long long unsigned int)h.subSpecification) + + fmt::format(" header size : {}\n", h.headerSize) + + fmt::format(" payloadSize : {}\n", (long long unsigned int)h.payloadSize) + + fmt::format(" firstTForbit : {}\n", h.firstTForbit) + + fmt::format(" tfCounter : {}\n", h.tfCounter) + + fmt::format(" runNumber : {}\n", h.runNumber) + + fmt::format(" split : {}/{}\n", h.splitPayloadIndex, h.splitPayloadParts); + return fmt::format_to(ctx.out(), "{}", res); + } else { + auto res = fmt::format("{}/{}/{}", + h.dataOrigin.str, + h.dataDescription.str, + (long long unsigned int)h.subSpecification); + return fmt::format_to(ctx.out(), "{}", res); + } } }; diff --git a/DataFormats/Headers/include/Headers/RAWDataHeader.h b/DataFormats/Headers/include/Headers/RAWDataHeader.h index 8c9f5ba43f8cd..8c8e1acced6bb 100644 --- a/DataFormats/Headers/include/Headers/RAWDataHeader.h +++ b/DataFormats/Headers/include/Headers/RAWDataHeader.h @@ -42,6 +42,102 @@ namespace header /// assume that this is the only type the software has to support (based on /// experience with previous systems) /// +/// RAWDataHeaderV7 +/// V7 introduces a detector source ID field +/// Description of the fields can be found here +/// https://gitlab.cern.ch/AliceO2Group/wp6-doc/-/blob/master/rdh/RDHv7.md +// +/// +/// 63 56 48 40 32 24 16 8 0 +/// |---------------|---------------|---------------|---------------| +/// +/// | reserved | priori| | header | +/// 0 | reserve zero |Source | ty bit| FEE id | size |version| +/// +/// 1 |ep | cru id |pcount|link id | memory size |offset nxt pack| +/// +/// 2 | orbit | reserved |bunch cross | +/// | data +/// 3 | reserved | format| +/// +/// 4 | zero | stop | page count | trigger type | +/// +/// 5 | reserved | +/// +/// 6 | zero | detector par | detector field | +/// +/// 5 | reserved | +/// +struct RAWDataHeaderV7 { + union { + // default value + uint64_t word0 = 0x00000000ffff4007; + // | | | version 67 + // | | | 8x64 bit words = 64 (0x40) byte + // | | invalid FEE id + // | priority bit 0 + struct { + uint64_t version : 8; /// bit 0 to 7: header version + uint64_t headerSize : 8; /// bit 8 to 15: header size + uint64_t feeId : 16; /// bit 16 to 31: FEE identifier + uint64_t priority : 8; /// bit 32 to 39: priority bit + uint64_t sourceID : 8; /// bit 40 to 47: source ID + uint64_t zero0 : 16; /// bit 48 to 63: zeroed + }; /// + }; /// + union { /// + uint64_t word1 = 0x0; /// data written by the CRU + struct { /// + uint32_t offsetToNext : 16; /// bit 64 to 79: offset to next packet in memory + uint32_t memorySize : 16; /// bit 80 to 95: memory size + uint32_t linkID : 8; /// bit 96 to 103: link id + uint32_t packetCounter : 8; /// bit 104 to 111: packet counter + uint16_t cruID : 12; /// bit 112 to 123: CRU ID + uint32_t endPointID : 4; /// bit 124 to 127: DATAPATH WRAPPER ID: number used to + }; /// identify one of the 2 End Points [0/1] + }; /// + union { /// + uint64_t word2 = 0x0; /// + struct { /// + uint32_t bunchCrossing : 12; /// bit 0 to 11: bunch crossing counter + uint32_t reserved2 : 20; /// bit 12 to 31: reserved + uint32_t orbit; /// bit 32 to 63: orbit + }; /// + }; /// + union { /// + uint64_t word3 = 0x0; /// + struct { /// + uint32_t dataFormat : 8; /// bit 0 to 7: data format: 0: padded (48 bit of padding + 80 bit of data) + /// 1: no padding, TOF LIKE (word#1 64 bit + word#0 64 bit = 128 bit) + /// 2: no padding, ITS/MFT like (appending words of 80 bit to build 128 bit) + uint64_t zero3 : 56; /// bit 8 to 63 zeroed + }; /// + }; /// + union { /// + uint64_t word4 = 0x0; /// + struct { /// + uint64_t triggerType : 32; /// bit 0 to 31: trigger type + uint64_t pageCnt : 16; /// bit 32 to 47: pages counter + uint64_t stop : 8; /// bit 48 to 53: stop code + uint64_t zero4 : 8; /// bit 54 to 63: zeroed + }; /// + }; /// + union { /// + uint64_t word5 = 0x0; /// bit 0 to 63: zeroed + }; /// + union { /// + uint64_t word6 = 0x0; /// + struct { /// + uint64_t detectorField : 32; /// bit 0 to 31: detector field + uint64_t detectorPAR : 16; /// bit 32 to 47: detector PAR (Pause and Reset) + uint64_t zero6 : 16; /// bit 48 to 63: zeroed + }; /// + }; /// + union { /// + uint64_t word7 = 0x0; /// bit 0 to 63: zeroed + }; +}; + /// RAWDataHeaderV6 /// V6 introduces a detector source ID field /// Description of the fields can be found here @@ -66,7 +162,7 @@ namespace header /// /// 6 | zero | detector par | detector field | /// -/// 5 | reserved | +/// 7 | reserved | struct RAWDataHeaderV6 { union { // default value @@ -159,7 +255,7 @@ struct RAWDataHeaderV6 { /// /// 6 | zero | detector par | detector field | /// -/// 5 | reserved | +/// 7 | reserved | struct RAWDataHeaderV5 { union { // default value @@ -254,7 +350,7 @@ struct RAWDataHeaderV5 { /// /// 6 |res| page count | stop | detector par |detector field | /// -/// 5 | reserved | +/// 7 | reserved | /// /// Field 1,3,5,7 are reserved fields and added to extend each word to 128 bit, marked /// grey in the documentation to indicate that those fields are added by the CRU @@ -425,9 +521,9 @@ struct RAWDataHeaderV2 { }; }; -using RAWDataHeader = RAWDataHeaderV6; // default version +using RAWDataHeader = RAWDataHeaderV7; // default version using RDHLowest = RAWDataHeaderV4; // link this to lowest version -using RDHHighest = RAWDataHeaderV6; // link this to highest version +using RDHHighest = RAWDataHeaderV7; // link this to highest version } // namespace header } // namespace o2 diff --git a/DataFormats/Headers/include/Headers/RDHAny.h b/DataFormats/Headers/include/Headers/RDHAny.h index 6105fb425d78c..8ccd441733b06 100644 --- a/DataFormats/Headers/include/Headers/RDHAny.h +++ b/DataFormats/Headers/include/Headers/RDHAny.h @@ -47,7 +47,8 @@ struct RDHAny { //------------------ service methods using RDHv4 = o2::header::RAWDataHeaderV4; // V3 == V4 using RDHv5 = o2::header::RAWDataHeaderV5; - using RDHv6 = o2::header::RAWDataHeaderV6; // update this for every new version + using RDHv6 = o2::header::RAWDataHeaderV6; + using RDHv7 = o2::header::RAWDataHeaderV7; // update this for every new version /// make sure we RDH is a legitimate RAWDataHeader template @@ -55,7 +56,7 @@ struct RDHAny { { #ifndef GPUCA_GPUCODE_DEVICE static_assert(std::is_same::value || std::is_same::value || - std::is_same::value, + std::is_same::value || std::is_same::value, "not an RDH"); #endif } @@ -66,7 +67,7 @@ struct RDHAny { { #ifndef GPUCA_GPUCODE_DEVICE static_assert(std::is_same::value || std::is_same::value || - std::is_same::value || std::is_same::value, + std::is_same::value || std::is_same::value || std::is_same::value, "not an RDH or RDHAny"); #endif } diff --git a/DataFormats/Headers/include/Headers/Stack.h b/DataFormats/Headers/include/Headers/Stack.h index 184733e35c339..9770df9fa54ef 100644 --- a/DataFormats/Headers/include/Headers/Stack.h +++ b/DataFormats/Headers/include/Headers/Stack.h @@ -14,9 +14,7 @@ #include "MemoryResources/MemoryResources.h" #include "Headers/DataHeader.h" -namespace o2 -{ -namespace header +namespace o2::header { //__________________________________________________________________________________________________ /// @struct Stack @@ -38,26 +36,27 @@ struct Stack { private: struct freeobj { - freeobj(memory_resource* mr) : resource(mr) {} + freeobj(memory_resource* mr, size_t s) : resource(mr), size(s) {} memory_resource* resource{nullptr}; - void operator()(std::byte* ptr) { resource->deallocate(ptr, 0, 0); } + size_t size{0}; + void operator()(std::byte* ptr) { resource->deallocate(ptr, size, alignof(std::max_align_t)); } }; public: - using allocator_type = boost::container::pmr::polymorphic_allocator; + using allocator_type = fair::mq::pmr::polymorphic_allocator; using value_type = std::byte; - using BufferType = std::unique_ptr; //this gives us proper default move semantics for free + using BufferType = std::unique_ptr; // this gives us proper default move semantics for free Stack() = default; Stack(Stack&&) = default; Stack(Stack&) = delete; Stack& operator=(Stack&) = delete; - Stack& operator=(Stack&&) = default; + Stack& operator=(Stack&&) = delete; - value_type* data() const { return buffer.get(); } - size_t size() const { return bufferSize; } + [[nodiscard]] value_type* data() const { return buffer.get(); } + [[nodiscard]] size_t size() const { return bufferSize; } allocator_type get_allocator() const { return allocator; } - const BaseHeader* first() const { return reinterpret_cast(this->data()); } + [[nodiscard]] const BaseHeader* first() const { return reinterpret_cast(this->data()); } static const BaseHeader* firstHeader(std::byte const* buf) { return BaseHeader::get(buf); } static const BaseHeader* lastHeader(std::byte const* buf) { @@ -89,9 +88,9 @@ struct Stack { /// all headers must derive from BaseHeader, in addition also other stacks can be passed to ctor. template >::value, int> = 0> + !std::is_convertible>::value, int> = 0> Stack(FirstArgType&& firstHeader, Headers&&... headers) - : Stack(boost::container::pmr::new_delete_resource(), std::forward(firstHeader), + : Stack(fair::mq::pmr::new_delete_resource(), std::forward(firstHeader), std::forward(headers)...) { } @@ -101,9 +100,13 @@ struct Stack { Stack(const allocator_type allocatorArg, Headers&&... headers) : allocator{allocatorArg}, bufferSize{calculateSize(std::forward(headers)...)}, - buffer{static_cast(allocator.resource()->allocate(bufferSize, alignof(std::max_align_t))), freeobj{allocator.resource()}} + buffer{static_cast(allocator.resource()->allocate(bufferSize, alignof(std::max_align_t))), freeobj{allocator.resource(), bufferSize}} { - inject(buffer.get(), std::forward(headers)...); + if constexpr (sizeof...(headers) > 1) { + injectAll(buffer.get(), std::forward(headers)...); + } else if (sizeof...(headers) == 1) { + injectBool(buffer.get(), std::forward(headers)..., false); + } } //______________________________________________________________________________________________ @@ -117,7 +120,7 @@ struct Stack { template constexpr static size_t calculateSize(T&& h) noexcept { - //if it's a pointer (to a stack) traverse it + // if it's a pointer (to a stack) traverse it if constexpr (std::is_convertible_v) { const BaseHeader* next = BaseHeader::get(std::forward(h)); if (!next) { @@ -128,23 +131,23 @@ struct Stack { size += next->size(); } return size; - //otherwise get the size directly + // otherwise get the size directly } else { return h.size(); } } - //recursion terminator + // recursion terminator constexpr static size_t calculateSize() { return 0; } private: - allocator_type allocator{boost::container::pmr::new_delete_resource()}; + allocator_type allocator{fair::mq::pmr::new_delete_resource()}; size_t bufferSize{0}; - BufferType buffer{nullptr, freeobj{allocator.resource()}}; + BufferType buffer{nullptr, freeobj{allocator.resource(), 0}}; //______________________________________________________________________________________________ template - static std::byte* inject(std::byte* here, T&& h, bool more = false) noexcept + static std::byte* injectBool(std::byte* here, T&& h, bool more) noexcept { using headerType = typename std::remove_cv::type>::type; if (here == nullptr) { @@ -189,11 +192,15 @@ struct Stack { //______________________________________________________________________________________________ template - static std::byte* inject(std::byte* here, T&& h, Args&&... args) noexcept + static std::byte* injectAll(std::byte* here, T&& h, Args&&... args) noexcept { bool more = hasNonEmptyArg(args...); - auto alsohere = inject(here, h, more); - return inject(alsohere, args...); + auto alsohere = injectBool(here, h, more); + if constexpr (sizeof...(args) > 1) { + return injectAll(alsohere, args...); + } else { + return injectBool(alsohere, args..., false); + } } //______________________________________________________________________________________________ @@ -222,7 +229,6 @@ struct Stack { } }; -} // namespace header -} // namespace o2 +} // namespace o2::header #endif // HEADERS_STACK_H diff --git a/DataFormats/Headers/include/Headers/SubframeMetadata.h b/DataFormats/Headers/include/Headers/SubframeMetadata.h deleted file mode 100644 index 255fa0ceb8db6..0000000000000 --- a/DataFormats/Headers/include/Headers/SubframeMetadata.h +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -#ifndef SUBFRAMEMETADATA_H -#define SUBFRAMEMETADATA_H - -#include - -namespace o2 -{ -namespace data_flow -{ - -struct SubframeMetadata { - // TODO: replace with timestamp struct - // IDEA: not timeframeID because can be calculcated with helper function - // QUESTION: isn't the duration set to ~22ms? - uint64_t startTime = ~(uint64_t)0; - uint64_t duration = ~(uint64_t)0; - - //further meta data to be added - - // putting data specific to FLP origin - int flpIndex; -}; - -// Helper function to derive the timeframe id from the actual timestamp. -// Timestamp is in nanoseconds. Each Timeframe is ~22ms i.e. 2^17 nanoseconds, -// so we can get a unique id by dividing by the timeframe period and masking -// the lower 16 bits. Overlaps will only happen every ~ 22 minutes. -constexpr uint16_t - timeframeIdFromTimestamp(uint64_t timestamp, uint64_t timeFrameDuration) -{ - return (timestamp / timeFrameDuration) & 0xffff; -} - -// A Mockup class to describe some TPC-like payload -struct TPCTestCluster { - float x = 0.f; - float y = 0.f; - float z = 1.5f; - float q = 0.; - uint64_t timeStamp; // the time this thing was digitized/recorded -}; - -struct TPCTestPayload { - std::vector clusters; -}; - -// a mockup class to describe some "ITS" payload -struct ITSRawData { - float x = -1.; - float y = 1.; - uint64_t timeStamp; -}; - -} // namespace data_flow -} // namespace o2 - -#endif diff --git a/DataFormats/Headers/include/Headers/TimeStamp.h b/DataFormats/Headers/include/Headers/TimeStamp.h index bd4bd8c8b94e9..b6d2b26ea4822 100644 --- a/DataFormats/Headers/include/Headers/TimeStamp.h +++ b/DataFormats/Headers/include/Headers/TimeStamp.h @@ -23,9 +23,7 @@ #include #include // for std::integral_constant -namespace o2 -{ -namespace header +namespace o2::header { // https://lhc-machine-outreach.web.cern.ch/lhc-machine-outreach/collisions.htm @@ -140,12 +138,14 @@ class TimeStamp static_assert(std::is_same::value && std::is_same::value, "only clock and duration types defining the rep and period member types are allowed"); using duration = std::chrono::duration; - if (mUnit == sClockLHC) { + static_assert(sizeof(mUnit) == sizeof(sClockLHC), "size mismatch of mUnit and sClockLHC"); + if (memcmp(&mUnit, &sClockLHC, sizeof(sClockLHC)) == 0) { // cast each part individually, if the precision of the return type // is smaller the values are simply truncated return std::chrono::duration_cast(LHCOrbitClock::duration(mPeriod) + LHCBunchClock::duration(mBCNumber)); } - if (mUnit == sMicroSeconds) { + static_assert(sizeof(mUnit) == sizeof(sMicroSeconds), "size mismatch of mUnit and sMicroSeconds"); + if (memcmp(&mUnit, &sMicroSeconds, sizeof(sMicroSeconds)) == 0) { // TODO: is there a better way to mark the subticks invalid for the // micro seconds representation? First step is probably to remove/rename the // variable @@ -159,7 +159,7 @@ class TimeStamp } // TODO: implement transformation from one unit to the other - //void transform(const TimeUnitID& unit) { + // void transform(const TimeUnitID& unit) { // if (mUnit == unit) return; // ... //} @@ -182,7 +182,6 @@ class TimeStamp }; }; }; -} //namespace header -} //namespace o2 +} // namespace o2::header #endif diff --git a/DataFormats/Headers/test/testDataHeader.cxx b/DataFormats/Headers/test/testDataHeader.cxx index 0703fc6c3ae71..2403c1a6230be 100644 --- a/DataFormats/Headers/test/testDataHeader.cxx +++ b/DataFormats/Headers/test/testDataHeader.cxx @@ -280,8 +280,8 @@ BOOST_AUTO_TEST_CASE(headerStack_test) Stack s2{s1, meta}; BOOST_CHECK(s2.size() == s1.size() + sizeof(decltype(meta))); - //check dynamic construction - where we don't have the type information and need to - //work with BaseHeader pointers + // check dynamic construction - where we don't have the type information and need to + // work with BaseHeader pointers const test::MetaHeader thead{2}; o2::header::BaseHeader const* bname = reinterpret_cast(&thead); Stack ds2(s1, *bname); @@ -313,8 +313,8 @@ BOOST_AUTO_TEST_CASE(headerStack_test) BOOST_REQUIRE(h3 != nullptr); BOOST_CHECK(h3->secret == 42); - //test constructing from a buffer and an additional header - using namespace boost::container::pmr; + // test constructing from a buffer and an additional header + using namespace fair::mq::pmr; Stack s5(new_delete_resource(), s1.data(), Stack{}, meta); BOOST_CHECK(s5.size() == s1.size() + sizeof(meta)); // check if we can find the header even though there was an empty stack in the middle diff --git a/DataFormats/Legacy/HLT/include/AliceHLT/TPCRawCluster.h b/DataFormats/Legacy/HLT/include/AliceHLT/TPCRawCluster.h deleted file mode 100644 index 59be2a86c3c2b..0000000000000 --- a/DataFormats/Legacy/HLT/include/AliceHLT/TPCRawCluster.h +++ /dev/null @@ -1,232 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -//-*- Mode: C++ -*- - -#ifndef TPCRAWCLUSTER_H -#define TPCRAWCLUSTER_H -//**************************************************************************** -//* This file is free software: you can redistribute it and/or modify * -//* it under the terms of the GNU General Public License as published by * -//* the Free Software Foundation, either version 3 of the License, or * -//* (at your option) any later version. * -//* * -//* Primary Authors: Matthias Richter * -//* * -//* The authors make no claims about the suitability of this software for * -//* any purpose. It is provided "as is" without express or implied warranty. * -//**************************************************************************** - -// @file TPCRawCluster.h -// @author Matthias Richter -// @since 2015-09-27 -// @brief ALICE HLT TPC raw cluster structure and tools - -#include -#include // ifstream -#include // memcpy - -namespace o2 -{ -namespace AliceHLT -{ - -/** - * @struct RawCluster - * This is a redefinition from AliRoot/HLT/TPCLib/AliHLTTPCRawCluster.h for the - * sake of reading HLT TPC raw cluster files into O2. - * - * TODO: there is no dependence on AliRoot, however, a test needs to be added - * to check consistency if AliRoot is available in the build. - */ -struct RawCluster { - - int16_t GetPadRow() const { return fPadRow; } - float GetPad() const { return fPad; } - float GetTime() const { return fTime; } - float GetSigmaPad2() const { return fSigmaPad2; } - float GetSigmaTime2() const { return fSigmaTime2; } - int32_t GetCharge() const { return fCharge; } - int32_t GetQMax() const { return fQMax; } - bool GetFlagSplitPad() const { return (fFlags & (1 << 0)); } - bool GetFlagSplitTime() const { return (fFlags & (1 << 1)); } - bool GetFlagSplitAny() const { return (fFlags & 3); } - uint16_t GetFlags() const { return (fFlags); } - - int16_t fPadRow; - uint16_t fFlags; //Flags: (1 << 0): Split in pad direction - // (1 << 1): Split in time direction - //During cluster merging, flags are or'd - float fPad; - float fTime; - float fSigmaPad2; - float fSigmaTime2; - uint16_t fCharge; - uint16_t fQMax; -}; - -/** - * @struct RawClusterData - * Header data struct for a raw cluster block - */ -struct RawClusterData { - uint32_t fVersion; // version number - uint32_t fCount; // number of clusters - RawCluster fClusters[0]; // array of clusters -}; - -std::ostream& operator<<(std::ostream& stream, const RawCluster& cluster) -{ - stream << "TPCRawCluster:" - << " " << cluster.GetPadRow() - << " " << cluster.GetPad() - << " " << cluster.GetTime() - << " " << cluster.GetSigmaPad2() - << " " << cluster.GetSigmaTime2() - << " " << cluster.GetCharge() - << " " << cluster.GetQMax(); - return stream; -} - -/** - * @class RawClusterArray Wrapper to binary data block of HLT TPC raw clusters - * Container class which provides access to the content of a binary block of - * HLT TPC raw clusters. - */ -class RawClusterArray -{ - public: - RawClusterArray() : mBuffer(nullptr), mBufferSize(0), mNClusters(0), mClusters(NULL), mClustersEnd(NULL) {} - RawClusterArray(const char* filename) : mBuffer(nullptr), mBufferSize(0), mNClusters(0), mClusters(NULL), mClustersEnd(NULL) - { - init(filename); - } - RawClusterArray(unsigned char* buffer, int size) : mBuffer(nullptr), mBufferSize(0), mNClusters(0), mClusters(NULL), mClustersEnd(NULL) - { - init(buffer, size); - } - ~RawClusterArray() {} - - typedef uint8_t Buffer_t; - - int init(const char* filename) - { - std::ifstream input(filename, std::ifstream::binary); - clear(0); - if (input) { - // get length of file: - input.seekg(0, input.end); - int length = input.tellg(); - input.seekg(0, input.beg); - - // allocate memory: - mBuffer = new Buffer_t[length]; - mBufferSize = length; - - // read data as a block: - input.read(reinterpret_cast(mBuffer), length); - if (!input.good()) { - clear(-1); - std::cerr << "failed to read " << length << " byte(s) from file " << filename << std::endl; - } - - input.close(); - return init(); - } - std::cerr << "failed to open file " << filename << std::endl; - return -1; - } - - int init(unsigned char* buffer, int size) - { - if (!buffer || size <= 0) - return -1; - clear(0); - mBuffer = new Buffer_t[size]; - mBufferSize = size; - memcpy(mBuffer, buffer, size); - return init(); - } - - int GetNClusters() const { return mNClusters; } - - RawCluster* begin() { return mClusters; } - - RawCluster* end() { return mClustersEnd; } - - RawCluster& operator[](int i) - { - if (i + 1 > mNClusters) { - // runtime exeption? - static RawCluster dummy; - return dummy; - } - return *(mClusters + i); - } - - void print() { print(std::cout); } - - template - StreamT& print(StreamT& stream) - { - std::cout << "RawClusterArray: " << mNClusters << " cluster(s)" << std::endl; - for (RawCluster* cluster = mClusters; cluster != mClustersEnd; cluster++) { - std::cout << " " << *cluster << std::endl; - } - return stream; - } - - private: - int init() - { - if (mBuffer == nullptr || mBufferSize == 0) - return 0; - if (mBufferSize < sizeof(RawClusterData)) - return -1; - RawClusterData& clusterData = *reinterpret_cast(mBuffer); - - if (clusterData.fCount * sizeof(RawCluster) + sizeof(RawClusterData) > mBufferSize) { - std::cerr << "Format error, " << clusterData.fCount << " cluster(s) " - << "would require " - << (clusterData.fCount * sizeof(RawCluster) + sizeof(RawClusterData)) - << " byte(s), but only " << mBufferSize << " available" << std::endl; - return clear(-1); - } - - mNClusters = clusterData.fCount; - mClusters = clusterData.fClusters; - mClustersEnd = mClusters + mNClusters; - - return mNClusters; - } - - int clear(int returnValue) - { - mNClusters = 0; - mClusters = NULL; - mClustersEnd = NULL; - delete[] mBuffer; - mBuffer = nullptr; - mBufferSize = 0; - - return returnValue; - } - - Buffer_t* mBuffer; - int mBufferSize; - int mNClusters; - RawCluster* mClusters; - RawCluster* mClustersEnd; -}; - -}; // namespace AliceHLT -}; // namespace o2 -#endif diff --git a/DataFormats/MemoryResources/CMakeLists.txt b/DataFormats/MemoryResources/CMakeLists.txt index 6a31d35872054..7075504786545 100644 --- a/DataFormats/MemoryResources/CMakeLists.txt +++ b/DataFormats/MemoryResources/CMakeLists.txt @@ -9,9 +9,8 @@ # granted to it by virtue of its status as an Intergovernmental Organization # or submit itself to any jurisdiction. -o2_add_library(MemoryResources - SOURCES src/MemoryResources.cxx - PUBLIC_LINK_LIBRARIES FairMQ::FairMQ) +o2_add_header_only_library(MemoryResources INTERFACE_LINK_LIBRARIES FairMQ::FairMQ) + if(NOT APPLE) o2_add_test(MemoryResources SOURCES test/testMemoryResources.cxx diff --git a/DataFormats/MemoryResources/include/MemoryResources/MemoryResources.h b/DataFormats/MemoryResources/include/MemoryResources/MemoryResources.h index 79796808c5dcc..b52f5c715575e 100644 --- a/DataFormats/MemoryResources/include/MemoryResources/MemoryResources.h +++ b/DataFormats/MemoryResources/include/MemoryResources/MemoryResources.h @@ -104,179 +104,23 @@ class MessageResource : public FairMQMemoryResource mUpstream->deallocate(p, bytes, alignment < 64 ? 64 : alignment); return; } - bool do_is_equal(const memory_resource& other) const noexcept override + bool do_is_equal(const memory_resource& /*other*/) const noexcept override { // since this uniquely owns the message it can never be equal to anybody else return false; } }; -//__________________________________________________________________________________________________ -// A spectator pmr memory resource which only watches the memory of the underlying buffer, does not -// carry out real allocation. It owns the underlying buffer which is destroyed on deallocation. -template -class SpectatorMemoryResource : public boost::container::pmr::memory_resource -{ - public: - using buffer_type = BufferType; - - SpectatorMemoryResource() noexcept = delete; - SpectatorMemoryResource(const SpectatorMemoryResource&) noexcept = delete; - SpectatorMemoryResource(SpectatorMemoryResource&&) noexcept = default; - SpectatorMemoryResource& operator=(const SpectatorMemoryResource&) = delete; - SpectatorMemoryResource& operator=(SpectatorMemoryResource&&) = default; - ~SpectatorMemoryResource() noexcept override = default; - - // the resource is the pointer managed by unique_ptr - template - SpectatorMemoryResource(std::unique_ptr&& buffer, size_t size) - : mBuffer{std::move(buffer)}, mPointer{mBuffer.get()}, mSize{size} - { - } - - // the resource is the data of the vector managed by unique ptr - template - SpectatorMemoryResource(std::unique_ptr, typename buffer_type::deleter_type>&& buffer) - : mBuffer{std::move(buffer)}, mPointer{mBuffer->data()}, mSize{mBuffer->size() * sizeof(T)} - { - } - - // TODO: the underlying resource can be directly the vector or the read only buffer - protected: - void* do_allocate(std::size_t bytes, std::size_t alignment) override - { - if (mSize > 0) { - if (bytes > mSize) { - throw std::bad_alloc(); - } - mSize = 0; - return mPointer; - } - throw std::runtime_error("Can not allocate: this memory resource is only supposed to provide spectator access to external buffer"); - } - - void do_deallocate(void* p, std::size_t bytes, std::size_t alignment) override - { - if (p == mPointer) { - mBuffer.reset(); - mPointer = nullptr; - } else if (mPointer == nullptr) { - // there is an error in the logic flow, this should never be called more than once - throw std::logic_error("underlying controlled resource has been released already"); - } else { - throw std::logic_error("this resource can only deallocate the controlled resource pointer"); - } - } - bool do_is_equal(const memory_resource& other) const noexcept override - { - // uniquely owns the underlying resource, can never be equal to any other instance - return false; - } - - private: - buffer_type mBuffer; - void* mPointer = nullptr; - size_t mSize = 0; -}; - -//__________________________________________________________________________________________________ -// This in general (as in STL) is a bad idea, but here it is safe to inherit from an allocator since we -// have no additional data and only override some methods so we don't get into slicing and other problems. -template -class SpectatorAllocator : public boost::container::pmr::polymorphic_allocator -{ - public: - using boost::container::pmr::polymorphic_allocator::polymorphic_allocator; - using propagate_on_container_move_assignment = std::true_type; - - // skip default construction of empty elements - // this is important for two reasons: one: it allows us to adopt an existing buffer (e.g. incoming message) and - // quickly construct large vectors while skipping the element initialization. - template - void construct(U*) - { - } - - // dont try to call destructors, makes no sense since resource is managed externally AND allowed - // types cannot have side effects - template - void destroy(U*) - { - } - - T* allocate(size_t size) { return reinterpret_cast(this->resource()->allocate(size * sizeof(T), 64)); } - void deallocate(T* ptr, size_t size) - { - this->resource()->deallocate(const_cast::type*>(ptr), size); - } -}; - -//__________________________________________________________________________________________________ -/// This allocator has a pmr-like interface, but keeps the unique MessageResource as internal state, -/// allowing full resource (associated message) management internally without any global state. -template -class OwningMessageSpectatorAllocator -{ - public: - using value_type = T; - - MessageResource mResource; - - OwningMessageSpectatorAllocator() noexcept = default; - OwningMessageSpectatorAllocator(const OwningMessageSpectatorAllocator&) noexcept = default; - OwningMessageSpectatorAllocator(OwningMessageSpectatorAllocator&&) noexcept = default; - OwningMessageSpectatorAllocator(MessageResource&& resource) noexcept : mResource{resource} {} - - template - OwningMessageSpectatorAllocator(const OwningMessageSpectatorAllocator& other) noexcept : mResource(other.mResource) - { - } - - OwningMessageSpectatorAllocator& operator=(const OwningMessageSpectatorAllocator& other) - { - mResource = other.mResource; - return *this; - } - - OwningMessageSpectatorAllocator select_on_container_copy_construction() const - { - return OwningMessageSpectatorAllocator(); - } - - boost::container::pmr::memory_resource* resource() { return &mResource; } - - // skip default construction of empty elements - // this is important for two reasons: one: it allows us to adopt an existing buffer (e.g. incoming message) and - // quickly construct large vectors while skipping the element initialization. - template - void construct(U*) - { - } - - // dont try to call destructors, makes no sense since resource is managed externally AND allowed - // types cannot have side effects - template - void destroy(U*) - { - } - - T* allocate(size_t size) { return reinterpret_cast(mResource.allocate(size * sizeof(T), 64)); } - void deallocate(T* ptr, size_t size) - { - mResource.deallocate(const_cast::type*>(ptr), size); - } -}; - // The NoConstructAllocator behaves like the normal pmr vector but does not call constructors / destructors template -class NoConstructAllocator : public boost::container::pmr::polymorphic_allocator +class NoConstructAllocator : public fair::mq::pmr::polymorphic_allocator { public: - using boost::container::pmr::polymorphic_allocator::polymorphic_allocator; + using fair::mq::pmr::polymorphic_allocator::polymorphic_allocator; using propagate_on_container_move_assignment = std::true_type; template - NoConstructAllocator(Args&&... args) : boost::container::pmr::polymorphic_allocator(std::forward(args)...) + NoConstructAllocator(Args&&... args) : fair::mq::pmr::polymorphic_allocator(std::forward(args)...) { } @@ -301,20 +145,9 @@ class NoConstructAllocator : public boost::container::pmr::polymorphic_allocator //__________________________________________________________________________________________________ //__________________________________________________________________________________________________ -using ByteSpectatorAllocator = SpectatorAllocator; -using BytePmrAllocator = boost::container::pmr::polymorphic_allocator; +using BytePmrAllocator = fair::mq::pmr::polymorphic_allocator; template -using vector = std::vector>; - -//__________________________________________________________________________________________________ -/// Return a std::vector spanned over the contents of the message, takes ownership of the message -template -auto adoptVector(size_t nelem, fair::mq::MessagePtr message) -{ - static_assert(std::is_trivially_destructible::value); - return std::vector>( - nelem, OwningMessageSpectatorAllocator(MessageResource{std::move(message)})); -}; +using vector = std::vector>; //__________________________________________________________________________________________________ /// Get the allocator associated to a transport factory @@ -323,6 +156,6 @@ inline static FairMQMemoryResource* getTransportAllocator(fair::mq::TransportFac return *factory; } -} //namespace o2::pmr +} // namespace o2::pmr #endif diff --git a/DataFormats/MemoryResources/src/MemoryResources.cxx b/DataFormats/MemoryResources/src/MemoryResources.cxx deleted file mode 100644 index c8c4f915e6f3f..0000000000000 --- a/DataFormats/MemoryResources/src/MemoryResources.cxx +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -#include "MemoryResources/MemoryResources.h" diff --git a/DataFormats/MemoryResources/test/testMemoryResources.cxx b/DataFormats/MemoryResources/test/testMemoryResources.cxx index 592e76c8ef1f4..a49cd00d75255 100644 --- a/DataFormats/MemoryResources/test/testMemoryResources.cxx +++ b/DataFormats/MemoryResources/test/testMemoryResources.cxx @@ -47,12 +47,12 @@ int testData::nconstructions = 0; BOOST_AUTO_TEST_CASE(transportallocatormap_test) { - size_t session{fair::mq::tools::UuidHash()}; + size_t session{(size_t)getpid() * 1000}; fair::mq::ProgOptions config; config.SetProperty("session", std::to_string(session)); auto factoryZMQ = fair::mq::TransportFactory::CreateTransportFactory("zeromq"); - auto factorySHM = fair::mq::TransportFactory::CreateTransportFactory("shmem"); + auto factorySHM = fair::mq::TransportFactory::CreateTransportFactory("shmem", "transportallocatormap_test", &config); auto allocZMQ = getTransportAllocator(factoryZMQ.get()); auto allocSHM = getTransportAllocator(factorySHM.get()); BOOST_CHECK(allocZMQ != nullptr && allocSHM != allocZMQ); @@ -60,16 +60,16 @@ BOOST_AUTO_TEST_CASE(transportallocatormap_test) BOOST_CHECK(_tmp == allocZMQ); } -using namespace boost::container::pmr; +using namespace fair::mq::pmr; BOOST_AUTO_TEST_CASE(allocator_test) { - size_t session{fair::mq::tools::UuidHash()}; + size_t session{(size_t)getpid() * 1000 + 1}; fair::mq::ProgOptions config; config.SetProperty("session", std::to_string(session)); auto factoryZMQ = fair::mq::TransportFactory::CreateTransportFactory("zeromq"); - auto factorySHM = fair::mq::TransportFactory::CreateTransportFactory("shmem"); + auto factorySHM = fair::mq::TransportFactory::CreateTransportFactory("shmem", "allocator_test", &config); auto allocZMQ = getTransportAllocator(factoryZMQ.get()); auto allocSHM = getTransportAllocator(factorySHM.get()); @@ -88,26 +88,17 @@ BOOST_AUTO_TEST_CASE(allocator_test) } testData::nconstructions = 0; - { - std::vector> v(SpectatorAllocator{allocZMQ}); - v.reserve(3); - BOOST_CHECK(allocZMQ->getNumberOfMessages() == 1); - v.emplace_back(1); - v.emplace_back(2); - v.emplace_back(3); - BOOST_CHECK(testData::nconstructions == 3); - } BOOST_CHECK(allocZMQ->getNumberOfMessages() == 0); } BOOST_AUTO_TEST_CASE(getMessage_test) { - size_t session{fair::mq::tools::UuidHash()}; + size_t session{(size_t)getpid() * 1000 + 2}; fair::mq::ProgOptions config; config.SetProperty("session", std::to_string(session)); auto factoryZMQ = fair::mq::TransportFactory::CreateTransportFactory("zeromq"); - auto factorySHM = fair::mq::TransportFactory::CreateTransportFactory("shmem"); + auto factorySHM = fair::mq::TransportFactory::CreateTransportFactory("shmem", "getMessage_test", &config); auto allocZMQ = getTransportAllocator(factoryZMQ.get()); auto allocSHM = getTransportAllocator(factorySHM.get()); @@ -147,73 +138,6 @@ BOOST_AUTO_TEST_CASE(getMessage_test) messageArray = static_cast(message->GetData()); BOOST_CHECK(messageArray[0] == 4 && messageArray[1] == 5 && messageArray[2] == 6); - { - std::vector> v(SpectatorAllocator{allocSHM}); - } -} - -BOOST_AUTO_TEST_CASE(adoptVector_test) -{ - size_t session{fair::mq::tools::UuidHash()}; - fair::mq::ProgOptions config; - config.SetProperty("session", std::to_string(session)); - - auto factoryZMQ = fair::mq::TransportFactory::CreateTransportFactory("zeromq"); - auto factorySHM = fair::mq::TransportFactory::CreateTransportFactory("shmem"); - auto allocZMQ = getTransportAllocator(factoryZMQ.get()); - auto allocSHM = getTransportAllocator(factorySHM.get()); - - testData::nconstructions = 0; - - //Create a bogus message - auto message = factoryZMQ->CreateMessage(3 * sizeof(testData)); - auto messageAddr = message.get(); - testData tmpBuf[3] = {3, 2, 1}; - std::memcpy(message->GetData(), tmpBuf, 3 * sizeof(testData)); - - auto adoptedOwner = adoptVector(3, std::move(message)); - BOOST_CHECK(adoptedOwner[0].i == 3); - BOOST_CHECK(adoptedOwner[1].i == 2); - BOOST_CHECK(adoptedOwner[2].i == 1); - - auto reclaimedMessage = o2::pmr::getMessage(std::move(adoptedOwner)); - BOOST_CHECK(reclaimedMessage.get() == messageAddr); - BOOST_CHECK(adoptedOwner.size() == 0); - - auto modified = adoptVector(3, std::move(reclaimedMessage)); - modified.emplace_back(9); - BOOST_CHECK(modified[3].i == 9); - BOOST_CHECK(modified.size() == 4); - BOOST_CHECK(testData::nconstructions == 7); - auto modifiedMessage = getMessage(std::move(modified)); - BOOST_CHECK(modifiedMessage != nullptr); - BOOST_CHECK(modifiedMessage.get() != messageAddr); -} - -BOOST_AUTO_TEST_CASE(test_SpectatorMemoryResource) -{ - constexpr int size = 5; - auto buffer = std::make_unique(size); - auto const* bufferdata = buffer.get(); - SpectatorMemoryResource resource(std::move(buffer), size * sizeof(int)); - std::vector> bufferclone(size, o2::pmr::SpectatorAllocator(&resource)); - BOOST_CHECK(bufferclone.data() == bufferdata); - BOOST_CHECK(bufferclone.size() == size); - BOOST_CHECK_THROW(bufferclone.resize(2 * size), std::runtime_error); - - auto vecbuf = std::make_unique>(size); - auto const* vectordata = vecbuf->data(); - SpectatorMemoryResource vecresource(std::move(vecbuf)); - std::vector> vecclone(size, o2::pmr::SpectatorAllocator(&vecresource)); - BOOST_CHECK(vecclone.data() == vectordata); - BOOST_CHECK(vecclone.size() == size); - BOOST_CHECK_THROW(vecclone.resize(2 * size), std::runtime_error); - - std::vector> vecmove; - vecmove = std::move(vecclone); - BOOST_CHECK(vecclone.size() == 0); - BOOST_CHECK(vecmove.data() == vectordata); - BOOST_CHECK(vecmove.size() == size); } }; // namespace o2::pmr diff --git a/DataFormats/Parameters/CMakeLists.txt b/DataFormats/Parameters/CMakeLists.txt index 92c39aafe5073..ed137a42f6565 100644 --- a/DataFormats/Parameters/CMakeLists.txt +++ b/DataFormats/Parameters/CMakeLists.txt @@ -14,9 +14,10 @@ o2_add_library(DataFormatsParameters src/GRPLHCIFData.cxx src/GRPECSObject.cxx src/GRPMagField.cxx - PUBLIC_LINK_LIBRARIES FairRoot::Base O2::CommonConstants - O2::CommonTypes - O2::DetectorsCommonDataFormats O2::SimConfig) + src/AggregatedRunInfo.cxx + PUBLIC_LINK_LIBRARIES FairRoot::Base O2::CommonConstants + O2::CommonTypes O2::CCDB + O2::DetectorsCommonDataFormats) o2_target_root_dictionary(DataFormatsParameters HEADERS include/DataFormatsParameters/ECSDataAdapters.h @@ -24,12 +25,13 @@ o2_target_root_dictionary(DataFormatsParameters include/DataFormatsParameters/GRPLHCIFData.h include/DataFormatsParameters/GRPECSObject.h include/DataFormatsParameters/GRPMagField.h + include/DataFormatsParameters/AggregatedRunInfo.h LINKDEF src/ParametersDataLinkDef.h) o2_add_executable(simgrp-tool COMPONENT_NAME grp SOURCES src/GRPTool.cxx - PUBLIC_LINK_LIBRARIES Boost::program_options O2::DataFormatsParameters) + PUBLIC_LINK_LIBRARIES Boost::program_options O2::DataFormatsParameters O2::SimConfig) # note we are explicitely giving the LINKDEF parameter as the LinkDef does not # follow the usual naming scheme [module]LinkDef.h diff --git a/DataFormats/Parameters/include/DataFormatsParameters/AggregatedRunInfo.h b/DataFormats/Parameters/include/DataFormatsParameters/AggregatedRunInfo.h new file mode 100644 index 0000000000000..d0347114b5b4c --- /dev/null +++ b/DataFormats/Parameters/include/DataFormatsParameters/AggregatedRunInfo.h @@ -0,0 +1,70 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file GRPECSObject.h +/// \brief Header of the AggregatedRunInfo struct +/// \author ruben.shahoyan@cern.ch sandro.wenzel@cern.ch + +#ifndef ALICEO2_DATA_AGGREGATEDRUNINFO_H_ +#define ALICEO2_DATA_AGGREGATEDRUNINFO_H_ + +#include +#include "CCDB/BasicCCDBManager.h" + +namespace o2::parameters +{ + +class GRPECSObject; +class GRPLHCIFData; + +/// Composite struct where one may collect important global properties of data "runs" +/// aggregated from various sources (GRPECS, RunInformation CCDB entries, etc.). +/// Also offers the authoritative algorithms to collect these information for easy reuse +/// across various algorithms (anchoredMC, analysis, ...) +struct AggregatedRunInfo { + int runNumber = 0; // run number + int64_t sor = 0; // best known timestamp for the start of run + int64_t eor = 0; // best known timestamp for end of run + int64_t orbitsPerTF = 0; // number of orbits per TF (takes precedence over that in GRPECS) + int64_t orbitReset = 0; // timestamp of orbit reset before run + int64_t orbitSOR = 0; // orbit when run starts after orbit reset + int64_t orbitEOR = 0; // orbit when run ends after orbit reset + + // we may have pointers to actual data source objects GRPECS, ... + const o2::parameters::GRPECSObject* grpECS = nullptr; // pointer to GRPECSobject (fetched during struct building) + const o2::parameters::GRPLHCIFData* grpLHC = nullptr; + + static AggregatedRunInfo buildAggregatedRunInfo(int runnumber, long sorMS, long eorMS, long orbitResetMUS, const o2::parameters::GRPECSObject* grpecs, const std::vector* ctfFirstRunOrbitVec, const o2::parameters::GRPLHCIFData* grplhcif = nullptr); + + // fills and returns AggregatedRunInfo for a given data run number. + static AggregatedRunInfo buildAggregatedRunInfo_DATA(o2::ccdb::CCDBManagerInstance& ccdb, int runnumber); + + // Returns the meta-data (MCProdInfo) associated to production lpm_prod_tag (performed by username) + static std::map getMCProdInfo(o2::ccdb::CCDBManagerInstance& ccdb, int runnumber, + std::string const& lpm_prod_tag, std::string const& username = "aliprod"); + + // function that adjusts with values from MC + void adjust_from_MC(o2::ccdb::CCDBManagerInstance& ccdb, int run_number, std::string const& lpm_prod_tag, std::string const& username = "aliprod"); + + // Fills and returns AggregatedRunInfo for a given run number. + // If a non-empty lpm_prod_tag is given, it will potentially override values with specifics from a + // MC production identified by that tag and username. + static AggregatedRunInfo buildAggregatedRunInfo(o2::ccdb::CCDBManagerInstance& ccdb, + int runnumber, + std::string const& lpm_prod_tag = "", + std::string const& username = "aliprod"); + + ClassDefNV(AggregatedRunInfo, 1); +}; + +} // namespace o2::parameters + +#endif diff --git a/DataFormats/Parameters/include/DataFormatsParameters/ECSDataAdapters.h b/DataFormats/Parameters/include/DataFormatsParameters/ECSDataAdapters.h index ee24d63936920..dab573b5ec5fe 100644 --- a/DataFormats/Parameters/include/DataFormatsParameters/ECSDataAdapters.h +++ b/DataFormats/Parameters/include/DataFormatsParameters/ECSDataAdapters.h @@ -46,6 +46,9 @@ enum RunType : int { CALIBRATION, COSMICS, SYNTHETIC, + NOISE, + CALIBRATION_PULSE_LENGTH, + CALIBRATION_VRESETD, NRUNTYPES }; static constexpr std::array RunTypeNames = { @@ -64,7 +67,10 @@ static constexpr std::array RunTypeNames = { "CALIBRATION_ALPIDE_SCAN", "CALIBRATION", "COSMICS", - "SYNTHETIC"}; + "SYNTHETIC", + "NOISE", + "CALIBRATION_PULSE_LENGTH", + "CALIBRATION_VRESETD"}; //_______________________________________________ static RunType string2RunType(const std::string& rts) @@ -103,6 +109,8 @@ static std::string getRawDataPersistencyMode(const std::string& runType, bool im case RunType::CALIBRATION_FHR: case RunType::CALIBRATION_ALPIDE_SCAN: case RunType::CALIBRATION: + case RunType::CALIBRATION_PULSE_LENGTH: + case RunType::CALIBRATION_VRESETD: ret = "calib"; default: break; @@ -114,4 +122,4 @@ static std::string getRawDataPersistencyMode(const std::string& runType, bool im } // namespace parameters } // namespace o2 -#endif \ No newline at end of file +#endif diff --git a/DataFormats/Parameters/include/DataFormatsParameters/GRPECSObject.h b/DataFormats/Parameters/include/DataFormatsParameters/GRPECSObject.h index 6f69b6ebc1793..e76660f0c8c9b 100644 --- a/DataFormats/Parameters/include/DataFormatsParameters/GRPECSObject.h +++ b/DataFormats/Parameters/include/DataFormatsParameters/GRPECSObject.h @@ -52,6 +52,14 @@ class GRPECSObject timePoint getTimeEnd() const { return mTimeEnd; } void setTimeEnd(timePoint t) { mTimeEnd = t; } + timePoint getTimeStartCTPasSet() const { return mTimeStartCTP; } + timePoint getTimeStartCTP() const { return mTimeStartCTP ? mTimeStartCTP : mTimeStart; } + void setTimeStartCTP(timePoint t) { mTimeStartCTP = t; } + + timePoint getTimeEndCTPasSet() const { return mTimeEndCTP; } + timePoint getTimeEndCTP() const { return mTimeEndCTP ? mTimeEndCTP : mTimeEnd; } + void setTimeEndCTP(timePoint t) { mTimeEndCTP = t; } + void setNHBFPerTF(uint32_t n) { mNHBFPerTF = n; } uint32_t getNHBFPerTF() const { return mNHBFPerTF; } @@ -109,6 +117,19 @@ class GRPECSObject /// same with comma-separate list of detector names DetID::mask_t getDetsReadOut(const std::string& only, const std::string& skip = "") const { return getDetsReadOut(DetID::getMask(only), DetID::getMask(skip)); } + // methods to manipulate the list of FLPs in the run + const std::vector& getListOfFLPs() const { return mFLPs; } + void addFLP(unsigned short flp) { mFLPs.push_back(flp); } + void setListOfFLPs(const std::vector& listFLPs) { mFLPs = listFLPs; } + bool getFLPStatus(unsigned short flp) const + { + return std::find(mFLPs.begin(), mFLPs.end(), flp) == mFLPs.end() ? false : true; + } + bool listOfFLPsSet() const + { + return mFLPs.size() > 0 ? true : false; + } + /// print itself void print() const; @@ -116,8 +137,10 @@ class GRPECSObject static constexpr bool alwaysTriggeredRO(DetID::ID det) { return DefTriggeredDets[det]; } private: - timePoint mTimeStart = 0; ///< DAQ_time_start entry from DAQ logbook - timePoint mTimeEnd = 0; ///< DAQ_time_end entry from DAQ logbook + timePoint mTimeStart = 0; ///< ECS start time + timePoint mTimeEnd = 0; ///< ECS end time + timePoint mTimeStartCTP = 0; ///< CTP start time + timePoint mTimeEndCTP = 0; ///< CTP end time uint32_t mNHBFPerTF = 128; /// Number of HBFrames per TF @@ -128,11 +151,12 @@ class GRPECSObject int mRun = 0; ///< run identifier RunType mRunType = RunType::NONE; ///< run type std::string mDataPeriod{}; ///< name of the period + std::vector mFLPs; ///< to store which FLPs were in the processing // detectors which are always readout in triggered mode. Others are continuous by default but exceptionally can be triggered static constexpr DetID::mask_t DefTriggeredDets = DetID::getMask(DetID::TRD) | DetID::getMask(DetID::PHS) | DetID::getMask(DetID::CPV) | DetID::getMask(DetID::EMC) | DetID::getMask(DetID::HMP); - ClassDefNV(GRPECSObject, 4); + ClassDefNV(GRPECSObject, 6); }; } // namespace parameters diff --git a/DataFormats/Parameters/include/DataFormatsParameters/GRPLHCIFData.h b/DataFormats/Parameters/include/DataFormatsParameters/GRPLHCIFData.h index 7ac6cd6b00133..ddcda8f66507c 100644 --- a/DataFormats/Parameters/include/DataFormatsParameters/GRPLHCIFData.h +++ b/DataFormats/Parameters/include/DataFormatsParameters/GRPLHCIFData.h @@ -41,6 +41,7 @@ class GRPLHCIFData std::pair getBeamEnergyPerZWithTime() const { return mBeamEnergyPerZ; } int32_t getBeamEnergyPerZ() const { return mBeamEnergyPerZ.second; } + float getBeamEnergyPerZinGeV() const { return mBeamEnergyPerZ.second * 0.12; } long getBeamEnergyPerZTime() const { return mBeamEnergyPerZ.first; } void setBeamEnergyPerZWithTime(std::pair p) { mBeamEnergyPerZ = p; } void setBeamEnergyPerZWithTime(long t, int32_t v) { mBeamEnergyPerZ = std::make_pair(t, v); } @@ -84,12 +85,18 @@ class GRPLHCIFData /// getters/setters for given beam A and Z info, encoded as A<<16+Z int getBeamZ(beamDirection beam) const { return mBeamAZ[static_cast(beam)] & 0xffff; } int getBeamA(beamDirection beam) const { return mBeamAZ[static_cast(beam)] >> 16; } + int getBeamZ(int beam) const { return mBeamAZ[beam] & 0xffff; } + int getBeamA(int beam) const { return mBeamAZ[beam] >> 16; } float getBeamZoverA(beamDirection beam) const; + float getBeamZoverA(int beam) const; void setBeamAZ(int a, int z, beamDirection beam) { mBeamAZ[static_cast(beam)] = (a << 16) + z; } void setBeamAZ(beamDirection beam); void setBeamAZ(); /// getters/setters for beam energy per charge and per nucleon float getBeamEnergyPerNucleon(beamDirection beam) const { return mBeamEnergyPerZ.second * getBeamZoverA(beam); } + float getBeamEnergyPerNucleon(int beam) const { return mBeamEnergyPerZ.second * getBeamZoverA(beam); } + float getBeamEnergyPerNucleonInGeV(beamDirection beam) const { return getBeamEnergyPerZinGeV() * getBeamZoverA(beam); } + float getBeamEnergyPerNucleonInGeV(int beam) const { return getBeamEnergyPerZinGeV() * getBeamZoverA(beam); } /// calculate center of mass energy per nucleon collision float getSqrtS() const; /// helper function for BunchFilling @@ -119,6 +126,14 @@ inline float GRPLHCIFData::getBeamZoverA(beamDirection b) const return a ? getBeamZ(b) / static_cast(a) : 0.f; } +//______________________________________________ +inline float GRPLHCIFData::getBeamZoverA(int b) const +{ + // Z/A of beam 0 or 1 + int a = getBeamA(b); + return a ? getBeamZ(b) / static_cast(a) : 0.f; +} + } // namespace parameters } // namespace o2 #endif diff --git a/DataFormats/Parameters/include/DataFormatsParameters/GRPMagField.h b/DataFormats/Parameters/include/DataFormatsParameters/GRPMagField.h index d35126f2787bc..f9749c2b2a36f 100644 --- a/DataFormats/Parameters/include/DataFormatsParameters/GRPMagField.h +++ b/DataFormats/Parameters/include/DataFormatsParameters/GRPMagField.h @@ -34,9 +34,20 @@ class GRPMagField ~GRPMagField() = default; /// getters/setters for magnets currents - o2::units::Current_t getL3Current() const { return mL3Current; } - o2::units::Current_t getDipoleCurrent() const { return mDipoleCurrent; } + o2::units::Current_t getL3Current() const + { + static float v = checkL3Override(); + return v == NOOVERRIDEVAL ? mL3Current : v; + } + + o2::units::Current_t getDipoleCurrent() const + { + static float v = checkDipoleOverride(); + return v == NOOVERRIDEVAL ? mDipoleCurrent : v; + } + bool getFieldUniformity() const { return mUniformField; } + int8_t getNominalL3Field(); void setL3Current(o2::units::Current_t v) { mL3Current = v; } void setDipoleCurrent(o2::units::Current_t v) { mDipoleCurrent = v; } void setFieldUniformity(bool v) { mUniformField = v; } @@ -50,10 +61,27 @@ class GRPMagField o2::units::Current_t mL3Current = 0.f; ///< signed current in L3 o2::units::Current_t mDipoleCurrent = 0.f; ///< signed current in Dipole bool mUniformField = false; ///< uniformity of magnetic field + int8_t mNominalL3Field = 0; //!< Nominal L3 field deduced from mL3Current + bool mNominalL3FieldValid = false; //!< Has the field been computed (for caching) + + static constexpr float NOOVERRIDEVAL = 1e99; + static float checkL3Override(); + static float checkDipoleOverride(); - ClassDefNV(GRPMagField, 1); + ClassDefNV(GRPMagField, 2); }; +inline int8_t GRPMagField::getNominalL3Field() +{ + // compute nominal L3 field in kG + + if (mNominalL3FieldValid == false) { + mNominalL3Field = std::lround(5.f * getL3Current() / 30000.f); + mNominalL3FieldValid = true; + } + return mNominalL3Field; +} + } // namespace parameters } // namespace o2 diff --git a/DataFormats/Parameters/src/AggregatedRunInfo.cxx b/DataFormats/Parameters/src/AggregatedRunInfo.cxx new file mode 100644 index 0000000000000..40402a33af68b --- /dev/null +++ b/DataFormats/Parameters/src/AggregatedRunInfo.cxx @@ -0,0 +1,151 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file AggregatedRunInfo.cxx +/// \author sandro.wenzel@cern.ch + +#include "DataFormatsParameters/AggregatedRunInfo.h" +#include "CCDB/BasicCCDBManager.h" +#include "DataFormatsParameters/GRPECSObject.h" +#include "DataFormatsParameters/GRPLHCIFData.h" +#include "CommonConstants/LHCConstants.h" +#include "Framework/Logger.h" +#include + +using namespace o2::parameters; + +o2::parameters::AggregatedRunInfo AggregatedRunInfo::buildAggregatedRunInfo_DATA(o2::ccdb::CCDBManagerInstance& ccdb, int runnumber) +{ + // TODO: could think about caching results per runnumber to + // avoid going to CCDB multiple times ---> but should be done inside the CCDBManagerInstance + + // we calculate the first orbit of a run based on sor (start-of-run) and eor + // we obtain these by calling getRunDuration + auto [sor, eor] = ccdb.getRunDuration(runnumber); + + // determine a good timestamp to query OrbitReset for this run + // --> the middle of the run is very appropriate and safer than just sor + auto run_mid_timestamp = sor + (eor - sor) / 2; + + // query the time of the orbit reset (when orbit is defined to be 0) + auto ctpx = ccdb.getForTimeStamp>("CTP/Calib/OrbitReset", run_mid_timestamp); + int64_t tsOrbitReset = (*ctpx)[0]; // us + + // get timeframe length from GRPECS + std::map metadata; + metadata["runNumber"] = Form("%d", runnumber); + auto grpecs = ccdb.getSpecific("GLO/Config/GRPECS", run_mid_timestamp, metadata); + auto grplhcif = ccdb.getSpecific("GLO/Config/GRPLHCIF", run_mid_timestamp); // no run metadata here + bool oldFatalState = ccdb.getFatalWhenNull(); + ccdb.setFatalWhenNull(false); + auto ctp_first_run_orbit = ccdb.getForTimeStamp>("CTP/Calib/FirstRunOrbit", run_mid_timestamp); + ccdb.setFatalWhenNull(oldFatalState); + return buildAggregatedRunInfo(runnumber, sor, eor, tsOrbitReset, grpecs, ctp_first_run_orbit, grplhcif); +} + +o2::parameters::AggregatedRunInfo AggregatedRunInfo::buildAggregatedRunInfo(int runnumber, long sorMS, long eorMS, long orbitResetMUS, const o2::parameters::GRPECSObject* grpecs, const std::vector* ctfFirstRunOrbitVec, const o2::parameters::GRPLHCIFData* grplhcif) +{ + auto nOrbitsPerTF = grpecs->getNHBFPerTF(); + // calculate SOR/EOR orbits + int64_t orbitSOR = -1; + if (ctfFirstRunOrbitVec && ctfFirstRunOrbitVec->size() >= 3) { // if we have CTP first run orbit available, we should use it + int64_t creation_timeIGNORED = (*ctfFirstRunOrbitVec)[0]; // do not use CTP start of run time! + int64_t ctp_run_number = (*ctfFirstRunOrbitVec)[1]; + int64_t ctp_orbitSOR = (*ctfFirstRunOrbitVec)[2]; + if (creation_timeIGNORED == -1 && ctp_run_number == -1 && ctp_orbitSOR == -1) { + LOGP(warn, "Default dummy CTP/Calib/FirstRunOrbit was provides, ignoring"); + } else if (ctp_run_number == runnumber) { + orbitSOR = ctp_orbitSOR; + auto sor_new = (int64_t)((orbitResetMUS + ctp_orbitSOR * o2::constants::lhc::LHCOrbitMUS) / 1000.); + if (sor_new != sorMS) { + LOGP(warn, "Adjusting SOR from {} to {}", sorMS, sor_new); + sorMS = sor_new; + } + } else { + LOGP(error, "AggregatedRunInfo: run number inconsistency found (asked: {} vs CTP found: {}, ignoring", runnumber, ctp_run_number); + } + } + int64_t orbitEOR = (eorMS * 1000 - orbitResetMUS) / o2::constants::lhc::LHCOrbitMUS; + if (runnumber > 523897) { // condition was introduced starting from LHC22o + orbitEOR = orbitEOR / nOrbitsPerTF * nOrbitsPerTF; + } + if (orbitSOR < 0) { // extract from SOR + orbitSOR = (sorMS * 1000 - orbitResetMUS) / o2::constants::lhc::LHCOrbitMUS; + if (runnumber > 523897) { + orbitSOR = (orbitSOR / nOrbitsPerTF + 1) * nOrbitsPerTF; + } + } + return AggregatedRunInfo{runnumber, sorMS, eorMS, nOrbitsPerTF, orbitResetMUS, orbitSOR, orbitEOR, grpecs, grplhcif}; +} + +namespace +{ + +// get path where to find MC production info +std::string getFullPath_MC(std::string const& username, std::string const& lpm_prod_tag) +{ + // construct the path where to lookup + std::string path = "/Users/" + std::string(1, username[0]) + "/" + username; + std::string fullpath = path + "/" + "MCProdInfo/" + lpm_prod_tag; + return fullpath; +} + +} // namespace + +std::map AggregatedRunInfo::getMCProdInfo(o2::ccdb::CCDBManagerInstance& ccdb, + int run_number, + std::string const& lpm_prod_tag, + std::string const& username) +{ + std::map metaDataFilter; + metaDataFilter["LPMProductionTag"] = lpm_prod_tag; + + // fetch the meta information for MC productions + auto header_data = ccdb.getCCDBAccessor().retrieveHeaders(getFullPath_MC(username, lpm_prod_tag), metaDataFilter, run_number); + return header_data; +} + +void AggregatedRunInfo::adjust_from_MC(o2::ccdb::CCDBManagerInstance& ccdb, + int run_number, + std::string const& lpm_prod_tag, + std::string const& username) +{ + auto header_data = AggregatedRunInfo::getMCProdInfo(ccdb, run_number, lpm_prod_tag, username); + + // adjust timeframe length if we find entry for MC production + auto iter = header_data.find("OrbitsPerTF"); + if (iter != header_data.end()) { + auto mc_orbitsPerTF = std::stoi(iter->second); + if (mc_orbitsPerTF != orbitsPerTF) { + LOG(info) << "Adjusting OrbitsPerTF from " << orbitsPerTF << " to " << mc_orbitsPerTF << " based on differing MC info"; + orbitsPerTF = mc_orbitsPerTF; + } + } else { + LOG(warn) << "No OrbitsPerTF information found for MC production " << lpm_prod_tag << " and run number " << run_number; + } +} + +AggregatedRunInfo AggregatedRunInfo::buildAggregatedRunInfo(o2::ccdb::CCDBManagerInstance& ccdb, int run_number, std::string const& lpm_prod_tag, std::string const& username) +{ + // (a) lookup the AggregatedRunInfo for the data run + // (b) modify/overwrite the info object with MC specific settings if lpm_prod_tag is given + + auto original_info = buildAggregatedRunInfo_DATA(ccdb, run_number); + + if (lpm_prod_tag.size() == 0) { + return original_info; + } + + // in this case we adjust the info from MC + original_info.adjust_from_MC(ccdb, run_number, lpm_prod_tag, username); + + return original_info; +} diff --git a/DataFormats/Parameters/src/GRPECSObject.cxx b/DataFormats/Parameters/src/GRPECSObject.cxx index 6fed3fde5032e..52af87195cfa7 100644 --- a/DataFormats/Parameters/src/GRPECSObject.cxx +++ b/DataFormats/Parameters/src/GRPECSObject.cxx @@ -39,8 +39,10 @@ void GRPECSObject::print() const } return {" N / A "}; }; - printf("Run %d of type %d, period %s, isMC: %d\n", mRun, int(mRunType), mDataPeriod.c_str(), isMC()); - printf("Start: %s | End: %s\n", timeStr(mTimeStart).c_str(), timeStr(mTimeEnd).c_str()); + std::string rtypName = int(mRunType) < GRPECS::RunType::NRUNTYPES ? GRPECS::RunTypeNames[int(mRunType)].data() : "INVALID"; + printf("Run %d of type %s, period %s, isMC: %d\n", mRun, rtypName.c_str(), mDataPeriod.c_str(), isMC()); + printf("ECS/CTP Start: %s/%s | End: %s/%s\n", timeStr(mTimeStart).c_str(), timeStr(mTimeStartCTP).c_str(), + timeStr(mTimeEnd).c_str(), timeStr(mTimeEndCTP).c_str()); printf("Number of HBF per timframe: %d\n", mNHBFPerTF); printf("Detectors: Cont.RO Triggers\n"); for (auto i = DetID::First; i <= DetID::Last; i++) { @@ -52,6 +54,11 @@ void GRPECSObject::print() const printf("%7s ", isDetTriggers(DetID(i)) ? " + " : " - "); printf("\n"); } + printf("List of FLPs used\n"); + for (auto flp : mFLPs) { + printf("%3hu ", flp); + } + printf("\n"); } //_______________________________________________ diff --git a/DataFormats/Parameters/src/GRPLHCIFData.cxx b/DataFormats/Parameters/src/GRPLHCIFData.cxx index dcbca0932cf69..d39569f79376b 100644 --- a/DataFormats/Parameters/src/GRPLHCIFData.cxx +++ b/DataFormats/Parameters/src/GRPLHCIFData.cxx @@ -28,6 +28,8 @@ using namespace o2::constants::lhc; const std::unordered_map GRPLHCIFData::mZtoA = { {1, 1}, + {8, 16}, + {10, 20}, {82, 208}}; //_______________________________________________ @@ -60,8 +62,8 @@ void GRPLHCIFData::setBeamAZ() float GRPLHCIFData::getSqrtS() const { // get center of mass energy - double e0 = getBeamEnergyPerNucleon(BeamC); - double e1 = getBeamEnergyPerNucleon(BeamA); + double e0 = getBeamEnergyPerNucleonInGeV(BeamC); + double e1 = getBeamEnergyPerNucleonInGeV(BeamA); if (e0 <= MassProton || e1 <= MassProton) { return 0.f; } diff --git a/DataFormats/Parameters/src/GRPMagField.cxx b/DataFormats/Parameters/src/GRPMagField.cxx index e3dd412f0d118..8f5ae0b414998 100644 --- a/DataFormats/Parameters/src/GRPMagField.cxx +++ b/DataFormats/Parameters/src/GRPMagField.cxx @@ -41,3 +41,25 @@ void GRPMagField::print() const { printf("magnet currents (A) L3 = %.3f, Dipole = %.f; uniformity = %s\n", getL3Current(), getDipoleCurrent(), mUniformField ? "true" : "false"); } + +o2::units::Current_t GRPMagField::checkDipoleOverride() +{ + static float v = getenv("O2_OVERRIDE_DIPOLE_CURRENT") ? atof(getenv("O2_OVERRIDE_DIPOLE_CURRENT")) : NOOVERRIDEVAL; + static bool alarmShown = false; + if (v != NOOVERRIDEVAL && !alarmShown) { + LOGP(error, "Overriding DIPOLE current to {}", v); + alarmShown = true; + } + return v; +} + +o2::units::Current_t GRPMagField::checkL3Override() +{ + static float v = getenv("O2_OVERRIDE_L3_CURRENT") ? atof(getenv("O2_OVERRIDE_L3_CURRENT")) : NOOVERRIDEVAL; + static bool alarmShown = false; + if (v != NOOVERRIDEVAL && !alarmShown) { + LOGP(error, "Overriding L3 current to {}", v); + alarmShown = true; + } + return v; +} diff --git a/DataFormats/Parameters/src/GRPObject.cxx b/DataFormats/Parameters/src/GRPObject.cxx index 79ec4afc0bb93..0c8f583f6d76a 100644 --- a/DataFormats/Parameters/src/GRPObject.cxx +++ b/DataFormats/Parameters/src/GRPObject.cxx @@ -13,7 +13,7 @@ /// \brief Implementation of General Run Parameters object /// \author ruben.shahoyan@cern.ch -#include +#include #include #include "DataFormatsParameters/GRPObject.h" #include diff --git a/DataFormats/Parameters/src/GRPTool.cxx b/DataFormats/Parameters/src/GRPTool.cxx index d0acfffa4fe15..e7561e6fc1ef6 100644 --- a/DataFormats/Parameters/src/GRPTool.cxx +++ b/DataFormats/Parameters/src/GRPTool.cxx @@ -15,7 +15,7 @@ #include "DataFormatsParameters/GRPMagField.h" #include "DataFormatsParameters/GRPLHCIFData.h" #include "CommonUtils/FileSystemUtils.h" -#include +#include #include #include #include @@ -23,6 +23,9 @@ #include #include #include +#include "CommonUtils/ConfigurableParam.h" +#include "DataFormatsCalibration/MeanVertexObject.h" +#include "SimConfig/InteractionDiamondParam.h" // // Created by Sandro Wenzel on 20.06.22. @@ -54,11 +57,44 @@ struct Options { std::string outprefix = ""; std::string fieldstring = ""; std::string bcPatternFile = ""; - bool print = false; // whether to print outcome of GRP operation + bool print = false; // whether to print outcome of GRP operation + bool lhciffromccdb = false; // whether only to take GRPLHCIF from CCDB std::string publishto = ""; std::string ccdbhost = "http://alice-ccdb.cern.ch"; + bool isRun5 = false; // whether or not this is supposed to be a Run5 detector configuration + std::string vertex = "ccdb"; + std::string configKeyValues = ""; + uint64_t timestamp = 0; + std::string detectorList; // detector layout }; +namespace +{ +class CCDBHelper +{ + public: + CCDBHelper(Options const& opts) + { + api.init(opts.ccdbhost); + auto soreor = o2::ccdb::BasicCCDBManager::getRunDuration(api, opts.run); + runStart = soreor.first; + runEnd = soreor.second; + if (opts.timestamp > 0) { + timestamp = opts.timestamp; + } else { + timestamp = runStart; + } + } + o2::ccdb::CcdbApi api; + uint64_t runStart = -1; + uint64_t runEnd = -1; + uint64_t timestamp = -1; +}; +} // namespace + +// a simple reusable CCDB wrapper; caching some info across functions +std::unique_ptr gCCDBWrapper; + void print_globalHelp(int argc, char* argv[]) { std::cout << "** A GRP utility **\n\n"; @@ -170,7 +206,7 @@ bool publish(std::string const& filename, std::string const& path, std::string C try { o2::utils::createDirectoriesIfAbsent(targetdir); } catch (std::exception e) { - LOGP(error, fmt::format("Could not create local snapshot cache directory {}, reason: {}", targetdir, e.what())); + LOGP(error, "Could not create local snapshot cache directory {}, reason: {}", targetdir, e.what()); return false; } @@ -186,13 +222,11 @@ bool publish(std::string const& filename, std::string const& path, std::string C // download a set of basic GRP files based on run number/time bool anchor_GRPs(Options const& opts, std::vector const& paths = {"GLO/Config/GRPECS", "GLO/Config/GRPMagField", "GLO/Config/GRPLHCIF"}) { - auto& ccdbmgr = o2::ccdb::BasicCCDBManager::instance(); - auto soreor = ccdbmgr.getRunDuration(opts.run); + if (!gCCDBWrapper) { + gCCDBWrapper = std::move(std::make_unique(opts)); + } // fix the timestamp early - uint64_t runStart = soreor.first; - - o2::ccdb::CcdbApi api; - api.init(opts.ccdbhost); + uint64_t timestamp = gCCDBWrapper->timestamp; const bool preserve_path = true; const std::string filename("snapshot.root"); @@ -200,11 +234,61 @@ bool anchor_GRPs(Options const& opts, std::vector const& paths = {" bool success = true; for (auto& p : paths) { LOG(info) << "Fetching " << p << " from CCDB"; - success &= api.retrieveBlob(p, opts.publishto, filter, runStart, preserve_path, filename); + success &= gCCDBWrapper->api.retrieveBlob(p, opts.publishto, filter, timestamp, preserve_path, filename); } return success; } +// creates a mean vertex object for further use (CCDB queries) in the CCDB cache +bool create_MeanVertexObject(Options const& opts) +{ + // either + const char* CCDBPATH = "/GLO/Calib/MeanVertex"; + if (opts.vertex == "ccdb") { + anchor_GRPs(opts, {CCDBPATH}); + } else { + LOG(info) << "Creating MeanVertex object on the fly using the InteractionDiamond params"; + auto const& param = o2::eventgen::InteractionDiamondParam::Instance(); + const auto& xyz = param.position; + const auto& sigma = param.width; + std::unique_ptr meanv(new o2::dataformats::MeanVertexObject(xyz[0], xyz[1], xyz[2], sigma[0], sigma[1], sigma[2], param.slopeX, param.slopeY)); + o2::ccdb::CcdbApi api; + api.init("file://" + opts.publishto); + std::map meta; + meta["Created-by"] = "Monte Carlo GRPTool"; + if (!gCCDBWrapper) { + gCCDBWrapper = std::move(std::make_unique(opts)); + } + api.storeAsTFileAny(meanv.get(), CCDBPATH, meta, gCCDBWrapper->runStart, gCCDBWrapper->runEnd); + + // we created a file not called snapshot.root ---> still need to do this ... so that this object get's picked up later on + // we pick up the latest produced file and will rename it to snapshot.root (thanks ChatGPT) + std::filesystem::path directory_path = opts.publishto + CCDBPATH; + // Timepoint to hold the latest modification time + std::filesystem::file_time_type latest_time = std::filesystem::file_time_type::min(); + // Path to hold the latest modified file + std::filesystem::path latest_file; + // Iterate over all files in the directory + for (const auto& file : std::filesystem::directory_iterator(directory_path)) { + // Check if the file is a regular file + if (file.is_regular_file()) { + // Get the last modification time of the file + std::filesystem::file_time_type mod_time = std::filesystem::last_write_time(file.path()); + // Check if the modification time is later than the latest time found so far + if (mod_time > latest_time) { + latest_time = mod_time; + latest_file = file.path(); + } + } + } + auto oldpath = latest_file; + auto newpath = latest_file.parent_path(); + newpath.append(std::string("snapshot.root")); + std::filesystem::rename(oldpath, newpath); + } + return true; +} + // creates a set of basic GRP files (for simulation) bool create_GRPs(Options const& opts) { @@ -212,6 +296,12 @@ bool create_GRPs(Options const& opts) uint64_t runStart = -1; // used in multiple GRPs + // MeanVertexObject + { + LOG(info) << "---- creating MeanVertex ----"; + create_MeanVertexObject(opts); + } + // GRPECS { LOG(info) << " --- creating GRP ECS -----"; @@ -222,10 +312,12 @@ bool create_GRPs(Options const& opts) auto soreor = ccdbmgr.getRunDuration(opts.run); runStart = soreor.first; grp.setTimeStart(runStart); - grp.setTimeEnd(runStart + 3600000); + grp.setTimeEnd(soreor.second); grp.setNHBFPerTF(opts.orbitsPerTF); std::vector modules{}; - o2::conf::SimConfig::determineActiveModules(opts.readout, std::vector(), modules); + if (!o2::conf::SimConfig::determineActiveModulesList(opts.detectorList, opts.readout, std::vector(), modules)) { + return false; + } std::vector readout{}; o2::conf::SimConfig::determineReadoutDetectors(modules, std::vector(), opts.skipreadout, readout); for (auto& detstr : readout) { @@ -267,7 +359,13 @@ bool create_GRPs(Options const& opts) if (fieldmode == o2::conf::SimFieldMode::kCCDB) { // we download the object from CCDB LOG(info) << "Downloading mag field directly from CCDB"; - anchor_GRPs(opts, {"GLO/Config/GRPMagField"}); + if (!anchor_GRPs(opts, {"GLO/Config/GRPMagField"})) { + LOG(fatal) << "Downloading mag field failed"; + } + if (opts.print) { + // print the object that was downloaded + printGRPMAG(std::string(opts.publishto) + std::string("/GLO/Config/GRPMagField/snapshot.root")); + } } else { // let's not create an actual mag field object for this // we only need to lookup the currents from the possible @@ -307,41 +405,51 @@ bool create_GRPs(Options const& opts) // GRPLHCIF --> complete it later { LOG(info) << " --- creating GRP LHCIF -----"; - o2::parameters::GRPLHCIFData grp; - // eventually we need to set the beam info from the generator, at the moment put some plausible values - grp.setFillNumberWithTime(runStart, 0); // RS FIXME - grp.setInjectionSchemeWithTime(runStart, ""); // RS FIXME - grp.setBeamEnergyPerZWithTime(runStart, 6.8e3); // RS FIXME - grp.setAtomicNumberB1WithTime(runStart, 1.); // RS FIXME - grp.setAtomicNumberB2WithTime(runStart, 1.); // RS FIXME - grp.setCrossingAngleWithTime(runStart, 0.); // RS FIXME - grp.setBeamAZ(); - - // set the BC pattern if necessary - if (opts.bcPatternFile.size() > 0) { - // load bunch filling from the file - auto* bc = o2::BunchFilling::loadFrom(opts.bcPatternFile); - if (!bc) { - LOG(fatal) << "Failed to load bunch filling from " << opts.bcPatternFile; - } - grp.setBunchFillingWithTime(grp.getBeamEnergyPerZTime(), *bc); // borrow the time from the existing entry - delete bc; + if (opts.lhciffromccdb) { // if we take the whole object it directly from CCDB, we can just download it + LOG(info) << "Downloading complete GRPLHCIF object directly from CCDB"; + anchor_GRPs(opts, {"GLO/Config/GRPLHCIF"}); } else { - // we initialize with a default bunch filling scheme; - LOG(info) << "Initializing with default bunch filling"; - o2::BunchFilling bc; - bc.setDefault(); - grp.setBunchFillingWithTime(grp.getBeamEnergyPerZTime(), bc); - } - std::string grpfilename = o2::base::NameConf::getGRPLHCIFFileName(opts.outprefix); - if (opts.print) { - grp.print(); - } - TFile grpF(grpfilename.c_str(), "recreate"); - grpF.WriteObjectAny(&grp, grp.Class(), o2::base::NameConf::CCDBOBJECT.data()); - grpF.Close(); - if (opts.publishto.size() > 0) { - publish(grpfilename, opts.publishto, "/GLO/Config/GRPLHCIF"); + o2::parameters::GRPLHCIFData grp; + // eventually we need to set the beam info from the generator, at the moment put some plausible values + grp.setFillNumberWithTime(runStart, 0); // RS FIXME + grp.setInjectionSchemeWithTime(runStart, ""); // RS FIXME + grp.setBeamEnergyPerZWithTime(runStart, 6.8e3); // RS FIXME + grp.setAtomicNumberB1WithTime(runStart, 1.); // RS FIXME + grp.setAtomicNumberB2WithTime(runStart, 1.); // RS FIXME + grp.setCrossingAngleWithTime(runStart, 0.); // RS FIXME + grp.setBeamAZ(); + + // set the BC pattern if necessary + if (opts.bcPatternFile.size() > 0) { + // load bunch filling from the file (with standard CCDB convention) + auto* bc = o2::BunchFilling::loadFrom(opts.bcPatternFile, "ccdb_object"); + if (!bc) { + // if it failed, retry with default naming + bc = o2::BunchFilling::loadFrom(opts.bcPatternFile); + } + if (!bc) { + LOG(fatal) << "Failed to load bunch filling from " << opts.bcPatternFile; + } + grp.setBunchFillingWithTime(grp.getBeamEnergyPerZTime(), *bc); // borrow the time from the existing entry + delete bc; + } else { + // we initialize with a default bunch filling scheme; + LOG(info) << "Initializing with default bunch filling"; + o2::BunchFilling bc; + bc.setDefault(); + grp.setBunchFillingWithTime(grp.getBeamEnergyPerZTime(), bc); + } + + std::string grpfilename = o2::base::NameConf::getGRPLHCIFFileName(opts.outprefix); + if (opts.print) { + grp.print(); + } + TFile grpF(grpfilename.c_str(), "recreate"); + grpF.WriteObjectAny(&grp, grp.Class(), o2::base::NameConf::CCDBOBJECT.data()); + grpF.Close(); + if (opts.publishto.size() > 0) { + publish(grpfilename, opts.publishto, "/GLO/Config/GRPLHCIF"); + } } } @@ -455,6 +563,7 @@ bool parseOptions(int argc, char* argv[], Options& optvalues) // ls command has the following options: bpo::options_description desc("create options"); + desc.add_options()("detectorList", bpo::value(&optvalues.detectorList)->default_value("ALICE2"), "Pick a specific version of ALICE, for specifics check the o2-sim description"); desc.add_options()("readoutDets", bpo::value>(&optvalues.readout)->multitoken()->default_value(std::vector({"all"}), "all Run3 detectors"), "Detector list to be readout/active"); desc.add_options()("skipReadout", bpo::value>(&optvalues.skipreadout)->multitoken()->default_value(std::vector(), "nothing skipped"), "list of inactive detectors (precendence over --readout)"); desc.add_options()("run", bpo::value(&optvalues.run)->default_value(-1), "Run number"); @@ -462,14 +571,29 @@ bool parseOptions(int argc, char* argv[], Options& optvalues) desc.add_options()("field", bpo::value(&optvalues.fieldstring)->default_value("-5"), "L3 field rounded to kGauss, allowed values +-2,+-5 and 0; +-U for uniform field"); desc.add_options()("outprefix,o", bpo::value(&optvalues.outprefix)->default_value("o2sim"), "Prefix for GRP output files"); desc.add_options()("bcPatternFile", bpo::value(&optvalues.bcPatternFile)->default_value(""), "Interacting BC pattern file (e.g. from CreateBCPattern.C)"); + desc.add_options()("lhcif-CCDB", "take GRPLHCIF directly from CCDB"); desc.add_options()("print", "print resulting GRPs"); desc.add_options()("publishto", bpo::value(&optvalues.publishto)->default_value(""), "Base path under which GRP objects should be published on disc. This path can serve as lookup for CCDB queries of the GRP objects."); + desc.add_options()("isRun5", bpo::bool_switch(&optvalues.isRun5), "Whether or not to expect a Run5 detector configuration. (deprecated, use detectorList option)"); + desc.add_options()("vertex", bpo::value(&optvalues.vertex)->default_value("ccdb"), "How the vertex is to be initialized. Default is CCDB. Alternative is \"Diamond\" which is constructing the mean vertex from the Diamond param via the configKeyValues path"); + desc.add_options()("timestamp", bpo::value(&optvalues.timestamp)->default_value(0), "Force timestamp to be used (useful when anchoring)"); + desc.add_options()("configKeyValues", bpo::value(&optvalues.configKeyValues)->default_value(""), "Semicolon separated key=value strings (e.g.: 'TPC.gasDensity=1;...')"); if (!subparse(desc, vm, "createGRPs")) { return false; } if (vm.count("print") > 0) { optvalues.print = true; } + if (vm.count("lhcif-CCDB") > 0) { + optvalues.lhciffromccdb = true; + } + auto vertexmode = vm["vertex"].as(); + if (!(vertexmode == "ccdb" || vertexmode == "Diamond")) { + return false; + } + // init params + o2::conf::ConfigurableParam::updateFromString(optvalues.configKeyValues); + } else if (cmd == "setROMode") { // set/modify the ROMode optvalues.command = GRPCommand::kSETROMODE; diff --git a/DataFormats/Parameters/src/ParametersDataLinkDef.h b/DataFormats/Parameters/src/ParametersDataLinkDef.h index e4817bd443302..d2cbee2049c52 100644 --- a/DataFormats/Parameters/src/ParametersDataLinkDef.h +++ b/DataFormats/Parameters/src/ParametersDataLinkDef.h @@ -30,5 +30,6 @@ #pragma link C++ class o2::parameters::GRPMagField + ; #pragma link C++ class std::unordered_map < unsigned int, unsigned int> + ; #pragma link C++ class std::pair < unsigned long, std::string> + ; +#pragma link C++ struct o2::parameters::AggregatedRunInfo + ; #endif diff --git a/DataFormats/QualityControl/CMakeLists.txt b/DataFormats/QualityControl/CMakeLists.txt index b9190d7fda324..c0f40a88f88c1 100644 --- a/DataFormats/QualityControl/CMakeLists.txt +++ b/DataFormats/QualityControl/CMakeLists.txt @@ -10,65 +10,65 @@ # or submit itself to any jurisdiction. file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/include) -### Prepare the list of methods in FlagReasonFactory -file(READ "etc/flagReasons.csv" CSV_FLAG_REASONS) +### Prepare the list of methods in FlagTypeFactory +file(READ "etc/flagTypes.csv" CSV_FLAG_TYPES) # delete the CSV file header -string(REPLACE \"id\",\"method\",\"name\",\"bad\",\"obsolete\" "" CSV_FLAG_REASONS ${CSV_FLAG_REASONS}) +string(REPLACE \"id\",\"method\",\"name\",\"bad\",\"obsolete\" "" CSV_FLAG_TYPES ${CSV_FLAG_TYPES}) # detects if there is obsolete flag '1' in the last column, adds [[deprecated]] if so and retains the rest of the string string(REGEX REPLACE \([0-9]+,\".[^\"]*.\",.[^\"]*.,[0-1]\),1 "[[deprecated]] \\1,1" - CSV_FLAG_REASONS - ${CSV_FLAG_REASONS}) + CSV_FLAG_TYPES + ${CSV_FLAG_TYPES}) # replaces the flag reason entry with a c++ method to create it string(REGEX REPLACE \([0-9]+\),\"\(.[^\"]*.\)\",\(.[^\"]*.\),\([0-1]\),[0-1] - "static FlagReason \\2\(\) { return { static_cast\(\\1\), \\3, static_cast\(\\4\) }; }" - CSV_FLAG_REASONS - ${CSV_FLAG_REASONS}) + "static FlagType \\2\(\) { return { static_cast\(\\1\), \\3, static_cast\(\\4\) }; }" + CSV_FLAG_TYPES + ${CSV_FLAG_TYPES}) # put the method lists inside the template -configure_file("include/DataFormatsQualityControl/FlagReasonFactory.h.in" - "${CMAKE_CURRENT_BINARY_DIR}/include/DataFormatsQualityControl/FlagReasonFactory.h" +configure_file("include/DataFormatsQualityControl/FlagTypeFactory.h.in" + "${CMAKE_CURRENT_BINARY_DIR}/include/DataFormatsQualityControl/FlagTypeFactory.h" @ONLY) -install(FILES "${CMAKE_CURRENT_BINARY_DIR}/include/DataFormatsQualityControl/FlagReasonFactory.h" +install(FILES "${CMAKE_CURRENT_BINARY_DIR}/include/DataFormatsQualityControl/FlagTypeFactory.h" DESTINATION include/DataFormatsQualityControl) o2_add_library(DataFormatsQualityControl - SOURCES src/FlagReasons.cxx - src/TimeRangeFlag.cxx - src/TimeRangeFlagCollection.cxx - PUBLIC_LINK_LIBRARIES O2::Headers - O2::FrameworkLogger + SOURCES src/FlagType.cxx + src/QualityControlFlag.cxx + src/QualityControlFlagCollection.cxx + PUBLIC_LINK_LIBRARIES O2::FrameworkLogger O2::DetectorsCommonDataFormats PUBLIC_INCLUDE_DIRECTORIES ${CMAKE_CURRENT_BINARY_DIR}/include include ) o2_target_root_dictionary(DataFormatsQualityControl - HEADERS include/DataFormatsQualityControl/FlagReasons.h - include/DataFormatsQualityControl/TimeRangeFlag.h - include/DataFormatsQualityControl/TimeRangeFlagCollection.h) + HEADERS include/DataFormatsQualityControl/FlagType.h + include/DataFormatsQualityControl/QualityControlFlag.h + include/DataFormatsQualityControl/QualityControlFlagCollection.h) - -o2_add_test(FlagReasons - SOURCES test/testFlagReasons.cxx +if(BUILD_TESTING) +o2_add_test(FlagTypes + SOURCES test/testFlagTypes.cxx COMPONENT_NAME DataFormatsQualityControl PUBLIC_LINK_LIBRARIES O2::DataFormatsQualityControl - TARGETVARNAME flagreasons) + TARGETVARNAME flagtypes) -target_include_directories(${flagreasons} PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/include) +target_include_directories(${flagtypes} PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/include) -o2_add_test(TimeRangeFlag - SOURCES test/testTimeRangeFlag.cxx +o2_add_test(QualityControlFlag + SOURCES test/testQualityControlFlag.cxx COMPONENT_NAME DataFormatsQualityControl PUBLIC_LINK_LIBRARIES O2::DataFormatsQualityControl - TARGETVARNAME timerangeflag) + TARGETVARNAME qualitycontrolflag) -target_include_directories(${timerangeflag} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/include) +target_include_directories(${qualitycontrolflag} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/include) -o2_add_test(TimeRangeFlagCollection - SOURCES test/testTimeRangeFlagCollection.cxx +o2_add_test(QualityControlFlagCollection + SOURCES test/testQualityControlFlagCollection.cxx COMPONENT_NAME DataFormatsQualityControl PUBLIC_LINK_LIBRARIES O2::DataFormatsQualityControl - TARGETVARNAME timerangeflagcollection) + TARGETVARNAME qualitycontrolflagcollection) -target_include_directories(${timerangeflagcollection} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/include) +target_include_directories(${qualitycontrolflagcollection} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/include) +endif() diff --git a/DataFormats/QualityControl/README.md b/DataFormats/QualityControl/README.md index ededa028c7b93..33821319b7316 100644 --- a/DataFormats/QualityControl/README.md +++ b/DataFormats/QualityControl/README.md @@ -1,63 +1,100 @@ \page refDataFormatsQualityControl Data Formats Quality Control -Data formats for tagging good quality data for analysis. +## Tagging Good Quality Data for Processing and Analysis -# Flagging time ranges -## General idea -* Each detector has its own CCDB (QCDB) entry - TimeRangeFlagCollection -* The CCDB Entry validity will be run or fill, depending on the final data taking granularity -* Each entry can define sub ranges (TimeRangeFlags) of certain data characteristics (FlagReasons) inside the CCDB Entry validity -* Flags are defined in a CSV master file, they are used to derive the Data Tags - the final filters for good quality data during analysis. -* Data Tag are stored as CCDB entries. They might require different detectors and may suppress different flags dependent on the analysis type. - -## Implementation - -[Flag Reason](include/DataFormatsQualityControl/FlagReasons.h) is defined with an identifier number, a name and a 'bad quality' determinant. -The latter decides if such a flag should mark the data quality as bad by default. -FlagReasons can be created only with FlagReasonFactory, so that the list of available reasons is common and centralised. -For example: -``` -id: 10 -name: Limited Acceptance -bad: true -``` -The list of available FlagReasons is defined in [etc/flagReasons.csv](etc/flagReasons.csv), which is used to generate the corresponding list of methods in FlagReasonFactory. -The existing flags should never be modified, except for marking them as obsolete. -New flags may be added via pull requests if there is no flag which conveys a similar meaning. +This document outlines the data formats used for tagging good quality data for further processing and analysis. +They allow us to describe problems affecting the data in concrete time intervals. +Using this information, data can be filtered out according to criteria specific to a given analysis type. + +### Data Quality Control workflow + +Data quality is determined through two methods: + +1. **Automated Checks:** The Quality Control framework runs Checks which may return quality Flags. +2. **Manual Review:** Detector experts review data using the Run Condition Table (RCT) and can modify or add Flags. + +Both methods utilize the same data format for Flags. +During processing (both synchronous and asynchronous), Checks produce Qualities and associate them with Flags. +The Quality Control framework then transmits these Flags to the RCT through a gRPC interface (more details in QC repository documentation). +Detector experts can then review the automatically generated Flags and make any necessary modifications or additions directly in the RCT. + +### Quality Control Flag Structure + +A [Quality Control Flag](include/DataFormatsQualityControl/QualityControlFlag.h) consists of the following elements: + +* **Flag Type:** This identifies the specific issue the flag represents. (More details below) +* **Time Range:** This defines the time period affected by the flag. +* **Comment (Optional):** This allows human-readable explanations for assigning the flag. +* **Source String:** This identifies the entity (person or software module) that created the flag. + +**Example:** -With [TimeRangeFlags](include/DataFormatsQualityControl/TimeRangeFlag.h) we can define the time range of a chosen FlagReason, add an additional comment and specify the source of this flag. -For example: ``` -start: 1612707603626 +flag: Limited Acceptance MC Reproducible +start: 1612707603626 end: 1613999652000 -flag: Limited Acceptance comment: Sector B in TPC inactive -source: o2::quality_control_modules::tpc::ClustersCheck +source: TPC/Clusters Check ``` -[TimeRangeFlagCollection](include/DataFormatsQualityControl/TimeRangeFlagCollection.h) contains all TimeRangeFlags for the validity range (run or fill). -TimeRangeFlags may overlap, e.g. if they use different FlagReasons and they are sorted by their start time. -If certain period does not include any TimeRangeFlags with *bad* FlagReasons, then the data quality can be considered as good. -The [TimeRangeFlagCollection test](test/testTimeRangeFlagCollection.cxx) shows the usage example. +### Flag Types + +[Flag Types](include/DataFormatsQualityControl/FlagType.h) define the specific categories of issues represented by flags. +Each Flag Type has the following attributes: + +* **Identifier Number:** A unique numerical ID for the Flag Type. +* **Name:** A human-readable name describing the issue. +* **"Bad Quality" Determinant:** This boolean constant indicates whether the type inherently signifies bad data quality. + +#### Creating and Managing Flag Types + +* **FlagTypeFactory** ensures a centralized and consistent list of available Flag Types. + New Flags can only be created through this factory. +* **[flagTypes.csv](etc/flagTypes.csv)** defines the existing Flag Types, including their ID, name, and "bad quality" determinant, factory method name and a switch to deprecate a flag. + The table serves as the source to automatically generate the corresponding methods in FlagTypeFactory. +* **Adding new Flag Types:** If a new issue requires a flag not currently defined, propose the addition by contacting the async QC coordinators. + They have the authority to add new Flag Types to the RCT. + These changes will then be reflected in the [flagTypes.csv](etc/flagTypes.csv) file through a pull request. + Any proposals for new Flag Types should describe the effects on usability of data from analyzer point of view and they should not be detector-specific unless well-argumented. +* **Modification of existing Flag Types:** Existing Flag Types should not be modified in terms of their definition. + Instead, one may create a new Flag Type and mark the existing one as obsolete in the CSV table. + This will add the `[[ deprecated ]]` attribute to the corresponding method. -TimeRangeFlagCollections are supposed to be created automatically with QC Post-processing Tasks based on Quality Objects created by QC Checks. -However, they might be created manually by QA experts as well. -The procedure to do that has to be defined. +#### Currently available Flag Types -## TODO -* Define the complete list of available Flag Reasons -* implement CCDB storage and access - - define CCDB storage place e.g. - * `/QC/QualityFlags` - * `Analysis/QualityFlags/` -* Data Tags Definitions and Data Tags +This section details the currently available Flag Types and provides a brief explanation of their intended use cases. -### Notes on plans for Data Tags +* **Good:** a Check or an expert sees nothing wrong with given time interval, but would like to add a comment. + Note that the absence of any flag for a run implies good data quality. + Thus, there is no need to mark it explicitly as such by using this flag type. +* **No Detector Data:** a complete and unexpected absence of data for a specific detector. +* **Limited Acceptance MC Not Reproducible:** a part of a detector did not acquire good data and this condition cannot be reproduced in Monte Carlo. + If an automated Check cannot determine MC reproducibility, it should default to "Not Reproducible" for later expert review. +* **Limited Acceptance MC Reproducible:** a part of a detector did not acquire good data, but this condition can be reproduced in Monte Carlo. +* **Bad Tracking:** analyses relying on accurate track reconstruction should not use this data. +* **Bad PID:** analyses relying on correct identification of all kinds of tracked particles should not use this data. +* **Bad Hadron PID:** analyses relying on correct hadron identification should not use this data. +* **Bad Electron PID:** analyses relying on correct electron identification should not use this data. +* **Bad Photon Calorimetry:** analyses relying on correct photon calorimetry should not use this data. +* **Bad EMCalorimetry:** analyses relying on correct electromagnetic calorimetry should not use this data. +* **Unknown:** the exact impact of an issue on the data is unclear, but it's likely bad. + Treat data with this flag with caution until further investigation. +* **Unknown Quality:** the quality of data could not be determined definitively. +* **Invalid:** there was an issue with processing the flags. + +## Usage in Analysis framework (plans, wishlist) + +## General idea +* RCT exports a read-only copy of the flags for each detector, run and pass combination in the CCDB. +* The Analysis framework uses the flags and user selection criteria to provide their Analysis Task with data matching these criteria. + **Data Tags** are the structures defining the good time intervals for the provided criteria. + +## Notes on plans for Data Tags Data Tag definition has: * name -* global suppression list for bad flag reasons (applies to all detectors) -* global requirement list for not bad flag reasons +* global suppression list for bad flag types (applies to all detectors) +* global requirement list for not bad flag types * list of needed detectors, * list of suppression list and requirement list specific to detectors @@ -94,10 +131,3 @@ Example configurations } ] ``` - -## Wishlist / Ideas -* executable to add flags to the flag store -* executable to extrags the flag store -* summary of all masks for one TimeRangeFlagCollection -* functionality to extract flags for a specific detector (from CCDB) -* cut class to specify detectors and flags which to exclude \ No newline at end of file diff --git a/DataFormats/QualityControl/etc/flagReasons.csv b/DataFormats/QualityControl/etc/flagReasons.csv deleted file mode 100644 index 23acc1b3000fa..0000000000000 --- a/DataFormats/QualityControl/etc/flagReasons.csv +++ /dev/null @@ -1,15 +0,0 @@ -"id","method","name","bad","obsolete" -1,"Unknown","Unknown",1,0 -2,"UnknownQuality","Unknown Quality",1,0 -3,"CertifiedByExpert","Certified by Expert",0,0 -10,"NoDetectorData","No Detector Data",1,0 -11,"LimitedAcceptance","Limited acceptance",1,0 -12,"BadPID","Bad PID",1,0 -13,"BadTracking","Bad Tracking",1,0 -14,"BadHadronPID","Bad Hadron PID",1,0 -15,"BadElectronPID","Bad Electron PID",1,0 -16,"BadEMCalorimetry","Bad EM Calorimetry",1,0 -17,"BadPhotonCalorimetry","Bad Photon Calorimetry",1,0 -65500,"ObsoleteFlagExample","Obsolete flag example",1,1 -65501,"NotBadFlagExample","Not bad flag example",0,0 -65535,"Invalid","Invalid",1,0 diff --git a/DataFormats/QualityControl/etc/flagTypes.csv b/DataFormats/QualityControl/etc/flagTypes.csv new file mode 100644 index 0000000000000..97107cd15a9e7 --- /dev/null +++ b/DataFormats/QualityControl/etc/flagTypes.csv @@ -0,0 +1,14 @@ +"id","method","name","bad","obsolete" +1,"UnknownQuality","Unknown Quality",1,0 +3,"NoDetectorData","No Detector Data",1,0 +4,"LimitedAcceptanceMCNotReproducible","Limited Acceptance MC Not Reproducible",1,0 +5,"LimitedAcceptanceMCReproducible","Limited Acceptance MC Reproducible",1,0 +6,"BadPID","Bad PID",1,0 +7,"BadTracking","Bad Tracking",1,0 +8,"BadHadronPID","Bad Hadron PID",1,0 +9,"Good","Good",0,0 +10,"Invalid","Invalid",1,0 +11,"BadElectronPID","Bad Electron PID",1,0 +12,"BadEMCalorimetry","Bad EM Calorimetry",1,0 +13,"BadPhotonCalorimetry","Bad Photon Calorimetry",1,0 +14,"Unknown","Unknown",1,0 diff --git a/DataFormats/QualityControl/include/DataFormatsQualityControl/FlagReasonFactory.h.in b/DataFormats/QualityControl/include/DataFormatsQualityControl/FlagReasonFactory.h.in deleted file mode 100644 index 59bd7b4c5628f..0000000000000 --- a/DataFormats/QualityControl/include/DataFormatsQualityControl/FlagReasonFactory.h.in +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// \file FlagReasonFactory.h -/// \brief A class to create FlagReasons based on the predefined CSV list. -/// \author Piotr Konopka, piotr.jan.konopka@cern.ch - -#ifndef O2_FLAGREASONFACTORY_H -#define O2_FLAGREASONFACTORY_H - -#include "DataFormatsQualityControl/FlagReasons.h" - -namespace o2::quality_control -{ - -class FlagReasonFactory { -public: -FlagReasonFactory() = delete; -@CSV_FLAG_REASONS@ -}; - -} // namespace o2::quality_control -#endif // O2_FLAGREASONFACTORY_H \ No newline at end of file diff --git a/DataFormats/QualityControl/include/DataFormatsQualityControl/FlagReasons.h b/DataFormats/QualityControl/include/DataFormatsQualityControl/FlagReasons.h deleted file mode 100644 index 67f5b4cc27cde..0000000000000 --- a/DataFormats/QualityControl/include/DataFormatsQualityControl/FlagReasons.h +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -#ifndef ALICEO2_ANALYSIS_FLAGREASONS -#define ALICEO2_ANALYSIS_FLAGREASONS - -/// \file FlagReasons.h -/// \brief classes keeping reasons for flagging time ranges -/// \author Jens Wiechula, jens.wiechula@ikf.uni-frankfurt.de -/// \author Piotr Konopka, piotr.jan.konopka@cern.ch - -// STL -#include -#include - -// ROOT includes -#include - -namespace o2 -{ -namespace quality_control -{ - -class FlagReasonFactory; -class TimeRangeFlagCollection; - -class FlagReason -{ - private: - uint16_t mId; - std::string mName; - bool mBad; // if true, data should become bad by default - - // By making the constructor private and FlagReasons available only in the FlagReasonFactory - // we forbid to declare any flags in the user code. If you need a new FlagReason, please add it FlagReasonFactory. - private: - FlagReason(uint16_t id, const char* name, bool bad) : mId(id), mName(name), mBad(bad) {} - - public: - FlagReason(); - FlagReason& operator=(const FlagReason&) = default; - FlagReason(const FlagReason&) = default; - bool operator==(const FlagReason& rhs) const; - bool operator!=(const FlagReason& rhs) const; - bool operator<(const FlagReason& rhs) const; - bool operator>(const FlagReason& rhs) const; - - uint16_t getID() const { return mId; } - const std::string& getName() const { return mName; } - bool getBad() const { return mBad; } - - friend std::ostream& operator<<(std::ostream& os, FlagReason const& me); - friend class FlagReasonFactory; - friend class TimeRangeFlagCollection; - - ClassDefNV(FlagReason, 1); -}; - -} // namespace quality_control -} // namespace o2 - -// TODO: remove once we include it in QualityControl -#include "DataFormatsQualityControl/FlagReasonFactory.h" - -#endif diff --git a/DataFormats/QualityControl/include/DataFormatsQualityControl/FlagType.h b/DataFormats/QualityControl/include/DataFormatsQualityControl/FlagType.h new file mode 100644 index 0000000000000..94e4a301a1784 --- /dev/null +++ b/DataFormats/QualityControl/include/DataFormatsQualityControl/FlagType.h @@ -0,0 +1,64 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef O2_QUALITYCONTROL_FLAGTYPE_H +#define O2_QUALITYCONTROL_FLAGTYPE_H + +// STL +#include +#include + +// ROOT includes +#include + +namespace o2 +{ +namespace quality_control +{ + +class FlagTypeFactory; +class QualityControlFlagCollection; + +class FlagType +{ + private: + uint16_t mId; + std::string mName; + bool mBad; // if true, data should become bad by default + + // By making the constructor private and FlagTypes available only in the FlagTypeFactory + // we forbid to declare any flags in the user code. If you need a new FlagType, please add it FlagTypeFactory. + private: + FlagType(uint16_t id, const char* name, bool bad) : mId(id), mName(name), mBad(bad) {} + + public: + FlagType(); + FlagType& operator=(const FlagType&) = default; + FlagType(const FlagType&) = default; + bool operator==(const FlagType& rhs) const; + bool operator!=(const FlagType& rhs) const; + bool operator<(const FlagType& rhs) const; + bool operator>(const FlagType& rhs) const; + + uint16_t getID() const { return mId; } + const std::string& getName() const { return mName; } + bool getBad() const { return mBad; } + + friend std::ostream& operator<<(std::ostream& os, FlagType const& me); + friend class FlagTypeFactory; + friend class QualityControlFlagCollection; + + ClassDefNV(FlagType, 1); +}; + +} // namespace quality_control +} // namespace o2 +#endif // O2_QUALITYCONTROL_FLAGTYPE_H diff --git a/DataFormats/QualityControl/include/DataFormatsQualityControl/FlagTypeFactory.h.in b/DataFormats/QualityControl/include/DataFormatsQualityControl/FlagTypeFactory.h.in new file mode 100644 index 0000000000000..38b6c463d239f --- /dev/null +++ b/DataFormats/QualityControl/include/DataFormatsQualityControl/FlagTypeFactory.h.in @@ -0,0 +1,31 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file FlagTypeFactory.h +/// \brief A class to create FlagTypes based on the predefined CSV list. +/// \author Piotr Konopka, piotr.jan.konopka@cern.ch + +#ifndef O2_FLAGTYPEFACTORY_H +#define O2_FLAGTYPEFACTORY_H + +#include "DataFormatsQualityControl/FlagType.h" + +namespace o2::quality_control +{ + +class FlagTypeFactory { +public: +FlagTypeFactory() = delete; +@CSV_FLAG_TYPES@ +}; + +} // namespace o2::quality_control +#endif // O2_FLAGTYPEFACTORY_H \ No newline at end of file diff --git a/DataFormats/QualityControl/include/DataFormatsQualityControl/QualityControlFlag.h b/DataFormats/QualityControl/include/DataFormatsQualityControl/QualityControlFlag.h new file mode 100644 index 0000000000000..9c38952562483 --- /dev/null +++ b/DataFormats/QualityControl/include/DataFormatsQualityControl/QualityControlFlag.h @@ -0,0 +1,88 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef O2_QUALITYCONTROL_QCFLAG_H +#define O2_QUALITYCONTROL_QCFLAG_H + +/// \file QualityControlFlag.h +/// \brief Class to define a flag type with a time range and comments +/// \author Jens Wiechula, jens.wiechula@ikf.uni-frankfurt.de +/// \author Piotr Konopka, piotr.jan.konopka@cern.ch + +// System includes +#include +#include + +// ROOT includes +#include + +#include + +#include "DataFormatsQualityControl/FlagType.h" +#include "DataFormatsQualityControl/FlagTypeFactory.h" + +namespace o2 +{ +namespace quality_control +{ + +/// \class QualityControlFlag +/// A Class for associating a bit mask with a time range +class QualityControlFlag +{ + public: + using time_type = uint64_t; + using RangeInterval = o2::math_utils::detail::Bracket; + + QualityControlFlag() = default; + QualityControlFlag(QualityControlFlag const&) = default; + QualityControlFlag(time_type start, time_type end, FlagType flag, std::string comment = "", std::string source = "Unknown"); + + time_type getStart() const { return mInterval.getMin(); } + time_type getEnd() const { return mInterval.getMax(); } + RangeInterval& getInterval() { return mInterval; } + const RangeInterval getInterval() const { return mInterval; } + FlagType getFlag() const { return mFlag; } + const std::string& getComment() const { return mComment; } + const std::string& getSource() const { return mSource; } + + void setStart(time_type start) { mInterval.setMin(start); } + void setEnd(time_type end) { mInterval.setMax(end); } + void setInterval(RangeInterval interval) { mInterval = interval; } + void setFlag(FlagType flag) { mFlag = flag; } + void setComment(const std::string& comment) { mComment = comment; } + void setSource(const std::string& source) { mSource = source; } + + /// equal operator + bool operator==(const QualityControlFlag& rhs) const; + + /// comparison operators + bool operator<(const QualityControlFlag& rhs) const; + bool operator>(const QualityControlFlag& rhs) const; + + /// write data to ostream + void streamTo(std::ostream& output) const; + + /// overloading output stream operator + friend std::ostream& operator<<(std::ostream& output, const QualityControlFlag& data); + + private: + RangeInterval mInterval = {}; ///< time interval of the masked range + FlagType mFlag; ///< flag reason + std::string mComment = ""; ///< optional comment, which may extend the reason + std::string mSource = "Unknown"; ///< optional (but encouraged) source of the flag (e.g. Qc Check name) + + ClassDefNV(QualityControlFlag, 1); +}; + +} // namespace quality_control +} // namespace o2 +#endif // O2_QUALITYCONTROL_QCFLAG_H diff --git a/DataFormats/QualityControl/include/DataFormatsQualityControl/QualityControlFlagCollection.h b/DataFormats/QualityControl/include/DataFormatsQualityControl/QualityControlFlagCollection.h new file mode 100644 index 0000000000000..120605f048cbe --- /dev/null +++ b/DataFormats/QualityControl/include/DataFormatsQualityControl/QualityControlFlagCollection.h @@ -0,0 +1,102 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef O2_QUALITYCONTROL_QCFLAGCOLLECTION_H +#define O2_QUALITYCONTROL_QCFLAGCOLLECTION_H +/// \file QualityControlFlagCollection.h +/// \brief classes for defining time ranges with a certain mask to be able to cut on +/// \author Jens Wiechula, jens.wiechula@ikf.uni-frankfurt.de + +// System includes +#include + +// ROOT includes +#include "Rtypes.h" + +// O2 includes +#include "MathUtils/detail/Bracket.h" +#include "DataFormatsQualityControl/QualityControlFlag.h" + +// STL +#include + +namespace o2 +{ +namespace quality_control +{ + +/// \class QualityControlFlagCollection +/// A Class for keeping several time ranges of type QualityControlFlag +class QualityControlFlagCollection +{ + public: + using collection_t = std::set; + using time_type = uint64_t; + using RangeInterval = o2::math_utils::detail::Bracket; + + explicit QualityControlFlagCollection(std::string name, std::string detector = "TST", RangeInterval validityRange = {}, + int runNumber = 0, std::string periodName = "Invalid", std::string passName = "Invalid", + std::string provenance = "qc"); + + void insert(QualityControlFlag&&); + void insert(const QualityControlFlag&); + + size_t size() const; + + // moves all non-repeating QualityControlFlags from other to this + void merge(QualityControlFlagCollection& other); + // add all non-repeating QualityControlFlags from other to this. + void merge(const QualityControlFlagCollection& other); + + collection_t::const_iterator begin() const; + collection_t::const_iterator end() const; + + const std::string& getName() const; + const std::string& getDetector() const; + int getRunNumber() const; + const std::string& getPeriodName() const; + const std::string& getPassName() const; + const std::string& getProvenance() const; + + time_type getStart() const { return mValidityRange.getMin(); } + time_type getEnd() const { return mValidityRange.getMax(); } + RangeInterval& getInterval() { return mValidityRange; } + + void setStart(time_type start) { mValidityRange.setMin(start); } + void setEnd(time_type end) { mValidityRange.setMax(end); } + void setInterval(RangeInterval interval) { mValidityRange = interval; } + + /// write data to ostream + void streamTo(std::ostream& output) const; + /// Read data from instream + void streamFrom(std::istream& input); + + /// overloading output stream operator + friend std::ostream& operator<<(std::ostream& output, const QualityControlFlagCollection& data); + + private: + std::string mDetID; // three letter detector code + std::string mName; // some description of the collection, e.g. "Raw data checks", "QA Expert masks" + // with std::set we can sort the flags in time and have merge() for granted. + collection_t mQualityControlFlags; + RangeInterval mValidityRange; // we need a validity range to e.g. state that there are no TRFs for given time interval + int mRunNumber; + std::string mPeriodName; + std::string mPassName; + std::string mProvenance; + + ClassDefNV(QualityControlFlagCollection, 1); +}; + +} // namespace quality_control +} // namespace o2 + +#endif // O2_QUALITYCONTROL_QCFLAGCOLLECTION_H diff --git a/DataFormats/QualityControl/include/DataFormatsQualityControl/TimeRangeFlag.h b/DataFormats/QualityControl/include/DataFormatsQualityControl/TimeRangeFlag.h deleted file mode 100644 index b8e1efbce1f40..0000000000000 --- a/DataFormats/QualityControl/include/DataFormatsQualityControl/TimeRangeFlag.h +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -#ifndef ALICEO2_ANALYSIS_TIMERANGEFLAGS -#define ALICEO2_ANALYSIS_TIMERANGEFLAGS - -/// \file TimeRangeFlag.h -/// \brief Class to define a time range of a flag type -/// \author Jens Wiechula, jens.wiechula@ikf.uni-frankfurt.de -/// \author Piotr Konopka, piotr.jan.konopka@cern.ch - -// System includes -#include -#include - -// ROOT includes -#include - -#include - -#include "DataFormatsQualityControl/FlagReasons.h" -#include "DataFormatsQualityControl/FlagReasonFactory.h" - -namespace o2 -{ -namespace quality_control -{ - -/// \class TimeRangeFlag -/// A Class for associating a bit mask with a time range -class TimeRangeFlag -{ - public: - using time_type = uint64_t; - using flag_type = FlagReason; - using RangeInterval = o2::math_utils::detail::Bracket; - - TimeRangeFlag() = default; - TimeRangeFlag(TimeRangeFlag const&) = default; - TimeRangeFlag(time_type start, time_type end, flag_type flag, std::string comment = "", std::string source = "Unknown"); - - time_type getStart() const { return mInterval.getMin(); } - time_type getEnd() const { return mInterval.getMax(); } - RangeInterval& getInterval() { return mInterval; } - flag_type getFlag() const { return mFlag; } - const std::string& getComment() const { return mComment; } - const std::string& getSource() const { return mSource; } - - void setStart(time_type start) { mInterval.setMin(start); } - void setEnd(time_type end) { mInterval.setMax(end); } - void setInterval(RangeInterval interval) { mInterval = interval; } - void setFlag(flag_type flag) { mFlag = flag; } - void setComment(const std::string& comment) { mComment = comment; } - void setSource(const std::string& source) { mSource = source; } - - /// equal operator - bool operator==(const TimeRangeFlag& rhs) const; - - /// comparison operators - bool operator<(const TimeRangeFlag& rhs) const; - bool operator>(const TimeRangeFlag& rhs) const; - - /// write data to ostream - void streamTo(std::ostream& output) const; - - /// overloading output stream operator - friend std::ostream& operator<<(std::ostream& output, const TimeRangeFlag& data); - - private: - RangeInterval mInterval = {}; ///< time interval of the masked range - flag_type mFlag; ///< flag reason - std::string mComment = ""; ///< optional comment, which may extend the reason - std::string mSource = "Unknown"; ///< optional (but encouraged) source of the flag (e.g. Qc Check name) - - ClassDefNV(TimeRangeFlag, 1); -}; - -} // namespace quality_control -} // namespace o2 -#endif diff --git a/DataFormats/QualityControl/include/DataFormatsQualityControl/TimeRangeFlagCollection.h b/DataFormats/QualityControl/include/DataFormatsQualityControl/TimeRangeFlagCollection.h deleted file mode 100644 index 30a8230fc66e2..0000000000000 --- a/DataFormats/QualityControl/include/DataFormatsQualityControl/TimeRangeFlagCollection.h +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -#ifndef ALICEO2_ANALYSIS_TIMERANGEMASK -#define ALICEO2_ANALYSIS_TIMERANGEMASK - -/// \file TimeRangeFlagCollection.h -/// \brief classes for defining time ranges with a certain mask to be able to cut on -/// \author Jens Wiechula, jens.wiechula@ikf.uni-frankfurt.de - -// System includes -#include - -// ROOT includes -#include "Rtypes.h" - -// O2 includes -#include "MathUtils/detail/Bracket.h" -#include "DataFormatsQualityControl/TimeRangeFlag.h" - -// STL -#include - -namespace o2 -{ -namespace quality_control -{ - -/// \class TimeRangeFlagCollection -/// A Class for keeping several time ranges of type TimeRangeFlag -class TimeRangeFlagCollection -{ - public: - using collection_t = std::set; - using time_type = uint64_t; - using RangeInterval = o2::math_utils::detail::Bracket; - - explicit TimeRangeFlagCollection(std::string name, std::string detector = "TST", RangeInterval validityRange = {}, - int runNumber = 0, std::string periodName = "Invalid", std::string passName = "Invalid", - std::string provenance = "qc"); - - void insert(TimeRangeFlag&&); - void insert(const TimeRangeFlag&); - - size_t size() const; - - // moves all non-repeating TimeRangeFlags from other to this - void merge(TimeRangeFlagCollection& other); - // add all non-repeating TimeRangeFlags from other to this. - void merge(const TimeRangeFlagCollection& other); - - collection_t::const_iterator begin() const; - collection_t::const_iterator end() const; - - const std::string& getName() const; - const std::string& getDetector() const; - int getRunNumber() const; - const std::string& getPeriodName() const; - const std::string& getPassName() const; - const std::string& getProvenance() const; - - time_type getStart() const { return mValidityRange.getMin(); } - time_type getEnd() const { return mValidityRange.getMax(); } - RangeInterval& getInterval() { return mValidityRange; } - - void setStart(time_type start) { mValidityRange.setMin(start); } - void setEnd(time_type end) { mValidityRange.setMax(end); } - void setInterval(RangeInterval interval) { mValidityRange = interval; } - - /// write data to ostream - void streamTo(std::ostream& output) const; - /// Read data from instream - void streamFrom(std::istream& input); - - /// overloading output stream operator - friend std::ostream& operator<<(std::ostream& output, const TimeRangeFlagCollection& data); - - private: - std::string mDetID; // three letter detector code - std::string mName; // some description of the collection, e.g. "Raw data checks", "QA Expert masks" - // with std::set we can sort the flags in time and have merge() for granted. - collection_t mTimeRangeFlags; - RangeInterval mValidityRange; // we need a validity range to e.g. state that there are no TRFs for given time interval - int mRunNumber; - std::string mPeriodName; - std::string mPassName; - std::string mProvenance; - - ClassDefNV(TimeRangeFlagCollection, 1); -}; - -} // namespace quality_control -} // namespace o2 -#endif diff --git a/DataFormats/QualityControl/src/DataFormatsQualityControlLinkDef.h b/DataFormats/QualityControl/src/DataFormatsQualityControlLinkDef.h index 8d77bccef4b7c..ab10c361b29f9 100644 --- a/DataFormats/QualityControl/src/DataFormatsQualityControlLinkDef.h +++ b/DataFormats/QualityControl/src/DataFormatsQualityControlLinkDef.h @@ -15,8 +15,8 @@ #pragma link off all classes; #pragma link off all functions; -#pragma link C++ class o2::quality_control::FlagReason + ; -#pragma link C++ class o2::quality_control::TimeRangeFlag + ; -#pragma link C++ class o2::quality_control::TimeRangeFlagCollection + ; +#pragma link C++ class o2::quality_control::FlagType + ; +#pragma link C++ class o2::quality_control::QualityControlFlag + ; +#pragma link C++ class o2::quality_control::QualityControlFlagCollection + ; #endif diff --git a/DataFormats/QualityControl/src/FlagReasons.cxx b/DataFormats/QualityControl/src/FlagReasons.cxx deleted file mode 100644 index e85a0d0c7faa7..0000000000000 --- a/DataFormats/QualityControl/src/FlagReasons.cxx +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -#include "DataFormatsQualityControl/FlagReasons.h" -#include "DataFormatsQualityControl/FlagReasonFactory.h" - -#include -#include - -namespace o2::quality_control -{ - -FlagReason::FlagReason() -{ - *this = FlagReasonFactory::Invalid(); -} - -std::ostream& operator<<(std::ostream& os, FlagReason const& my) -{ - os << "Flag Reason: id - " << my.mId << ", name - " << my.mName << ", bad - " << (my.mBad ? "true" : "false"); - return os; -} -bool FlagReason::operator==(const FlagReason& rhs) const -{ - return std::tie(mId, mName, mBad) == std::tie(rhs.mId, rhs.mName, rhs.mBad); -} -bool FlagReason::operator!=(const FlagReason& rhs) const -{ - return std::tie(mId, mName, mBad) != std::tie(rhs.mId, rhs.mName, rhs.mBad); -} -bool FlagReason::operator<(const FlagReason& rhs) const -{ - return std::tie(mId, mName, mBad) < std::tie(rhs.mId, rhs.mName, rhs.mBad); -} -bool FlagReason::operator>(const FlagReason& rhs) const -{ - return std::tie(mId, mName, mBad) > std::tie(rhs.mId, rhs.mName, rhs.mBad); -} - -} // namespace o2::quality_control diff --git a/DataFormats/QualityControl/src/FlagType.cxx b/DataFormats/QualityControl/src/FlagType.cxx new file mode 100644 index 0000000000000..27e59911ed775 --- /dev/null +++ b/DataFormats/QualityControl/src/FlagType.cxx @@ -0,0 +1,48 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include "DataFormatsQualityControl/FlagType.h" +#include "DataFormatsQualityControl/FlagTypeFactory.h" + +#include +#include + +namespace o2::quality_control +{ + +FlagType::FlagType() +{ + *this = FlagTypeFactory::Invalid(); +} + +std::ostream& operator<<(std::ostream& os, FlagType const& my) +{ + os << "Flag Reason: id - " << my.mId << ", name - " << my.mName << ", bad - " << (my.mBad ? "true" : "false"); + return os; +} +bool FlagType::operator==(const FlagType& rhs) const +{ + return std::tie(mId, mName, mBad) == std::tie(rhs.mId, rhs.mName, rhs.mBad); +} +bool FlagType::operator!=(const FlagType& rhs) const +{ + return std::tie(mId, mName, mBad) != std::tie(rhs.mId, rhs.mName, rhs.mBad); +} +bool FlagType::operator<(const FlagType& rhs) const +{ + return std::tie(mId, mName, mBad) < std::tie(rhs.mId, rhs.mName, rhs.mBad); +} +bool FlagType::operator>(const FlagType& rhs) const +{ + return std::tie(mId, mName, mBad) > std::tie(rhs.mId, rhs.mName, rhs.mBad); +} + +} // namespace o2::quality_control diff --git a/DataFormats/QualityControl/src/QualityControlFlag.cxx b/DataFormats/QualityControl/src/QualityControlFlag.cxx new file mode 100644 index 0000000000000..ce749cad5315f --- /dev/null +++ b/DataFormats/QualityControl/src/QualityControlFlag.cxx @@ -0,0 +1,64 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include "DataFormatsQualityControl/QualityControlFlag.h" + +#include +#include + +namespace o2::quality_control +{ + +QualityControlFlag::QualityControlFlag(time_type start, time_type end, FlagType flag, std::string comment, std::string source) + : mInterval(start, end), mFlag(flag), mComment(comment), mSource(source) +{ + if (mInterval.isInvalid()) { + throw std::runtime_error("QualityControlFlag start time '" + std::to_string(mInterval.getMin()) + "' is larger than end time '" + std::to_string(mInterval.getMax()) + "'"); + } +} + +bool QualityControlFlag::operator==(const QualityControlFlag& rhs) const +{ + return std::tie(mInterval, mFlag, mComment, mSource) == std::tie(rhs.mInterval, rhs.mFlag, rhs.mComment, rhs.mSource); +} + +bool QualityControlFlag::operator<(const QualityControlFlag& rhs) const +{ + // We don't use the comparison mechanism in Bracket, + // because std::set which is used in TRFCollection assumes that a < b, a > b <=> a == b. + // Using relation operators in Bracket would break insertion and merging. + return std::tie(static_cast(mInterval.getMin()), static_cast(mInterval.getMax()), mFlag, mComment, mSource) < std::tie(static_cast(rhs.mInterval.getMin()), static_cast(rhs.mInterval.getMax()), rhs.mFlag, rhs.mComment, rhs.mSource); +} + +bool QualityControlFlag::operator>(const QualityControlFlag& rhs) const +{ + // we don't use the comparison mechanism in Bracket, + // because std::set which is used in TRFCollection assumes that a < b, a > b <=> a == b + return std::tie(static_cast(mInterval.getMin()), static_cast(mInterval.getMax()), mFlag, mComment, mSource) > std::tie(static_cast(rhs.mInterval.getMin()), static_cast(rhs.mInterval.getMax()), rhs.mFlag, rhs.mComment, rhs.mSource); +} + +void QualityControlFlag::streamTo(std::ostream& output) const +{ + output << "QualityControlFlag:\n"; + output << "- Start: " << mInterval.getMin() << "\n"; + output << "- End: " << mInterval.getMax() << "\n"; + output << "- " << mFlag << "\n"; + output << "- Comment: " << mComment << "\n"; + output << "- Source: " << mSource; +} + +std::ostream& operator<<(std::ostream& output, const QualityControlFlag& data) +{ + data.streamTo(output); + return output; +} + +} // namespace o2::quality_control diff --git a/DataFormats/QualityControl/src/QualityControlFlagCollection.cxx b/DataFormats/QualityControl/src/QualityControlFlagCollection.cxx new file mode 100644 index 0000000000000..042ef2f218d09 --- /dev/null +++ b/DataFormats/QualityControl/src/QualityControlFlagCollection.cxx @@ -0,0 +1,217 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +// O2 include +#include "DataFormatsQualityControl/QualityControlFlagCollection.h" +#include "DataFormatsQualityControl/FlagTypeFactory.h" +#include "Framework/Logger.h" + +#include +#include +#include +#include + +namespace o2::quality_control +{ + +constexpr const char* csvHeader = "start,end,flag_id,flag_name,flag_bad,comment,source"; +constexpr size_t csvColumns = 7; + +QualityControlFlagCollection::QualityControlFlagCollection(std::string name, std::string detector, RangeInterval validityRange, + int runNumber, std::string periodName, std::string passName, + std::string provenance) + : mName(std::move(name)), mDetID(std::move(detector)), mValidityRange(validityRange), mRunNumber(runNumber), mPeriodName(std::move(periodName)), mPassName(passName), mProvenance(std::move(provenance)) +{ +} + +void QualityControlFlagCollection::insert(QualityControlFlag&& trf) +{ + mQualityControlFlags.insert(std::move(trf)); +} + +void QualityControlFlagCollection::insert(const QualityControlFlag& trf) +{ + mQualityControlFlags.insert(trf); +} + +size_t QualityControlFlagCollection::size() const +{ + return mQualityControlFlags.size(); +} + +void QualityControlFlagCollection::merge(QualityControlFlagCollection& other) +{ + if (mDetID != other.mDetID) { + // We assume that one QualityControlFlagCollection should correspond to one detector at most. + // However, if this becomes annoying, we can reconsider it. + throw std::runtime_error( + "The detector ID of the target collection '" + mDetID + "' is different than the other '" + mDetID); + } + mQualityControlFlags.merge(other.mQualityControlFlags); +} + +void QualityControlFlagCollection::merge(const QualityControlFlagCollection& other) +{ + QualityControlFlagCollection otherCopy{other}; + merge(otherCopy); +} + +QualityControlFlagCollection::collection_t::const_iterator QualityControlFlagCollection::begin() const +{ + return mQualityControlFlags.begin(); +} + +QualityControlFlagCollection::collection_t::const_iterator QualityControlFlagCollection::end() const +{ + return mQualityControlFlags.end(); +} + +void QualityControlFlagCollection::streamTo(std::ostream& output) const +{ + auto escapeComma = [](const std::string& str) { + return boost::algorithm::replace_all_copy(str, ",", "\\,"); + }; + output << csvHeader << '\n'; + for (const auto& trf : *this) { + output << fmt::format("{},{},{},\"{}\",{:d},\"{}\",\"{}\"\n", + trf.getStart(), trf.getEnd(), + trf.getFlag().getID(), escapeComma(trf.getFlag().getName()), trf.getFlag().getBad(), + escapeComma(trf.getComment()), escapeComma(trf.getSource())); + } +} + +void QualityControlFlagCollection::streamFrom(std::istream& input) +{ + std::string line; + std::getline(input, line); + if (line != csvHeader) { + throw std::runtime_error( + "Unsupported TRFCollection format, the first line is \"" + line + "\" instead of \"" + csvHeader + "\""); + } + + while (std::getline(input, line)) { + boost::tokenizer> tok(line); + + QualityControlFlag::time_type start = 0; + QualityControlFlag::time_type end = 0; + FlagType flag = FlagTypeFactory::Invalid(); + std::string comment; + std::string source; + auto it = tok.begin(); + bool valid = true; + size_t pos = 0; + for (; it != tok.end() && valid; pos++, it++) { + switch (pos) { + case 0: { + if (it->empty()) { + LOG(error) << "Invalid line, empty start time of a flag, skipping..."; + valid = false; + } else { + start = static_cast(std::stoull(*it)); + } + break; + } + case 1: { + if (it->empty()) { + LOG(error) << "Invalid line, empty end time of a flag, skipping..."; + valid = false; + } else { + end = static_cast(std::stoull(*it)); + } + break; + } + case 2: { + if (it->empty()) { + LOG(error) << "Invalid line, empty flag id, skipping..."; + valid = false; + } else { + flag.mId = std::stoul(*it); + } + break; + } + case 3: { + if (it->empty()) { + LOG(error) << "Invalid line, empty flag name, skipping..."; + valid = false; + } else { + flag.mName = *it; + } + break; + } + case 4: { + if (it->empty()) { + LOG(error) << "Invalid line, empty flag 'bad' field, skipping..."; + valid = false; + } else { + flag.mBad = static_cast(std::stoul(*it)); + } + break; + } + case 5: { + comment = *it; + break; + } + case 6: { + source = *it; + break; + } + default: { + LOG(error) << "More columns (" << pos + 1 << ") than expected (" << csvColumns + << ") in this line, skipping..."; + valid = false; + break; + } + } + } + if (valid && pos < csvColumns) { + LOG(error) << "Less columns (" << pos << ") than expected (" << csvColumns << ") in this line, skipping..."; + } else if (valid) { + insert({start, end, flag, comment, source}); + } + } +} + +std::ostream& operator<<(std::ostream& output, const QualityControlFlagCollection& data) +{ + data.streamTo(output); + return output; +} + +const std::string& QualityControlFlagCollection::getName() const +{ + return mName; +} + +const std::string& QualityControlFlagCollection::getDetector() const +{ + return mDetID; +} + +int QualityControlFlagCollection::getRunNumber() const +{ + return mRunNumber; +} + +const std::string& QualityControlFlagCollection::getPeriodName() const +{ + return mPeriodName; +} + +const std::string& QualityControlFlagCollection::getPassName() const +{ + return mPassName; +} +const std::string& QualityControlFlagCollection::getProvenance() const +{ + return mProvenance; +} + +} // namespace o2::quality_control diff --git a/DataFormats/QualityControl/src/TimeRangeFlag.cxx b/DataFormats/QualityControl/src/TimeRangeFlag.cxx deleted file mode 100644 index f96d5bc9b7b72..0000000000000 --- a/DataFormats/QualityControl/src/TimeRangeFlag.cxx +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -#include "DataFormatsQualityControl/TimeRangeFlag.h" - -#include -#include - -namespace o2::quality_control -{ - -TimeRangeFlag::TimeRangeFlag(time_type start, time_type end, flag_type flag, std::string comment, std::string source) - : mInterval(start, end), mFlag(flag), mComment(comment), mSource(source) -{ - if (mInterval.isInvalid()) { - throw std::runtime_error("TimeRangeFlag start time '" + std::to_string(mInterval.getMin()) + "' is larger than end time '" + std::to_string(mInterval.getMax()) + "'"); - } -} - -bool TimeRangeFlag::operator==(const TimeRangeFlag& rhs) const -{ - return std::tie(mInterval, mFlag, mComment, mSource) == std::tie(rhs.mInterval, rhs.mFlag, rhs.mComment, rhs.mSource); -} - -bool TimeRangeFlag::operator<(const TimeRangeFlag& rhs) const -{ - // We don't use the comparison mechanism in Bracket, - // because std::set which is used in TRFCollection assumes that a < b, a > b <=> a == b. - // Using relation operators in Bracket would break insertion and merging. - return std::tie(static_cast(mInterval.getMin()), static_cast(mInterval.getMax()), mFlag, mComment, mSource) < std::tie(static_cast(rhs.mInterval.getMin()), static_cast(rhs.mInterval.getMax()), rhs.mFlag, rhs.mComment, rhs.mSource); -} - -bool TimeRangeFlag::operator>(const TimeRangeFlag& rhs) const -{ - // we don't use the comparison mechanism in Bracket, - // because std::set which is used in TRFCollection assumes that a < b, a > b <=> a == b - return std::tie(static_cast(mInterval.getMin()), static_cast(mInterval.getMax()), mFlag, mComment, mSource) > std::tie(static_cast(rhs.mInterval.getMin()), static_cast(rhs.mInterval.getMax()), rhs.mFlag, rhs.mComment, rhs.mSource); -} - -void TimeRangeFlag::streamTo(std::ostream& output) const -{ - output << "TimeRangeFlag:\n"; - output << "- Start: " << mInterval.getMin() << "\n"; - output << "- End: " << mInterval.getMax() << "\n"; - output << "- " << mFlag << "\n"; - output << "- Comment: " << mComment << "\n"; - output << "- Source: " << mSource; -} - -std::ostream& operator<<(std::ostream& output, const TimeRangeFlag& data) -{ - data.streamTo(output); - return output; -} - -} // namespace o2::quality_control diff --git a/DataFormats/QualityControl/src/TimeRangeFlagCollection.cxx b/DataFormats/QualityControl/src/TimeRangeFlagCollection.cxx deleted file mode 100644 index ee3adc8dc7f2f..0000000000000 --- a/DataFormats/QualityControl/src/TimeRangeFlagCollection.cxx +++ /dev/null @@ -1,217 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -// O2 include -#include "DataFormatsQualityControl/TimeRangeFlagCollection.h" -#include "DataFormatsQualityControl/FlagReasonFactory.h" -#include "Framework/Logger.h" - -#include -#include -#include -#include - -namespace o2::quality_control -{ - -constexpr const char* csvHeader = "start,end,flag_id,flag_name,flag_bad,comment,source"; -constexpr size_t csvColumns = 7; - -TimeRangeFlagCollection::TimeRangeFlagCollection(std::string name, std::string detector, RangeInterval validityRange, - int runNumber, std::string periodName, std::string passName, - std::string provenance) - : mName(std::move(name)), mDetID(std::move(detector)), mValidityRange(validityRange), mRunNumber(runNumber), mPeriodName(std::move(periodName)), mPassName(passName), mProvenance(std::move(provenance)) -{ -} - -void TimeRangeFlagCollection::insert(TimeRangeFlag&& trf) -{ - mTimeRangeFlags.insert(std::move(trf)); -} - -void TimeRangeFlagCollection::insert(const TimeRangeFlag& trf) -{ - mTimeRangeFlags.insert(trf); -} - -size_t TimeRangeFlagCollection::size() const -{ - return mTimeRangeFlags.size(); -} - -void TimeRangeFlagCollection::merge(TimeRangeFlagCollection& other) -{ - if (mDetID != other.mDetID) { - // We assume that one TimeRangeFlagCollection should correspond to one detector at most. - // However, if this becomes annoying, we can reconsider it. - throw std::runtime_error( - "The detector ID of the target collection '" + mDetID + "' is different than the other '" + mDetID); - } - mTimeRangeFlags.merge(other.mTimeRangeFlags); -} - -void TimeRangeFlagCollection::merge(const TimeRangeFlagCollection& other) -{ - TimeRangeFlagCollection otherCopy{other}; - merge(otherCopy); -} - -TimeRangeFlagCollection::collection_t::const_iterator TimeRangeFlagCollection::begin() const -{ - return mTimeRangeFlags.begin(); -} - -TimeRangeFlagCollection::collection_t::const_iterator TimeRangeFlagCollection::end() const -{ - return mTimeRangeFlags.end(); -} - -void TimeRangeFlagCollection::streamTo(std::ostream& output) const -{ - auto escapeComma = [](const std::string& str) { - return boost::algorithm::replace_all_copy(str, ",", "\\,"); - }; - output << csvHeader << '\n'; - for (const auto& trf : *this) { - output << fmt::format("{},{},{},\"{}\",{:d},\"{}\",\"{}\"\n", - trf.getStart(), trf.getEnd(), - trf.getFlag().getID(), escapeComma(trf.getFlag().getName()), trf.getFlag().getBad(), - escapeComma(trf.getComment()), escapeComma(trf.getSource())); - } -} - -void TimeRangeFlagCollection::streamFrom(std::istream& input) -{ - std::string line; - std::getline(input, line); - if (line != csvHeader) { - throw std::runtime_error( - "Unsupported TRFCollection format, the first line is \"" + line + "\" instead of \"" + csvHeader + "\""); - } - - while (std::getline(input, line)) { - boost::tokenizer> tok(line); - - TimeRangeFlag::time_type start = 0; - TimeRangeFlag::time_type end = 0; - FlagReason flag = FlagReasonFactory::Invalid(); - std::string comment; - std::string source; - auto it = tok.begin(); - bool valid = true; - size_t pos = 0; - for (; it != tok.end() && valid; pos++, it++) { - switch (pos) { - case 0: { - if (it->empty()) { - LOG(error) << "Invalid line, empty start time of a flag, skipping..."; - valid = false; - } else { - start = static_cast(std::stoull(*it)); - } - break; - } - case 1: { - if (it->empty()) { - LOG(error) << "Invalid line, empty end time of a flag, skipping..."; - valid = false; - } else { - end = static_cast(std::stoull(*it)); - } - break; - } - case 2: { - if (it->empty()) { - LOG(error) << "Invalid line, empty flag id, skipping..."; - valid = false; - } else { - flag.mId = std::stoul(*it); - } - break; - } - case 3: { - if (it->empty()) { - LOG(error) << "Invalid line, empty flag name, skipping..."; - valid = false; - } else { - flag.mName = *it; - } - break; - } - case 4: { - if (it->empty()) { - LOG(error) << "Invalid line, empty flag 'bad' field, skipping..."; - valid = false; - } else { - flag.mBad = static_cast(std::stoul(*it)); - } - break; - } - case 5: { - comment = *it; - break; - } - case 6: { - source = *it; - break; - } - default: { - LOG(error) << "More columns (" << pos + 1 << ") than expected (" << csvColumns - << ") in this line, skipping..."; - valid = false; - break; - } - } - } - if (valid && pos < csvColumns) { - LOG(error) << "Less columns (" << pos << ") than expected (" << csvColumns << ") in this line, skipping..."; - } else if (valid) { - insert({start, end, flag, comment, source}); - } - } -} - -std::ostream& operator<<(std::ostream& output, const TimeRangeFlagCollection& data) -{ - data.streamTo(output); - return output; -} - -const std::string& TimeRangeFlagCollection::getName() const -{ - return mName; -} - -const std::string& TimeRangeFlagCollection::getDetector() const -{ - return mDetID; -} - -int TimeRangeFlagCollection::getRunNumber() const -{ - return mRunNumber; -} - -const std::string& TimeRangeFlagCollection::getPeriodName() const -{ - return mPeriodName; -} - -const std::string& TimeRangeFlagCollection::getPassName() const -{ - return mPassName; -} -const std::string& TimeRangeFlagCollection::getProvenance() const -{ - return mProvenance; -} - -} // namespace o2::quality_control diff --git a/DataFormats/QualityControl/test/testFlagReasons.cxx b/DataFormats/QualityControl/test/testFlagReasons.cxx deleted file mode 100644 index 9e7ff15383674..0000000000000 --- a/DataFormats/QualityControl/test/testFlagReasons.cxx +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -#define BOOST_TEST_MODULE Test quality_control FlagReasons class -#define BOOST_TEST_MAIN -#define BOOST_TEST_DYN_LINK - -#include - -#include -#include - -// o2 includes -#include "DataFormatsQualityControl/FlagReasons.h" -#include "DataFormatsQualityControl/FlagReasonFactory.h" - -using namespace o2::quality_control; - -BOOST_AUTO_TEST_CASE(FlagReasons) -{ - static_assert(std::is_constructible::value == false, - "FlagReason should not be constructible outside of its static methods and the factory."); - - FlagReason rDefault; - BOOST_CHECK_EQUAL(rDefault, FlagReasonFactory::Invalid()); - - auto r1 = FlagReasonFactory::Unknown(); - BOOST_CHECK_EQUAL(r1.getID(), 1); - BOOST_CHECK_EQUAL(r1.getName(), "Unknown"); - BOOST_CHECK_EQUAL(r1.getBad(), true); - - std::cout << r1 << std::endl; - - auto r2 = r1; - BOOST_CHECK_EQUAL(r2.getID(), 1); - BOOST_CHECK_EQUAL(r1.getName(), r2.getName()); - BOOST_CHECK_EQUAL(r2.getName(), "Unknown"); - BOOST_CHECK_EQUAL(r2.getBad(), true); - - BOOST_CHECK_EQUAL(r1, r2); - BOOST_CHECK((r1 != r2) == false); - BOOST_CHECK(!(r1 < r2)); - BOOST_CHECK(!(r1 > r2)); - - auto r3 = FlagReasonFactory::LimitedAcceptance(); - BOOST_CHECK(r3 > r1); - BOOST_CHECK(!(r3 < r1)); -} diff --git a/DataFormats/QualityControl/test/testFlagTypes.cxx b/DataFormats/QualityControl/test/testFlagTypes.cxx new file mode 100644 index 0000000000000..2fe904aa1c167 --- /dev/null +++ b/DataFormats/QualityControl/test/testFlagTypes.cxx @@ -0,0 +1,56 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#define BOOST_TEST_MODULE Test quality_control FlagTypes class +#define BOOST_TEST_MAIN +#define BOOST_TEST_DYN_LINK + +#include + +#include +#include + +// o2 includes +#include "DataFormatsQualityControl/FlagType.h" +#include "DataFormatsQualityControl/FlagTypeFactory.h" + +using namespace o2::quality_control; + +BOOST_AUTO_TEST_CASE(FlagTypes) +{ + static_assert(std::is_constructible::value == false, + "FlagType should not be constructible outside of its static methods and the factory."); + + FlagType fDefault; + BOOST_CHECK_EQUAL(fDefault, FlagTypeFactory::Invalid()); + + auto f1 = FlagTypeFactory::Unknown(); + BOOST_CHECK_EQUAL(f1.getID(), 14); + BOOST_CHECK_EQUAL(f1.getName(), "Unknown"); + BOOST_CHECK_EQUAL(f1.getBad(), true); + + BOOST_CHECK_NO_THROW(std::cout << f1 << std::endl); + + auto f2 = f1; + BOOST_CHECK_EQUAL(f2.getID(), 14); + BOOST_CHECK_EQUAL(f1.getName(), f2.getName()); + BOOST_CHECK_EQUAL(f2.getName(), "Unknown"); + BOOST_CHECK_EQUAL(f2.getBad(), true); + + BOOST_CHECK_EQUAL(f1, f2); + BOOST_CHECK((f1 != f2) == false); + BOOST_CHECK(!(f1 < f2)); + BOOST_CHECK(!(f1 > f2)); + + auto f3 = FlagTypeFactory::LimitedAcceptanceMCNotReproducible(); + BOOST_CHECK(f3 < f1); + BOOST_CHECK(!(f3 > f1)); +} diff --git a/DataFormats/QualityControl/test/testQualityControlFlag.cxx b/DataFormats/QualityControl/test/testQualityControlFlag.cxx new file mode 100644 index 0000000000000..2990b3bbd355f --- /dev/null +++ b/DataFormats/QualityControl/test/testQualityControlFlag.cxx @@ -0,0 +1,40 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#define BOOST_TEST_MODULE Test quality_control QualityControlFlag class +#define BOOST_TEST_MAIN +#define BOOST_TEST_DYN_LINK + +// boost includes +#include + +// o2 includes +#include "DataFormatsQualityControl/QualityControlFlag.h" + +using namespace o2::quality_control; + +BOOST_AUTO_TEST_CASE(test_QualityControlFlag) +{ + QualityControlFlag qcFlag1{12, 34, FlagTypeFactory::BadTracking(), "comment", "source"}; + + BOOST_CHECK_EQUAL(qcFlag1.getStart(), 12); + BOOST_CHECK_EQUAL(qcFlag1.getEnd(), 34); + BOOST_CHECK_EQUAL(qcFlag1.getFlag(), FlagTypeFactory::BadTracking()); + BOOST_CHECK_EQUAL(qcFlag1.getComment(), "comment"); + BOOST_CHECK_EQUAL(qcFlag1.getSource(), "source"); + + BOOST_CHECK_THROW((QualityControlFlag{12, 0, FlagTypeFactory::BadTracking()}), std::runtime_error); + + QualityControlFlag qcFlag2{10, 34, FlagTypeFactory::BadTracking(), "comment", "source"}; + + BOOST_CHECK(qcFlag1 > qcFlag2); + BOOST_CHECK(!(qcFlag1 < qcFlag2)); +} \ No newline at end of file diff --git a/DataFormats/QualityControl/test/testQualityControlFlagCollection.cxx b/DataFormats/QualityControl/test/testQualityControlFlagCollection.cxx new file mode 100644 index 0000000000000..4eabfc5908741 --- /dev/null +++ b/DataFormats/QualityControl/test/testQualityControlFlagCollection.cxx @@ -0,0 +1,136 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#define BOOST_TEST_MODULE Test quality_control QualityControlFlagCollection class +#define BOOST_TEST_MAIN +#define BOOST_TEST_DYN_LINK + +// boost includes +#include +// STL +#include +// o2 includes +#include "DataFormatsQualityControl/QualityControlFlagCollection.h" +#include "DataFormatsQualityControl/QualityControlFlag.h" +#include "DataFormatsQualityControl/FlagTypeFactory.h" + +using namespace o2::quality_control; + +BOOST_AUTO_TEST_CASE(test_QualityControlFlagCollection_Methods) +{ + QualityControlFlag flag1{12, 34, FlagTypeFactory::BadTracking(), "comment", "source"}; + QualityControlFlag flag2{10, 34, FlagTypeFactory::BadTracking(), "comment", "source"}; + + QualityControlFlagCollection qcfc1{"Raw data checks", "TOF", {10, 20000}, 12345, "LHC22k5", "passMC", "qc_mc"}; + qcfc1.insert(flag1); // by copy + qcfc1.insert(flag2); + qcfc1.insert({50, 77, FlagTypeFactory::Invalid()}); // by move + BOOST_CHECK_EQUAL(qcfc1.size(), 3); + BOOST_CHECK_EQUAL(qcfc1.getName(), "Raw data checks"); + BOOST_CHECK_EQUAL(qcfc1.getDetector(), "TOF"); + BOOST_CHECK_EQUAL(qcfc1.getStart(), 10); + BOOST_CHECK_EQUAL(qcfc1.getEnd(), 20000); + BOOST_CHECK_EQUAL(qcfc1.getRunNumber(), 12345); + BOOST_CHECK_EQUAL(qcfc1.getPeriodName(), "LHC22k5"); + BOOST_CHECK_EQUAL(qcfc1.getPassName(), "passMC"); + BOOST_CHECK_EQUAL(qcfc1.getProvenance(), "qc_mc"); + + QualityControlFlagCollection qcfc2{"Reco checks", "TOF"}; + qcfc2.insert({50, 77, FlagTypeFactory::Invalid()}); // this is a duplicate to an entry in qcfc1 + qcfc2.insert({51, 77, FlagTypeFactory::Invalid()}); + qcfc2.insert({1234, 3434, FlagTypeFactory::LimitedAcceptanceMCNotReproducible()}); + qcfc2.insert({50, 77, FlagTypeFactory::LimitedAcceptanceMCNotReproducible()}); + BOOST_CHECK_EQUAL(qcfc2.size(), 4); + + // Try merging. Duplicate entries should be left in the 'other' objects. + // Notice that we merge the two partial TRFCs into the third, which covers all cases + QualityControlFlagCollection qcfc3{"ALL", "TOF"}; + qcfc3.merge(qcfc1); + qcfc3.merge(qcfc2); + BOOST_CHECK_EQUAL(qcfc1.size(), 0); + BOOST_CHECK_EQUAL(qcfc2.size(), 1); + BOOST_CHECK_EQUAL(qcfc3.size(), 6); + + // Try const merging. It should copy the elements and keep the 'other' intact. + QualityControlFlagCollection qcfc4{"ALL", "TOF"}; + const auto& constTrfc3 = qcfc3; + qcfc4.merge(constTrfc3); + BOOST_CHECK_EQUAL(qcfc3.size(), 6); + BOOST_CHECK_EQUAL(qcfc4.size(), 6); + + // Try merging different detectors - it should throw. + QualityControlFlagCollection qcfc5{"ALL", "TPC"}; + BOOST_CHECK_THROW(qcfc5.merge(qcfc3), std::runtime_error); + BOOST_CHECK_THROW(qcfc5.merge(constTrfc3), std::runtime_error); + + // try printing + std::cout << qcfc3 << std::endl; + + // iterating + for (const auto& flag : qcfc3) { + (void)flag; + } +} + +BOOST_AUTO_TEST_CASE(test_QualityControlFlagCollection_IO) +{ + { + QualityControlFlagCollection qcfc1{"xyz", "TST"}; + + std::stringstream store; + qcfc1.streamTo(store); + + QualityControlFlagCollection qcfc2{"xyz", "TST"}; + qcfc2.streamFrom(store); + + BOOST_CHECK_EQUAL(qcfc2.size(), 0); + } + { + QualityControlFlagCollection qcfc1{"xyz", "TST"}; + qcfc1.insert({50, 77, FlagTypeFactory::Invalid(), "a comment", "a source"}); + qcfc1.insert({51, 77, FlagTypeFactory::Invalid()}); + qcfc1.insert({1234, 3434, FlagTypeFactory::LimitedAcceptanceMCNotReproducible()}); + qcfc1.insert({50, 77, FlagTypeFactory::LimitedAcceptanceMCNotReproducible()}); + qcfc1.insert({43434, 63421, FlagTypeFactory::Good()}); + + std::stringstream store; + qcfc1.streamTo(store); + + QualityControlFlagCollection qcfc2{"xyz", "TST"}; + qcfc2.streamFrom(store); + + BOOST_REQUIRE_EQUAL(qcfc1.size(), qcfc2.size()); + for (auto it1 = qcfc1.begin(), it2 = qcfc2.begin(); it1 != qcfc1.end() && it2 != qcfc2.end(); ++it1, ++it2) { + BOOST_CHECK_EQUAL(*it1, *it2); + } + } + { + std::stringstream store; + store << "start,end,flag_id,invalid,header,format\n"; + store << R"(123,345,11,"fdsa",1,"comment","source")"; + QualityControlFlagCollection qcfc1{"A", "TST"}; + BOOST_CHECK_THROW(qcfc1.streamFrom(store), std::runtime_error); + } + { + std::stringstream store; + store << "start,end,flag_id,flag_name,flag_bad,comment,source\n"; + store << R"(123,345,11,"fdsa",1,"comment","source","toomanycolumns")" << '\n'; + store << R"(123,345,11,"fdsa",1)" << '\n'; + store << R"(123,,11,"fdsa",1,"comment","source")" << '\n'; + store << R"(,345,11,"fdsa",1,"comment","source")" << '\n'; + store << R"(123,345,,"fdsa",1,"comment","source")" << '\n'; + store << R"(123,345,11,"",1,"comment","source")" << '\n'; + store << R"(123,345,11,"fdsa",,"comment","source")" << '\n'; + QualityControlFlagCollection qcfc1{"A", "TST"}; + BOOST_CHECK_NO_THROW(qcfc1.streamFrom(store)); + BOOST_CHECK_EQUAL(qcfc1.size(), 0); + } +} diff --git a/DataFormats/QualityControl/test/testTimeRangeFlag.cxx b/DataFormats/QualityControl/test/testTimeRangeFlag.cxx deleted file mode 100644 index 47eef61199103..0000000000000 --- a/DataFormats/QualityControl/test/testTimeRangeFlag.cxx +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -#define BOOST_TEST_MODULE Test quality_control TimeRangeFlag class -#define BOOST_TEST_MAIN -#define BOOST_TEST_DYN_LINK - -// boost includes -#include - -// o2 includes -#include "DataFormatsQualityControl/TimeRangeFlag.h" - -using namespace o2::quality_control; - -BOOST_AUTO_TEST_CASE(test_TimeRangeFlag) -{ - TimeRangeFlag trf1{12, 34, FlagReasonFactory::BadTracking(), "comment", "source"}; - - BOOST_CHECK_EQUAL(trf1.getStart(), 12); - BOOST_CHECK_EQUAL(trf1.getEnd(), 34); - BOOST_CHECK_EQUAL(trf1.getFlag(), FlagReasonFactory::BadTracking()); - BOOST_CHECK_EQUAL(trf1.getComment(), "comment"); - BOOST_CHECK_EQUAL(trf1.getSource(), "source"); - - BOOST_CHECK_THROW((TimeRangeFlag{12, 0, FlagReasonFactory::BadTracking()}), std::runtime_error); - - TimeRangeFlag trf2{10, 34, FlagReasonFactory::BadTracking(), "comment", "source"}; - - BOOST_CHECK(trf1 > trf2); - BOOST_CHECK(!(trf1 < trf2)); -} \ No newline at end of file diff --git a/DataFormats/QualityControl/test/testTimeRangeFlagCollection.cxx b/DataFormats/QualityControl/test/testTimeRangeFlagCollection.cxx deleted file mode 100644 index 9f5468ce00865..0000000000000 --- a/DataFormats/QualityControl/test/testTimeRangeFlagCollection.cxx +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -#define BOOST_TEST_MODULE Test quality_control TimeRangeFlagCollection class -#define BOOST_TEST_MAIN -#define BOOST_TEST_DYN_LINK - -// boost includes -#include -// STL -#include -// o2 includes -#include "DataFormatsQualityControl/TimeRangeFlagCollection.h" -#include "DataFormatsQualityControl/TimeRangeFlag.h" - -using namespace o2::quality_control; - -BOOST_AUTO_TEST_CASE(test_TimeRangeFlagCollection_Methods) -{ - TimeRangeFlag trf1{12, 34, FlagReasonFactory::BadTracking(), "comment", "source"}; - TimeRangeFlag trf2{10, 34, FlagReasonFactory::BadTracking(), "comment", "source"}; - - TimeRangeFlagCollection trfc1{"Raw data checks", "TOF", {10, 20000}, 12345, "LHC22k5", "passMC", "qc_mc"}; - trfc1.insert(trf1); // by copy - trfc1.insert(trf2); - trfc1.insert({50, 77, FlagReasonFactory::Invalid()}); // by move - BOOST_CHECK_EQUAL(trfc1.size(), 3); - BOOST_CHECK_EQUAL(trfc1.getName(), "Raw data checks"); - BOOST_CHECK_EQUAL(trfc1.getDetector(), "TOF"); - BOOST_CHECK_EQUAL(trfc1.getStart(), 10); - BOOST_CHECK_EQUAL(trfc1.getEnd(), 20000); - BOOST_CHECK_EQUAL(trfc1.getRunNumber(), 12345); - BOOST_CHECK_EQUAL(trfc1.getPeriodName(), "LHC22k5"); - BOOST_CHECK_EQUAL(trfc1.getPassName(), "passMC"); - BOOST_CHECK_EQUAL(trfc1.getProvenance(), "qc_mc"); - - TimeRangeFlagCollection trfc2{"Reco checks", "TOF"}; - trfc2.insert({50, 77, FlagReasonFactory::Invalid()}); // this is a duplicate to an entry in trfc1 - trfc2.insert({51, 77, FlagReasonFactory::Invalid()}); - trfc2.insert({1234, 3434, FlagReasonFactory::LimitedAcceptance()}); - trfc2.insert({50, 77, FlagReasonFactory::LimitedAcceptance()}); - BOOST_CHECK_EQUAL(trfc2.size(), 4); - - // Try merging. Duplicate entries should be left in the 'other' objects. - // Notice that we merge the two partial TRFCs into the third, which covers all cases - TimeRangeFlagCollection trfc3{"ALL", "TOF"}; - trfc3.merge(trfc1); - trfc3.merge(trfc2); - BOOST_CHECK_EQUAL(trfc1.size(), 0); - BOOST_CHECK_EQUAL(trfc2.size(), 1); - BOOST_CHECK_EQUAL(trfc3.size(), 6); - - // Try const merging. It should copy the elements and keep the 'other' intact. - TimeRangeFlagCollection trfc4{"ALL", "TOF"}; - const auto& constTrfc3 = trfc3; - trfc4.merge(constTrfc3); - BOOST_CHECK_EQUAL(trfc3.size(), 6); - BOOST_CHECK_EQUAL(trfc4.size(), 6); - - // Try merging different detectors - it should throw. - TimeRangeFlagCollection trfc5{"ALL", "TPC"}; - BOOST_CHECK_THROW(trfc5.merge(trfc3), std::runtime_error); - BOOST_CHECK_THROW(trfc5.merge(constTrfc3), std::runtime_error); - - // try printing - std::cout << trfc3 << std::endl; - - // iterating - for (const auto& trf : trfc3) { - (void)trf; - } -} - -BOOST_AUTO_TEST_CASE(test_TimeRangeFlagCollection_IO) -{ - { - TimeRangeFlagCollection trfc1{"xyz", "TST"}; - - std::stringstream store; - trfc1.streamTo(store); - - TimeRangeFlagCollection trfc2{"xyz", "TST"}; - trfc2.streamFrom(store); - - BOOST_CHECK_EQUAL(trfc2.size(), 0); - } - { - TimeRangeFlagCollection trfc1{"xyz", "TST"}; - trfc1.insert({50, 77, FlagReasonFactory::Invalid(), "a comment", "a source"}); - trfc1.insert({51, 77, FlagReasonFactory::Invalid()}); - trfc1.insert({1234, 3434, FlagReasonFactory::LimitedAcceptance()}); - trfc1.insert({50, 77, FlagReasonFactory::LimitedAcceptance()}); - trfc1.insert({43434, 63421, FlagReasonFactory::NotBadFlagExample()}); - - std::stringstream store; - trfc1.streamTo(store); - - TimeRangeFlagCollection trfc2{"xyz", "TST"}; - trfc2.streamFrom(store); - - BOOST_REQUIRE_EQUAL(trfc1.size(), trfc2.size()); - for (auto it1 = trfc1.begin(), it2 = trfc2.begin(); it1 != trfc1.end() && it2 != trfc2.end(); ++it1, ++it2) { - BOOST_CHECK_EQUAL(*it1, *it2); - } - } - { - std::stringstream store; - store << "start,end,flag_id,invalid,header,format\n"; - store << R"(123,345,11,"fdsa",1,"comment","source")"; - TimeRangeFlagCollection trfc1{"A", "TST"}; - BOOST_CHECK_THROW(trfc1.streamFrom(store), std::runtime_error); - } - { - std::stringstream store; - store << "start,end,flag_id,flag_name,flag_bad,comment,source\n"; - store << R"(123,345,11,"fdsa",1,"comment","source","toomanycolumns")" << '\n'; - store << R"(123,345,11,"fdsa",1)" << '\n'; - store << R"(123,,11,"fdsa",1,"comment","source")" << '\n'; - store << R"(,345,11,"fdsa",1,"comment","source")" << '\n'; - store << R"(123,345,,"fdsa",1,"comment","source")" << '\n'; - store << R"(123,345,11,"",1,"comment","source")" << '\n'; - store << R"(123,345,11,"fdsa",,"comment","source")" << '\n'; - TimeRangeFlagCollection trfc1{"A", "TST"}; - BOOST_CHECK_NO_THROW(trfc1.streamFrom(store)); - BOOST_CHECK_EQUAL(trfc1.size(), 0); - } -} diff --git a/DataFormats/Reconstruction/CMakeLists.txt b/DataFormats/Reconstruction/CMakeLists.txt index 7ed154348a2c3..d3ca8fdc70ad6 100644 --- a/DataFormats/Reconstruction/CMakeLists.txt +++ b/DataFormats/Reconstruction/CMakeLists.txt @@ -8,6 +8,7 @@ # In applying this license CERN does not waive the privileges and immunities # granted to it by virtue of its status as an Intergovernmental Organization # or submit itself to any jurisdiction. +# add_compile_options(-O0 -g -fPIC) o2_add_library(ReconstructionDataFormats SOURCES src/TrackParametrization.cxx @@ -17,6 +18,7 @@ o2_add_library(ReconstructionDataFormats src/TrackTPCITS.cxx src/Vertex.cxx src/PrimaryVertex.cxx + src/PrimaryVertexExt.cxx src/MatchInfoTOF.cxx src/MatchInfoTOFReco.cxx src/TrackLTIntegral.cxx @@ -24,18 +26,23 @@ o2_add_library(ReconstructionDataFormats src/DCA.cxx src/V0.cxx src/Cascade.cxx + src/Decay3Body.cxx + src/DecayNBodyIndex.cxx + src/StrangeTrack.cxx src/GlobalTrackID.cxx src/VtxTrackIndex.cxx src/VtxTrackRef.cxx src/TrackTPCTOF.cxx src/TrackCosmics.cxx src/TrackMCHMID.cxx + src/TrackHMP.cxx src/MatchInfoHMP.cxx PUBLIC_LINK_LIBRARIES O2::GPUCommon O2::GPUUtils O2::FrameworkLogger O2::DetectorsCommonDataFormats - O2::CommonDataFormat) + O2::CommonDataFormat + O2::Field) o2_target_root_dictionary( ReconstructionDataFormats @@ -46,6 +53,7 @@ o2_target_root_dictionary( include/ReconstructionDataFormats/GlobalFwdTrack.h include/ReconstructionDataFormats/Vertex.h include/ReconstructionDataFormats/PrimaryVertex.h + include/ReconstructionDataFormats/PrimaryVertexExt.h include/ReconstructionDataFormats/MatchInfoTOF.h include/ReconstructionDataFormats/MatchInfoTOFReco.h include/ReconstructionDataFormats/TrackLTIntegral.h @@ -53,6 +61,9 @@ o2_target_root_dictionary( include/ReconstructionDataFormats/DCA.h include/ReconstructionDataFormats/V0.h include/ReconstructionDataFormats/Cascade.h + include/ReconstructionDataFormats/DecayNBodyIndex.h + include/ReconstructionDataFormats/Decay3Body.h + include/ReconstructionDataFormats/StrangeTrack.h include/ReconstructionDataFormats/GlobalTrackID.h include/ReconstructionDataFormats/VtxTrackIndex.h include/ReconstructionDataFormats/VtxTrackRef.h @@ -61,7 +72,9 @@ o2_target_root_dictionary( include/ReconstructionDataFormats/TrackMCHMID.h include/ReconstructionDataFormats/MatchInfoFwd.h include/ReconstructionDataFormats/BCRange.h + include/ReconstructionDataFormats/TrackHMP.h include/ReconstructionDataFormats/MatchInfoHMP.h + include/ReconstructionDataFormats/HelixHelper.h ) o2_add_test(Vertex diff --git a/DataFormats/Reconstruction/include/ReconstructionDataFormats/BCRange.h b/DataFormats/Reconstruction/include/ReconstructionDataFormats/BCRange.h index 35719675f3b6c..392fd899fcceb 100644 --- a/DataFormats/Reconstruction/include/ReconstructionDataFormats/BCRange.h +++ b/DataFormats/Reconstruction/include/ReconstructionDataFormats/BCRange.h @@ -114,7 +114,7 @@ struct bcRanges { // is merging required? if (!isMerged || toForce) { std::vector tmpList; - uint64_t ifirst = 0, ilast; + uint64_t ifirst = 0, ilast = 0; // apply sorting of the ranges if (!isSorted) { diff --git a/DataFormats/Reconstruction/include/ReconstructionDataFormats/Cascade.h b/DataFormats/Reconstruction/include/ReconstructionDataFormats/Cascade.h index 77047357c092f..7744036247f85 100644 --- a/DataFormats/Reconstruction/include/ReconstructionDataFormats/Cascade.h +++ b/DataFormats/Reconstruction/include/ReconstructionDataFormats/Cascade.h @@ -27,16 +27,12 @@ namespace dataformats class Cascade : public V0 { public: + using V0::V0; Cascade() = default; + Cascade(const std::array& xyz, const std::array& pxyz, const std::array& covxyz, const o2::track::TrackParCov& v0, const o2::track::TrackParCov& bachelor, - int v0ID, GIndex bachelorID, o2::track::PID pid = o2::track::PID::XiMinus); - - GIndex getBachelorID() const { return mProngIDs[1]; } - void setBachelorID(GIndex gid) { mProngIDs[1] = gid; } - - int getV0ID() const { return int(mProngIDs[0].getRaw()); } - void setV0ID(int vid) { mProngIDs[0].setRaw(GIndex::Base_t(vid)); } + o2::track::PID pid = o2::track::PID::XiMinus) : V0(xyz, pxyz, covxyz, v0, bachelor, pid) {} const Track& getV0Track() const { return mProngs[0]; } Track& getV0Track() { return mProngs[0]; } @@ -47,10 +43,10 @@ class Cascade : public V0 void setV0Track(const Track& t) { mProngs[0] = t; } void setBachelorTrack(const Track& t) { mProngs[1] = t; } - protected: - GIndex getProngID(int i) const = delete; + float calcMass2AsXiMinus() const { return calcMass2PID(PID::Lambda, PID::Pion); } + float calcMass2AsOmegaMinus() const { return calcMass2PID(PID::Lambda, PID::Kaon); } - ClassDefNV(Cascade, 1); + ClassDefNV(Cascade, 2); }; } // namespace dataformats diff --git a/DataFormats/Reconstruction/include/ReconstructionDataFormats/DCA.h b/DataFormats/Reconstruction/include/ReconstructionDataFormats/DCA.h index 922470f8992f5..6eb41b798e101 100644 --- a/DataFormats/Reconstruction/include/ReconstructionDataFormats/DCA.h +++ b/DataFormats/Reconstruction/include/ReconstructionDataFormats/DCA.h @@ -14,10 +14,10 @@ #include "GPUCommonDef.h" #include "GPUCommonRtypes.h" -#include "GPUCommonArray.h" #ifndef GPUCA_GPUCODE_DEVICE #include +#include #endif /// \author ruben.shahoyan@cern.ch @@ -67,7 +67,7 @@ class DCA private: float mY = 0.f; float mZ = 0.f; - gpu::gpustd::array mCov; ///< s2y, syz, s2z + std::array mCov; ///< s2y, syz, s2z ClassDefNV(DCA, 1); }; diff --git a/DataFormats/Reconstruction/include/ReconstructionDataFormats/Decay3Body.h b/DataFormats/Reconstruction/include/ReconstructionDataFormats/Decay3Body.h new file mode 100644 index 0000000000000..f7f18976fa508 --- /dev/null +++ b/DataFormats/Reconstruction/include/ReconstructionDataFormats/Decay3Body.h @@ -0,0 +1,60 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef ALICEO2_3BODY_H +#define ALICEO2_3BODY_H + +#include "ReconstructionDataFormats/Track.h" +#include "ReconstructionDataFormats/PID.h" +#include +#include + +namespace o2 +{ +namespace dataformats +{ + +class Decay3Body : public o2::track::TrackParCov /// TO BE DONE: extend to generic N body vertex +{ + public: + using Track = o2::track::TrackParCov; + using PID = o2::track::PID; + + Decay3Body() = default; + Decay3Body(const std::array& xyz, const std::array& pxyz, const std::array& covxyz, const Track& tr0, const Track& tr1, const Track& tr2, o2::track::PID pid = o2::track::PID::HyperTriton); + + const Track& getProng(int i) const { return mProngs[i]; } + Track& getProng(int i) { return mProngs[i]; } + void setProng(int i, const Track& t) { mProngs[i] = t; } + + float getCosPA() const { return mCosPA; } + void setCosPA(float c) { mCosPA = c; } + + float getDCA() const { return mDCA; } + void setDCA(float d) { mDCA = d; } + + float calcMass2() const { return calcMass2PID(mProngs[0].getPID(), mProngs[1].getPID(), mProngs[2].getPID()); } + float calcMass2PID(int pid0, int pid1, int pid2) const { return calcMass2(PID::getMass2(pid0), PID::getMass2(pid1), PID::getMass2(pid2)); } + float calcMass2(float mass0, float mass1, float mass2) const; + + float calcR2() const { return getX() * getX() + getY() * getY(); } + + protected: + std::array mProngs; // prongs kinematics at vertex + float mCosPA = 0; // cos of pointing angle + float mDCA = 9990; // distance of closest approach of prongs + + ClassDefNV(Decay3Body, 1); +}; + +} // namespace dataformats +} // namespace o2 +#endif diff --git a/DataFormats/Reconstruction/include/ReconstructionDataFormats/DecayNBodyIndex.h b/DataFormats/Reconstruction/include/ReconstructionDataFormats/DecayNBodyIndex.h new file mode 100644 index 0000000000000..5a5a8a9e64cca --- /dev/null +++ b/DataFormats/Reconstruction/include/ReconstructionDataFormats/DecayNBodyIndex.h @@ -0,0 +1,108 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef ALICEO2_NBODY_INDEX_H +#define ALICEO2_NBODY_INDEX_H + +#include "ReconstructionDataFormats/VtxTrackIndex.h" + +namespace o2::dataformats +{ + +template +class DecayNBodyIndex +{ + public: + using GIndex = o2::dataformats::VtxTrackIndex; + DecayNBodyIndex() = default; + DecayNBodyIndex(int v, const std::array& arr) : mVertexID(v), mProngIDs(arr) {} + DecayNBodyIndex(int v, std::initializer_list l) : mVertexID(v) + { + assert(l.size() == N); + int i = 0; + for (auto e : l) { + mProngIDs[i++] = e; + } + } + GIndex getProngID(int i) const { return mProngIDs[i]; } + void setProngID(int i, GIndex gid) { mProngIDs[i] = gid; } + int getVertexID() const { return mVertexID; } + void setVertexID(int id) { mVertexID = id; } + uint8_t getBits() const { return mBits; } + bool testBit(int i) const { return (mBits & (0x1 << i)) != 0; } + void setBit(int i) { mBits |= (0x1 << i); } + void resetBit(int i) { mBits &= ~(0x1 << i); } + + const std::array& getProngs() const { return mProngIDs; } + static constexpr int getNProngs() { return N; } + + protected: + int mVertexID = -1; // id of parent vertex + std::array mProngIDs{}; // global IDs of prongs + uint8_t mBits = 0; // user defined bits + + ClassDefNV(DecayNBodyIndex, 2); +}; + +class V0Index : public DecayNBodyIndex<2> +{ + public: + enum V0Type : uint8_t { + kStandaloneV0 = 0, + kPhotonOnly, + kCollinear, + }; + using DecayNBodyIndex<2>::DecayNBodyIndex; + V0Index(int v, GIndex p, GIndex n) : DecayNBodyIndex<2>(v, {p, n}) {} + bool isStandaloneV0() const { return testBit(kStandaloneV0); } + bool isPhotonOnly() const { return testBit(kPhotonOnly); } + bool isCollinear() const { return testBit(kCollinear); } + void setStandaloneV0() { setBit(kStandaloneV0); } + void setPhotonOnly() { setBit(kPhotonOnly); } + void setCollinear() { setBit(kCollinear); } + ClassDefNV(V0Index, 1); +}; + +class Decay3BodyIndex : public DecayNBodyIndex<3> +{ + public: + using DecayNBodyIndex<3>::DecayNBodyIndex; + Decay3BodyIndex(int v, GIndex p0, GIndex p1, GIndex p2) : DecayNBodyIndex<3>(v, {p0, p1, p2}) {} + ClassDefNV(Decay3BodyIndex, 1); +}; + +class CascadeIndex +{ + public: + using GIndex = o2::dataformats::VtxTrackIndex; + CascadeIndex() = default; + CascadeIndex(int v, int v0id, GIndex bachelorID) : mVertexID(v), mV0ID(v0id), mBach(bachelorID) {} + + GIndex getBachelorID() const { return mBach; } + void setBachelorID(GIndex gid) { mBach = gid; } + + int getV0ID() const { return mV0ID; } + void setV0ID(int vid) { mV0ID = vid; } + + int getVertexID() const { return mVertexID; } + void setVertexID(int id) { mVertexID = id; } + + protected: + int mVertexID = -1; + int mV0ID = -1; + GIndex mBach{}; + + ClassDefNV(CascadeIndex, 1); +}; + +} // namespace o2::dataformats + +#endif diff --git a/DataFormats/Reconstruction/include/ReconstructionDataFormats/GlobalFwdTrack.h b/DataFormats/Reconstruction/include/ReconstructionDataFormats/GlobalFwdTrack.h index 15d816b7b7727..5d13a216316ef 100644 --- a/DataFormats/Reconstruction/include/ReconstructionDataFormats/GlobalFwdTrack.h +++ b/DataFormats/Reconstruction/include/ReconstructionDataFormats/GlobalFwdTrack.h @@ -31,6 +31,7 @@ class GlobalFwdTrack : public o2::track::TrackParCovFwd, public o2::dataformats: public: GlobalFwdTrack() = default; GlobalFwdTrack(const GlobalFwdTrack& t) = default; + GlobalFwdTrack(o2::track::TrackParCovFwd const& t) { *this = t; } ~GlobalFwdTrack() = default; SMatrix5 computeResiduals2Cov(const o2::track::TrackParCovFwd& t) const diff --git a/DataFormats/Reconstruction/include/ReconstructionDataFormats/GlobalTrackID.h b/DataFormats/Reconstruction/include/ReconstructionDataFormats/GlobalTrackID.h index 99bd6ddd762cd..06d3b50de03f0 100644 --- a/DataFormats/Reconstruction/include/ReconstructionDataFormats/GlobalTrackID.h +++ b/DataFormats/Reconstruction/include/ReconstructionDataFormats/GlobalTrackID.h @@ -93,7 +93,7 @@ class GlobalTrackID : public AbstractRef<25, 5, 2> #ifndef GPUCA_GPUCODE static auto getSourceName(int s) { - return DetID::getNames(getSourceDetectorsMask(s), '-'); + return s == ITSAB ? std::string{"ITSAB"} : DetID::getNames(getSourceDetectorsMask(s), '-'); } static mask_t getSourcesMask(const std::string_view srcList); static std::string getSourcesNames(mask_t srcm); @@ -157,7 +157,7 @@ GPUconstexpr() GlobalTrackID::mask_t sMasks[GlobalTrackID::NSources] = ///< dete GlobalTrackID::mask_t(math_utils::bit2Mask(GlobalTrackID::CPV)), GlobalTrackID::mask_t(math_utils::bit2Mask(GlobalTrackID::EMC)), GlobalTrackID::mask_t(math_utils::bit2Mask(GlobalTrackID::HMP)), GlobalTrackID::mask_t(math_utils::bit2Mask(GlobalTrackID::MFT)), GlobalTrackID::mask_t(math_utils::bit2Mask(GlobalTrackID::MCH)), GlobalTrackID::mask_t(math_utils::bit2Mask(GlobalTrackID::MID)), GlobalTrackID::mask_t(math_utils::bit2Mask(GlobalTrackID::ZDC)), GlobalTrackID::mask_t(math_utils::bit2Mask(GlobalTrackID::FT0)), GlobalTrackID::mask_t(math_utils::bit2Mask(GlobalTrackID::FV0)), GlobalTrackID::mask_t(math_utils::bit2Mask(GlobalTrackID::FDD)), GlobalTrackID::mask_t(math_utils::bit2Mask(GlobalTrackID::ITSTPC)), GlobalTrackID::mask_t(math_utils::bit2Mask(GlobalTrackID::TPCTOF)), GlobalTrackID::mask_t(math_utils::bit2Mask(GlobalTrackID::TPCTRD)), GlobalTrackID::mask_t(math_utils::bit2Mask(GlobalTrackID::MFTMCH)), GlobalTrackID::mask_t(math_utils::bit2Mask(GlobalTrackID::ITSTPCTRD)), - GlobalTrackID::mask_t(math_utils::bit2Mask(GlobalTrackID::ITSTPCTOF)), GlobalTrackID::mask_t(math_utils::bit2Mask(GlobalTrackID::TPCTRDTOF)), GlobalTrackID::mask_t(math_utils::bit2Mask(GlobalTrackID::ITSTPCTRDTOF)), GlobalTrackID::mask_t(math_utils::bit2Mask(GlobalTrackID::MFTMCHMID)), + GlobalTrackID::mask_t(math_utils::bit2Mask(GlobalTrackID::ITSTPCTOF)), GlobalTrackID::mask_t(math_utils::bit2Mask(GlobalTrackID::TPCTRDTOF)), GlobalTrackID::mask_t(math_utils::bit2Mask(GlobalTrackID::MFTMCHMID)), GlobalTrackID::mask_t(math_utils::bit2Mask(GlobalTrackID::ITSTPCTRDTOF)), GlobalTrackID::mask_t(math_utils::bit2Mask(GlobalTrackID::ITSAB)), GlobalTrackID::mask_t(math_utils::bit2Mask(GlobalTrackID::CTP)), GlobalTrackID::mask_t(math_utils::bit2Mask(GlobalTrackID::MCHMID)) // Temporary ordering diff --git a/DataFormats/Reconstruction/include/ReconstructionDataFormats/HelixHelper.h b/DataFormats/Reconstruction/include/ReconstructionDataFormats/HelixHelper.h new file mode 100644 index 0000000000000..d197cba256c0e --- /dev/null +++ b/DataFormats/Reconstruction/include/ReconstructionDataFormats/HelixHelper.h @@ -0,0 +1,307 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file HelixHelper.h +/// \brief Helper classes for helical tracks manipulations +/// \author ruben.shahoyan@cern.ch + +#ifndef _ALICEO2_HELIX_HELPER_ +#define _ALICEO2_HELIX_HELPER_ + +#include "CommonConstants/MathConstants.h" +#include "MathUtils/Utils.h" +#include "MathUtils/Primitive2D.h" + +namespace o2 +{ +namespace track +{ + +///__________________________________________________________________________ +//< precalculated track radius, center, alpha sin,cos and their combinations +struct TrackAuxPar : public o2::math_utils::CircleXYf_t { + float c, s, cc, ss, cs; // cos ans sin of track alpha and their products + + GPUdDefault() TrackAuxPar() = default; + + template + GPUd() TrackAuxPar(const T& trc, float bz) + { + set(trc, bz); + } + GPUd() float cosDif(const TrackAuxPar& t) const { return c * t.c + s * t.s; } // cos(alpha_this - alha_t) + GPUd() float sinDif(const TrackAuxPar& t) const { return s * t.c - c * t.s; } // sin(alpha_this - alha_t) + + template + GPUd() void set(const T& trc, float bz) + { + trc.getCircleParams(bz, *this, s, c); + cc = c * c; + ss = s * s; + cs = c * s; + } + ClassDefNV(TrackAuxPar, 1); +}; + +//__________________________________________________________ +//< crossing coordinates of 2 circles +struct CrossInfo { + static constexpr float MaxDistXYDef = 10.; + float xDCA[2] = {}; + float yDCA[2] = {}; + int nDCA = 0; + + GPUd() int circlesCrossInfo(const TrackAuxPar& trax0, const TrackAuxPar& trax1, float maxDistXY = MaxDistXYDef, bool isCollinear = false) + { + const auto& trcA = trax0.rC > trax1.rC ? trax0 : trax1; // designate the largest circle as A + const auto& trcB = trax0.rC > trax1.rC ? trax1 : trax0; + nDCA = 0; + float xDist = trcB.xC - trcA.xC, yDist = trcB.yC - trcA.yC; + float dist2 = xDist * xDist + yDist * yDist, dist = o2::gpu::GPUCommonMath::Sqrt(dist2), rsum = trcA.rC + trcB.rC; + if (dist < 1e-12) { + return nDCA; // circles are concentric? + } + if (dist > rsum) { // circles don't touch, chose a point in between + // the parametric equation of lines connecting the centers is + // x = x0 + t/dist * (x1-x0), y = y0 + t/dist * (y1-y0) + if (dist - rsum > maxDistXY) { // too large distance + return nDCA; + } + notTouchingXY(dist, xDist, yDist, trcA, trcB.rC, isCollinear); + } else if (auto dfr = dist + trcB.rC - trcA.rC; dfr < 0.) { // the small circle is nestled into large one w/o touching + if (dfr > -maxDistXY) { + // select the point of closest approach of 2 circles + notTouchingXY(dist, xDist, yDist, trcA, -trcB.rC, isCollinear); + } else { + return nDCA; + } + } else { // 2 intersection points + if (isCollinear) { + /// collinear tracks, e.g. electrons from photon conversion + /// if there are 2 crossings of the circle it is better to take + /// a weighted average of the crossing points as a radius + float r2r = trcA.rC + trcB.rC; + float r1_r = trcA.rC / r2r; + float r2_r = trcB.rC / r2r; + xDCA[0] = r2_r * trcA.xC + r1_r * trcB.xC; + yDCA[0] = r2_r * trcA.yC + r1_r * trcB.yC; + nDCA = 1; + } else if (o2::gpu::GPUCommonMath::Abs(xDist) < o2::gpu::GPUCommonMath::Abs(yDist)) { + // to simplify calculations, we move to new frame x->x+Xc0, y->y+Yc0, so that + // the 1st one is centered in origin + float a = (trcA.rC * trcA.rC - trcB.rC * trcB.rC + dist2) / (2. * yDist), b = -xDist / yDist, ab = a * b, bb = b * b; + float det = ab * ab - (1. + bb) * (a * a - trcA.rC * trcA.rC); + if (det > 0.) { + det = o2::gpu::GPUCommonMath::Sqrt(det); + xDCA[0] = (-ab + det) / (1. + b * b); + yDCA[0] = a + b * xDCA[0] + trcA.yC; + xDCA[0] += trcA.xC; + xDCA[1] = (-ab - det) / (1. + b * b); + yDCA[1] = a + b * xDCA[1] + trcA.yC; + xDCA[1] += trcA.xC; + nDCA = 2; + } else { // due to the finite precision the det<=0, i.e. the circles are barely touching, fall back to this special case + notTouchingXY(dist, xDist, yDist, trcA, trcB.rC); + } + } else { + float a = (trcA.rC * trcA.rC - trcB.rC * trcB.rC + dist2) / (2. * xDist), b = -yDist / xDist, ab = a * b, bb = b * b; + float det = ab * ab - (1. + bb) * (a * a - trcA.rC * trcA.rC); + if (det > 0.) { + det = o2::gpu::GPUCommonMath::Sqrt(det); + yDCA[0] = (-ab + det) / (1. + bb); + xDCA[0] = a + b * yDCA[0] + trcA.xC; + yDCA[0] += trcA.yC; + yDCA[1] = (-ab - det) / (1. + bb); + xDCA[1] = a + b * yDCA[1] + trcA.xC; + yDCA[1] += trcA.yC; + nDCA = 2; + } else { // due to the finite precision the det<=0, i.e. the circles are barely touching, fall back to this special case + notTouchingXY(dist, xDist, yDist, trcA, trcB.rC); + } + } + } + return nDCA; + } + + GPUd() void notTouchingXY(float dist, float xDist, float yDist, const TrackAuxPar& trcA, float rBSign, bool isCollinear = false) + { + if (isCollinear) { + /// for collinear tracks it is better to take + /// a weighted average of the crossing points as a radius + float r2r = trcA.rC + rBSign; + float r1_r = trcA.rC / r2r; + float r2_r = rBSign / r2r; + xDCA[0] = r2_r * trcA.xC + r1_r * (xDist + trcA.xC); + yDCA[0] = r2_r * trcA.yC + r1_r * (yDist + trcA.yC); + } else { + // fast method to calculate DCA between 2 circles, assuming that they don't touch each outer: + // the parametric equation of lines connecting the centers is x = xA + t/dist * xDist, y = yA + t/dist * yDist + // with xA,yY being the center of the circle A ( = trcA.xC, trcA.yC ), xDist = trcB.xC = trcA.xC ... + // There are 2 special cases: + // (a) small circle is inside the large one: provide rBSign as -trcB.rC + // (b) circle are side by side: provide rBSign as trcB.rC + auto t2d = (dist + trcA.rC - rBSign) / dist; + xDCA[0] = trcA.xC + 0.5 * (xDist * t2d); + yDCA[0] = trcA.yC + 0.5 * (yDist * t2d); + } + nDCA = 1; + } + + template + GPUd() int linesCrossInfo(const TrackAuxPar& trax0, const T& tr0, + const TrackAuxPar& trax1, const T& tr1, float maxDistXY = MaxDistXYDef) + { + /// closest approach of 2 straight lines + /// TrackParam propagation can be parameterized in lab in a form + /// xLab(t) = (x*cosAlp - y*sinAlp) + t*(cosAlp - sinAlp* snp/csp) = xLab0 + t*(cosAlp - sinAlp* snp/csp) + /// yLab(t) = (x*sinAlp + y*cosAlp) + t*(sinAlp + cosAlp* snp/csp) = yLab0 + t*(sinAlp + cosAlp* snp/csp) + /// zLab(t) = z + t * tgl / csp = zLab0 + t * tgl / csp + /// where t is the x-step in the track alpha-frame, xLab,yLab,zLab are reference track coordinates in lab + /// frame (filled by TrackAuxPar for straight line tracks). + /// + /// Therefore, for the parametric track equation in lab 3D we have (wrt tracking-X increment t) + /// xL(t) = xL + t Kx; Kx = (cosAlp - sinAlp* snp/csp) + /// yL(t) = yL + t Ky; Ky = (sinAlp + cosAlp* snp/csp) + /// zL(t) = zL + t Kz; Kz = tgl / csp + /// Note that Kx^2 + Ky^2 + Kz^2 = (1+tgl^2) / csp^2 + nDCA = 0; + float dx = trax1.xC - trax0.xC; // for straight line TrackAuxPar stores lab coordinates at referene point!!! + float dy = trax1.yC - trax0.yC; // + float dz = tr1.getZ() - tr0.getZ(); + auto csp0i2 = 1. / tr0.getCsp2(); // 1 / csp^2 + auto csp0i = o2::gpu::GPUCommonMath::Sqrt(csp0i2); + auto tgp0 = tr0.getSnp() * csp0i; + float kx0 = trax0.c - trax0.s * tgp0; + float ky0 = trax0.s + trax0.c * tgp0; + float kz0 = tr0.getTgl() * csp0i; + auto csp1i2 = 1. / tr1.getCsp2(); // 1 / csp^2 + auto csp1i = o2::gpu::GPUCommonMath::Sqrt(csp1i2); + auto tgp1 = tr1.getSnp() * o2::gpu::GPUCommonMath::Sqrt(csp1i2); + float kx1 = trax1.c - trax1.s * tgp1; + float ky1 = trax1.s + trax1.c * tgp1; + float kz1 = tr1.getTgl() * csp1i; + /// Minimize |vecL1 - vecL0|^2 wrt t0 and t1: point of closest approach + /// Leads to system + /// A Dx = B with Dx = {dx0, dx1} + /// with A = + /// | kx0^2+ky0^2+kz0^2 -(kx0*kx1+ky0*ky1+kz0*kz1) | = (1+tgl0^2) / csp0^2 .... + /// | -(kx0*kx1+ky0*ky1+kz0*kz1) kx0^2+ky0^2+kz0^2 | ..... (1+tgl1^2) / csp1^2 + /// and B = {(dx Kx0 + dy Ky0 + dz Kz0), -(dx Kx1 + dy Ky1 + dz Kz1) } + /// + float a00 = (1.f + tr0.getTgl() * tr0.getTgl()) * csp0i2, a11 = (1.f + tr1.getTgl() * tr1.getTgl()) * csp1i2, a01 = -(kx0 * kx1 + ky0 * ky1 + kz0 * kz1); + float b0 = dx * kx0 + dy * ky0 + dz * kz0, b1 = -(dx * kx1 + dy * ky1 + dz * kz1); + float det = a00 * a11 - a01 * a01, det0 = b0 * a11 - b1 * a01, det1 = a00 * b1 - a01 * b0; + if (o2::gpu::GPUCommonMath::Sqrt(det) > o2::constants::math::Almost0) { + auto detI = 1. / det; + auto t0 = det0 * detI; + auto t1 = det1 * detI; + float addx0 = kx0 * t0, addy0 = ky0 * t0, addx1 = kx1 * t1, addy1 = ky1 * t1; + dx += addx1 - addx0; // recalculate XY distance at DCA + dy += addy1 - addy0; + if (dx * dx + dy * dy > maxDistXY * maxDistXY) { + return nDCA; + } + xDCA[0] = (trax0.xC + addx0 + trax1.xC + addx1) * 0.5; + yDCA[0] = (trax0.yC + addy0 + trax1.yC + addy1) * 0.5; + nDCA = 1; + } + return nDCA; + } + + template + GPUd() int circleLineCrossInfo(const TrackAuxPar& trax0, const T& tr0, + const TrackAuxPar& trax1, const T& tr1, float maxDistXY = MaxDistXYDef) + { + /// closest approach of line and circle + /// TrackParam propagation can be parameterized in lab in a form + /// xLab(t) = (x*cosAlp - y*sinAlp) + t*(cosAlp - sinAlp* snp/csp) = xLab0 + t*(cosAlp - sinAlp* snp/csp) + /// yLab(t) = (x*sinAlp + y*cosAlp) + t*(sinAlp + cosAlp* snp/csp) = yLab0 + t*(sinAlp + cosAlp* snp/csp) + /// zLab(t) = z + t * tgl / csp = zLab0 + t * tgl / csp + /// where t is the x-step in the track alpha-frame, xLab,yLab,zLab are reference track coordinates in lab + /// frame (filled by TrackAuxPar for straight line tracks). + /// + /// Therefore, for the parametric track equation in lab 3D we have (wrt tracking-X increment t) + /// xL(t) = xL + t Kx; Kx = (cosAlp - sinAlp* snp/csp) + /// yL(t) = yL + t Ky; Ky = (sinAlp + cosAlp* snp/csp) + /// zL(t) = zL + t Kz; Kz = tgl / csp + /// Note that Kx^2 + Ky^2 = 1 / csp^2 + + const auto& traxH = trax0.rC > trax1.rC ? trax0 : trax1; // circle (for the line rC is set to 0) + const auto& traxL = trax0.rC > trax1.rC ? trax1 : trax0; // line + const auto& trcL = trax0.rC > trax1.rC ? tr1 : tr0; // track of the line + + // solve quadratic equation of line crossing the circle + float dx = traxL.xC - traxH.xC; // X distance between the line lab reference and circle center + float dy = traxL.yC - traxH.yC; // Y... + // t^2(kx^2+ky^2) + 2t(dx*kx+dy*ky) + dx^2 + dy^2 - r^2 = 0 + auto cspi2 = 1. / trcL.getCsp2(); // 1 / csp^2 == kx^2 + ky^2 + auto cspi = o2::gpu::GPUCommonMath::Sqrt(cspi2); + auto tgp = trcL.getSnp() * cspi; + float kx = traxL.c - traxL.s * tgp; + float ky = traxL.s + traxL.c * tgp; + double dk = dx * kx + dy * ky; + double det = dk * dk - cspi2 * (dx * dx + dy * dy - traxH.rC * traxH.rC); + if (det > 0) { // 2 crossings + det = o2::gpu::GPUCommonMath::Sqrt(det); + float t0 = (-dk + det) * cspi2; + float t1 = (-dk - det) * cspi2; + xDCA[0] = traxL.xC + kx * t0; + yDCA[0] = traxL.yC + ky * t0; + xDCA[1] = traxL.xC + kx * t1; + yDCA[1] = traxL.yC + ky * t1; + nDCA = 2; + } else { + // there is no crossing, find the point of the closest approach on the line which is closest to the circle center + float t = -dk * cspi2; + float xL = traxL.xC + kx * t, yL = traxL.yC + ky * t; // point on the line, need to average with point on the circle + float dxc = xL - traxH.xC, dyc = yL - traxH.yC, dist = o2::gpu::GPUCommonMath::Sqrt(dxc * dxc + dyc * dyc); + if (dist - traxH.rC > maxDistXY) { // too large distance + return nDCA; + } + float drcf = traxH.rC / dist; // radius / distance to circle center + float xH = traxH.xC + dxc * drcf, yH = traxH.yC + dyc * drcf; + xDCA[0] = (xL + xH) * 0.5; + yDCA[0] = (yL + yH) * 0.5; + nDCA = 1; + } + return nDCA; + } + + template + GPUd() int set(const TrackAuxPar& trax0, const T& tr0, const TrackAuxPar& trax1, const T& tr1, float maxDistXY = MaxDistXYDef, bool isCollinear = false) + { + // calculate up to 2 crossings between 2 circles + nDCA = 0; + if (trax0.rC > o2::constants::math::Almost0 && trax1.rC > o2::constants::math::Almost0) { // both are not straight lines + nDCA = circlesCrossInfo(trax0, trax1, maxDistXY, isCollinear); + } else if (trax0.rC < o2::constants::math::Almost0 && trax1.rC < o2::constants::math::Almost0) { // both are straigt lines + nDCA = linesCrossInfo(trax0, tr0, trax1, tr1, maxDistXY); + } else { + nDCA = circleLineCrossInfo(trax0, tr0, trax1, tr1, maxDistXY); + } + // + return nDCA; + } + + GPUdDefault() CrossInfo() = default; + + template + GPUd() CrossInfo(const TrackAuxPar& trax0, const T& tr0, const TrackAuxPar& trax1, const T& tr1, float maxDistXY = MaxDistXYDef, bool isCollinear = false) + { + set(trax0, tr0, trax1, tr1, maxDistXY, isCollinear); + } + ClassDefNV(CrossInfo, 1); +}; + +} // namespace track +} // namespace o2 + +#endif diff --git a/DataFormats/Reconstruction/include/ReconstructionDataFormats/MatchInfoHMP.h b/DataFormats/Reconstruction/include/ReconstructionDataFormats/MatchInfoHMP.h index 2d614201c2220..e065b3d7dfe40 100644 --- a/DataFormats/Reconstruction/include/ReconstructionDataFormats/MatchInfoHMP.h +++ b/DataFormats/Reconstruction/include/ReconstructionDataFormats/MatchInfoHMP.h @@ -28,22 +28,43 @@ class MatchInfoHMP using GTrackID = o2::dataformats::GlobalTrackID; public: - MatchInfoHMP(int idxHMPClus, GTrackID idxTrack, float angle = 0, float q = 0, float size = 0, int idxPhotClus = 0) : mIdxHMPClus(idxHMPClus), mIdxTrack(idxTrack), mCkovAngle(angle), mMipCluQ(q), mMipCluSize(size), mIdxPhotClus(idxPhotClus){}; + MatchInfoHMP(int idxHMPClus, GTrackID idxTrack, float xmip = 0, float ymip = 0, float xtrk = 0, float ytrk = 0, float theta = 0, float phi = 0, float angle = 0, float size = 0, int idxPhotClus = 0, int hmpqn = 0, float hmpMom = 0) : mIdxHMPClus(idxHMPClus), mIdxTrack(idxTrack), mCkovAngle(angle), mMipX(xmip), mMipY(ymip), mTrkX(xtrk), mTrkY(ytrk), mTrkTheta(theta), mTrkPhi(phi), mMipCluSize(size), mIdxPhotClus(idxPhotClus), mHMPqn(hmpqn), mHmpMom(hmpMom) + { // Initialize the mPhotCharge array + for (int i = 0; i < 10; i++) { + mPhotCharge[i] = 0.0; + } + }; MatchInfoHMP() = default; - void setIdxHMPClus(int index) { mIdxHMPClus = index; } + void setIdxHMPClus(int ch, int idx) { mIdxHMPClus = ch * 1000000 + idx; } int getIdxHMPClus() const { return mIdxHMPClus; } void setIdxTrack(GTrackID index) { mIdxTrack = index; } + int getTrackIndex() const { return mIdxTrack.getIndex(); } + GTrackID getTrackRef() const { return mIdxTrack; } - int getTrackIndex() const { return mIdxTrack.getIndex(); } + void setMipX(float x) { mMipX = x; } + float getMipX() const { return mMipX; } - void setCkovAngle(float angle) { mCkovAngle = angle; } - float getCkovAngle() const { return mCkovAngle; } + void setMipY(float y) { mMipY = y; } + float getMipY() const { return mMipY; } - void setMipClusQ(float q) { mMipCluQ = q; } - float getMipClusQ() const { return mMipCluQ; } + void setTrkX(float x) { mTrkX = x; } + float getTrkX() const { return mTrkX; } + + void setTrkY(float y) { mTrkY = y; } + float getTrkY() const { return mTrkY; } + + void setHMPsignal(float angle) { mCkovAngle = angle; } + float getHMPsignal() const + { + if (mCkovAngle > 0) { + return mCkovAngle - (Int_t)mCkovAngle; + } else { + return mCkovAngle; + } + } void setMipClusSize(int size) { mMipCluSize = size; } int getMipClusSize() const { return mMipCluSize; } @@ -54,18 +75,71 @@ class MatchInfoHMP void setPhotIndex(int idx) { mIdxPhotClus = idx; } int getPhotIndex() const { return mIdxPhotClus; } + float getOccupancy() const { return (Int_t)mCkovAngle / 10.0; } + + void setHMPIDtrk(float x, float y, float th, float ph) + { + mTrkX = x; + mTrkY = y; + mTrkTheta = th; + mTrkPhi = ph; + } + + void getHMPIDtrk(float& x, float& y, float& th, float& ph) const + { + x = mTrkX; + y = mTrkY; + th = mTrkTheta; + ph = mTrkPhi; + } + + void setHMPIDmip(float x, float y, int q, int nph = 0) + { + mMipX = x; + mMipY = y; + mHMPqn = 1000000 * nph + q; + } + + void getHMPIDmip(float& x, float& y, int& q, int& nph) const + { + x = mMipX; + y = mMipY; + q = mHMPqn % 1000000; + nph = mHMPqn / 1000000; + } + + void setHmpMom(float p) { mHmpMom = p; } + float getHmpMom() const { return mHmpMom; } + + void setPhotCharge(float* chargeArray) + { + for (int i = 0; i < 10; i++) { + mPhotCharge[i] = chargeArray[i]; + } + } + + const float* getPhotCharge() const { return mPhotCharge; } + void print() const; private: int mIdxHMPClus; // Idx for HMP cluster GTrackID mIdxTrack; // Idx for track + float mMipX; // local x coordinate of macthed cluster + float mMipY; // local y coordinate of macthed cluster + float mTrkX; // local x coordinate of extrapolated track intersection point + float mTrkY; // local y coordinate of extrapolated track intersection point + float mTrkTheta; // theta track + float mTrkPhi; // phi track float mCkovAngle; // emission angle value - float mMipCluQ = 0.0; // MIP cluster charge int mMipCluSize = 0.0; // MIP cluster size int mNPhots = 0.0; // number of candidate photo-electrons int mIdxPhotClus; // index of the first photo + int mHMPqn; // 1000000*number of photon clusters + QDC + float mHmpMom; // track momentum at HMPID chambers + float mPhotCharge[10] = {}; - ClassDefNV(MatchInfoHMP, 1); + ClassDefNV(MatchInfoHMP, 2); }; } // namespace dataformats } // namespace o2 diff --git a/DataFormats/Reconstruction/include/ReconstructionDataFormats/MatchInfoTOF.h b/DataFormats/Reconstruction/include/ReconstructionDataFormats/MatchInfoTOF.h index 9f6b2f00bb37c..7bcfd7af0911a 100644 --- a/DataFormats/Reconstruction/include/ReconstructionDataFormats/MatchInfoTOF.h +++ b/DataFormats/Reconstruction/include/ReconstructionDataFormats/MatchInfoTOF.h @@ -28,7 +28,7 @@ class MatchInfoTOF using GTrackID = o2::dataformats::GlobalTrackID; public: - MatchInfoTOF(int idLocal, int idxTOFCl, double time, float chi2, o2::track::TrackLTIntegral trkIntLT, GTrackID idxTrack, float dt = 0, float z = 0, float dx = 0, float dz = 0) : mIdLocal(idLocal), mIdxTOFCl(idxTOFCl), mSignal(time), mChi2(chi2), mIntLT(trkIntLT), mIdxTrack(idxTrack), mDeltaT(dt), mZatTOF(z), mDXatTOF(dx), mDZatTOF(dz){}; + MatchInfoTOF(int idLocal, int idxTOFCl, double time, float chi2, o2::track::TrackLTIntegral trkIntLT, GTrackID idxTrack, float dt = 0, float z = 0, float dx = 0, float dz = 0, float dy = 0, float geanttime = 0.0, double t0 = 0.0) : mIdLocal(idLocal), mIdxTOFCl(idxTOFCl), mSignal(time), mChi2(chi2), mIntLT(trkIntLT), mIdxTrack(idxTrack), mDeltaT(dt), mZatTOF(z), mDXatTOF(dx), mDZatTOF(dz), mDYatTOF(dy), mTgeant(geanttime), mT0true(t0){}; MatchInfoTOF() = default; void setIdxTOFCl(int index) { mIdxTOFCl = index; } void setIdxTrack(GTrackID index) { mIdxTrack = index; } @@ -38,9 +38,15 @@ class MatchInfoTOF int getTOFClIndex() const { return mIdxTOFCl; } int getTrackIndex() const { return mIdxTrack.getIndex(); } - void setChi2(int chi2) { mChi2 = chi2; } + void setChi2(float chi2) { mChi2 = chi2; } float getChi2() const { return mChi2; } + void setHitPatternUpDown(bool v) { mHitUpDown = v; } + bool getHitPatternUpDown() const { return mHitUpDown; } + + void setHitPatternLeftRight(bool v) { mHitLeftRight = v; } + bool getHitPatternLeftRight() const { return mHitLeftRight; } + o2::track::TrackLTIntegral& getLTIntegralOut() { return mIntLT; } const o2::track::TrackLTIntegral& getLTIntegralOut() const { return mIntLT; } void print() const; @@ -53,11 +59,41 @@ class MatchInfoTOF float getDZatTOF() const { return mDZatTOF; } void setDXatTOF(float val) { mDXatTOF = val; } float getDXatTOF() const { return mDXatTOF; } + void setDYatTOF(float val) { mDYatTOF = val; } + float getDYatTOF() const { return mDYatTOF; } void setSignal(double time) { mSignal = time; } double getSignal() const { return mSignal; } int getIdLocal() const { return mIdLocal; } + float getVz() const { return mVz; } + void setVz(float val) { mVz = val; } + int getChannel() const { return mChannel; } + void setChannel(int val) { mChannel = val; } + float getTgeant() const { return mTgeant; } + void setTgeant(float val) { mTgeant = val; } + double getT0true() const { return mT0true; } + void setT0true(double val) { mT0true = val; } + + enum QualityFlags { isMultiHitX = 0x1 << 0, + isMultiHitZ = 0x1 << 1, + badDy = 0x1 << 2, + isMultiStrip = 0x1 << 3, + isNotInPad = 0x1 << 4, + chiGT3 = 0x1 << 5, + chiGT5 = 0x1 << 6, + hasT0sameBC = 0x1 << 7, + hasT0_1BCbefore = 0x1 << 8, + hasT0_2BCbefore = 0x1 << 9 }; + + void setFT0Best(double val, float res = 200.) + { + mFT0Best = val; + mFT0BestRes = res; + } + double getFT0Best() const { return mFT0Best; } + float getFT0BestRes() const { return mFT0BestRes; } + private: int mIdLocal; // track id in sector of the pair track-TOFcluster float mChi2; // chi2 of the pair track-TOFcluster @@ -67,10 +103,21 @@ class MatchInfoTOF float mZatTOF = 0.0; ///< Z position at TOF float mDXatTOF = 0.0; ///< DX position at TOF float mDZatTOF = 0.0; ///< DZ position at TOF + float mDYatTOF = 0.0; ///< DY position at TOF float mDeltaT = 0.0; ///< tTOF - TPC (microsec) double mSignal = 0.0; ///< TOF time in ps + float mVz = 0.0; ///< Vz from TOF match + int mChannel = -1; ///< channel + // Hit pattern information + bool mHitUpDown = false; ///< hit pattern in TOF up-down + bool mHitLeftRight = false; ///< hit pattern in TOF left-right + float mTgeant = 0.0; ///< geant time in MC + double mT0true = 0.0; ///< t0true + + double mFT0Best = 0.0; //< best info for collision time + float mFT0BestRes = 200.0; //< resolution (in ps) of the best info for collision time - ClassDefNV(MatchInfoTOF, 5); + ClassDefNV(MatchInfoTOF, 9); }; } // namespace dataformats } // namespace o2 diff --git a/DataFormats/Reconstruction/include/ReconstructionDataFormats/MatchInfoTOFReco.h b/DataFormats/Reconstruction/include/ReconstructionDataFormats/MatchInfoTOFReco.h index 64e29ed4f3d04..f1b555301bf80 100644 --- a/DataFormats/Reconstruction/include/ReconstructionDataFormats/MatchInfoTOFReco.h +++ b/DataFormats/Reconstruction/include/ReconstructionDataFormats/MatchInfoTOFReco.h @@ -35,16 +35,34 @@ class MatchInfoTOFReco : public MatchInfoTOF ITSTPCTRD, SIZEALL }; - MatchInfoTOFReco(int idLocal, int idxTOFCl, double time, float chi2, o2::track::TrackLTIntegral trkIntLT, GTrackID idxTrack, TrackType trkType, float dt = 0, float z = 0, float dx = 0, float dz = 0) : MatchInfoTOF(idLocal, idxTOFCl, time, chi2, trkIntLT, idxTrack, dt, z, dx, dz), mTrackType(trkType){}; + MatchInfoTOFReco(int idLocal, int idxTOFCl, double time, float chi2, o2::track::TrackLTIntegral trkIntLT, GTrackID idxTrack, TrackType trkType, float dt = 0, float z = 0, float dx = 0, float dz = 0, float dy = 0) : MatchInfoTOF(idLocal, idxTOFCl, time, chi2, trkIntLT, idxTrack, dt, z, dx, dz, dy), mTrackType(trkType){}; MatchInfoTOFReco() = default; + void setFakeMatch() { mFakeMC = true; } + void resetFakeMatch() { mFakeMC = false; } + bool isFake() const { return mFakeMC; } + float pt() const { return mPt; } + void setPt(float pt) { mPt = pt; } + + void setResX(float val) { mResX = val; } + void setResZ(float val) { mResZ = val; } + void setResT(float val) { mResT = val; } + float getResX() const { return mResX; } + float getResZ() const { return mResZ; } + float getResT() const { return mResT; } + void setTrackType(TrackType value) { mTrackType = value; } TrackType getTrackType() const { return mTrackType; } private: TrackType mTrackType; ///< track type (TPC, ITSTPC, TPCTRD, ITSTPCTRD) - ClassDefNV(MatchInfoTOFReco, 2); + bool mFakeMC = false; + float mPt = 0; + float mResX = 1; + float mResZ = 1; + float mResT = 1; + ClassDefNV(MatchInfoTOFReco, 5); }; } // namespace dataformats } // namespace o2 diff --git a/DataFormats/Reconstruction/include/ReconstructionDataFormats/PID.h b/DataFormats/Reconstruction/include/ReconstructionDataFormats/PID.h index f7124f12902d3..ce70e69aa6ddd 100644 --- a/DataFormats/Reconstruction/include/ReconstructionDataFormats/PID.h +++ b/DataFormats/Reconstruction/include/ReconstructionDataFormats/PID.h @@ -16,6 +16,10 @@ #ifndef ALICEO2_track_PID_H_ #define ALICEO2_track_PID_H_ +#ifndef GPUCA_GPUCODE_DEVICE +#include +#endif + #include "GPUCommonDef.h" #include "GPUCommonRtypes.h" #include "CommonConstants/PhysicsConstants.h" @@ -29,16 +33,19 @@ namespace o2cp = o2::constants::physics; namespace pid_constants // GPUs currently cannot have static constexpr array members { typedef uint8_t ID; -static constexpr ID NIDsTot = 17; +static constexpr ID NIDsTot = 19; + +#if !defined(GPUCA_GPUCODE_DEVICE) || defined(GPUCA_GPU_DEBUG_PRINT) GPUconstexpr() const char* sNames[NIDsTot + 1] = ///< defined particle names {"Electron", "Muon", "Pion", "Kaon", "Proton", "Deuteron", "Triton", "He3", "Alpha", - "Pion0", "Photon", "K0", "Lambda", "HyperTriton", "Hyperhydrog4", "XiMinus", "OmegaMinus", nullptr}; + "Pion0", "Photon", "K0", "Lambda", "HyperTriton", "Hyperhydrog4", "XiMinus", "OmegaMinus", "HyperHelium4", "HyperHelium5", nullptr}; +#endif GPUconstexpr() const float sMasses[NIDsTot] = ///< defined particle masses {o2cp::MassElectron, o2cp::MassMuon, o2cp::MassPionCharged, o2cp::MassKaonCharged, o2cp::MassProton, o2cp::MassDeuteron, o2cp::MassTriton, o2cp::MassHelium3, o2cp::MassAlpha, o2cp::MassPionNeutral, o2cp::MassPhoton, - o2cp::MassKaonNeutral, o2cp::MassLambda, o2cp::MassHyperTriton, o2cp::MassHyperhydrog4, o2cp::MassXiMinus, o2cp::MassOmegaMinus}; + o2cp::MassKaonNeutral, o2cp::MassLambda, o2cp::MassHyperTriton, o2cp::MassHyperhydrog4, o2cp::MassXiMinus, o2cp::MassOmegaMinus, o2cp::MassHyperHelium4, o2cp::MassHyperHelium5}; GPUconstexpr() const float sMasses2[NIDsTot] = ///< defined particle masses^2 {o2cp::MassElectron * o2cp::MassElectron, @@ -57,7 +64,9 @@ GPUconstexpr() const float sMasses2[NIDsTot] = ///< defined particle masses^2 o2cp::MassHyperTriton* o2cp::MassHyperTriton, o2cp::MassHyperhydrog4* o2cp::MassHyperhydrog4, o2cp::MassXiMinus* o2cp::MassXiMinus, - o2cp::MassOmegaMinus* o2cp::MassOmegaMinus}; + o2cp::MassOmegaMinus* o2cp::MassOmegaMinus, + o2cp::MassHyperHelium4* o2cp::MassHyperHelium4, + o2cp::MassHyperHelium5* o2cp::MassHyperHelium5}; GPUconstexpr() const float sMasses2Z[NIDsTot] = ///< defined particle masses / Z {o2cp::MassElectron, o2cp::MassMuon, @@ -66,12 +75,14 @@ GPUconstexpr() const float sMasses2Z[NIDsTot] = ///< defined particle masses / Z o2cp::MassTriton, o2cp::MassHelium3 / 2., o2cp::MassAlpha / 2., 0, 0, 0, 0, o2cp::MassHyperTriton, o2cp::MassHyperhydrog4, - o2cp::MassXiMinus, o2cp::MassOmegaMinus}; + o2cp::MassXiMinus, o2cp::MassOmegaMinus, + o2cp::MassHyperHelium4 / 2., o2cp::MassHyperHelium5 / 2.}; GPUconstexpr() const int sCharges[NIDsTot] = ///< defined particle charges {1, 1, 1, 1, 1, 1, 1, 2, 2, 0, 0, 0, 0, 1, 1, - 1, 1}; + 1, 1, + 2, 2}; } // namespace pid_constants class PID @@ -103,8 +114,10 @@ class PID static constexpr ID Hyperhydrog4 = 14; static constexpr ID XiMinus = 15; static constexpr ID OmegaMinus = 16; + static constexpr ID HyperHelium4 = 17; + static constexpr ID HyperHelium5 = 18; static constexpr ID FirstExt = PI0; - static constexpr ID LastExt = OmegaMinus; + static constexpr ID LastExt = HyperHelium5; static constexpr ID NIDsTot = pid_constants::NIDsTot; ///< total number of defined IDs static_assert(NIDsTot == LastExt + 1, "Incorrect NIDsTot, please update!"); @@ -112,7 +125,7 @@ class PID GPUd() PID(ID id) : mID(id) {} GPUd() PID(const char* name); GPUdDefault() PID(const PID& src) = default; - GPUdDefault() PID& operator=(const PID& src) = default; + GPUhdDefault() PID& operator=(const PID& src) = default; GPUd() ID getID() const { return mID; } GPUd() operator ID() const { return getID(); } @@ -126,7 +139,7 @@ class PID GPUd() static float getMass2(ID id) { return pid_constants::sMasses2[id]; } GPUd() static float getMass2Z(ID id) { return pid_constants::sMasses2Z[id]; } GPUd() static int getCharge(ID id) { return pid_constants::sCharges[id]; } -#ifndef GPUCA_GPUCODE_DEVICE +#if !defined(GPUCA_GPUCODE_DEVICE) || defined(GPUCA_GPU_DEBUG_PRINT) GPUd() const char* getName() const { return getName(mID); @@ -137,13 +150,13 @@ class PID private: ID mID = Pion; +#if !defined(GPUCA_GPUCODE_DEVICE) || defined(GPUCA_GPU_DEBUG_PRINT) // are 2 strings equal ? (trick from Giulio) GPUdi() static constexpr bool sameStr(char const* x, char const* y) { return !*x && !*y ? true : /* default */ (*x == *y && sameStr(x + 1, y + 1)); } -#ifndef GPUCA_GPUCODE_DEVICE GPUdi() static constexpr ID nameToID(char const* name, ID id) { return id > LastExt ? id : sameStr(name, pid_constants::sNames[id]) ? id : nameToID(name, id + 1); diff --git a/DataFormats/Reconstruction/include/ReconstructionDataFormats/PrimaryVertex.h b/DataFormats/Reconstruction/include/ReconstructionDataFormats/PrimaryVertex.h index ce470361034f5..5343d26ec5ce5 100644 --- a/DataFormats/Reconstruction/include/ReconstructionDataFormats/PrimaryVertex.h +++ b/DataFormats/Reconstruction/include/ReconstructionDataFormats/PrimaryVertex.h @@ -37,6 +37,11 @@ class PrimaryVertex : public Vertex> void setIR(const InteractionRecord& ir) { mIRMin = mIRMax = ir; } bool hasUniqueIR() const { return !mIRMin.isDummy() && (mIRMin == mIRMax); } + float getTMAD() const { return mTMAD; } + void setTMAD(float v) { mTMAD = v; } + float getZMAD() const { return mZMAD; } + void setZMAD(float v) { mZMAD = v; } + #ifndef GPUCA_ALIGPUCODE void print() const; std::string asString() const; @@ -45,8 +50,10 @@ class PrimaryVertex : public Vertex> protected: InteractionRecord mIRMin{}; ///< by default not assigned! InteractionRecord mIRMax{}; ///< by default not assigned! + float mTMAD = -1; ///< MAD estimator for Tsigma + float mZMAD = -1; ///< MAD estimator for Zsigma - ClassDefNV(PrimaryVertex, 1); + ClassDefNV(PrimaryVertex, 2); }; #ifndef GPUCA_ALIGPUCODE diff --git a/DataFormats/Reconstruction/include/ReconstructionDataFormats/PrimaryVertexExt.h b/DataFormats/Reconstruction/include/ReconstructionDataFormats/PrimaryVertexExt.h new file mode 100644 index 0000000000000..bf47ed03f3b39 --- /dev/null +++ b/DataFormats/Reconstruction/include/ReconstructionDataFormats/PrimaryVertexExt.h @@ -0,0 +1,64 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef ALICEO2_PRIMARYVERTEX_EXT_H +#define ALICEO2_PRIMARYVERTEX_EXT_H + +#include "ReconstructionDataFormats/PrimaryVertex.h" +#include "ReconstructionDataFormats/GlobalTrackID.h" + +namespace o2 +{ +namespace dataformats +{ + +// extended primary vertex info + +struct PrimaryVertexExt : public PrimaryVertex { + using PrimaryVertex::PrimaryVertex; + std::array nSrc{}; // N contributors for each source type + std::array nSrcA{}; // N associated and passing cuts for each source type + std::array nSrcAU{}; // N ambgous associated and passing cuts for each source type + double FT0Time = -1.; // time of closest FT0 trigger + float FT0A = -1; // amplitude of closest FT0 A side + float FT0C = -1; // amplitude of closest FT0 C side + int VtxID = -1; // original vtx ID + + int getNSrc(int i) const { return nSrc[i]; } + int getNSrcA(int i) const { return nSrcA[i]; } + int getNSrcAU(int i) const { return nSrcAU[i]; } + +#ifndef GPUCA_ALIGPUCODE + void print() const; + std::string asString() const; +#endif + + ClassDefNV(PrimaryVertexExt, 6); +}; + +#ifndef GPUCA_ALIGPUCODE +std::ostream& operator<<(std::ostream& os, const o2::dataformats::PrimaryVertexExt& v); +#endif + +} // namespace dataformats + +/// Defining PrimaryVertexExt explicitly as messageable +namespace framework +{ +template +struct is_messageable; +template <> +struct is_messageable : std::true_type { +}; +} // namespace framework + +} // namespace o2 +#endif diff --git a/DataFormats/Reconstruction/include/ReconstructionDataFormats/StrangeTrack.h b/DataFormats/Reconstruction/include/ReconstructionDataFormats/StrangeTrack.h new file mode 100644 index 0000000000000..12e376f10f3d8 --- /dev/null +++ b/DataFormats/Reconstruction/include/ReconstructionDataFormats/StrangeTrack.h @@ -0,0 +1,86 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file StrangeTrack.h +/// \brief +/// + +#ifndef _ALICEO2_STRANGETRACK_ +#define _ALICEO2_STRANGETRACK_ + +#include +#include "ReconstructionDataFormats/Track.h" + +namespace o2 +{ +namespace dataformats +{ + +enum kPartType { kStrkV0, + kStrkCascade, + kStrkThreeBody }; + +struct StrangeTrack { + kPartType mPartType; + o2::track::TrackParCovF mMother; + unsigned int mITSRef = -1; + unsigned int mDecayRef = -1; + std::array mDecayVtx; + std::array mDecayMom; + std::array mMasses; // V0: hypertriton and hyperhydrongen4, cascade: Xi and Omega. + unsigned int mClusterSizes = 0u; // same encoding used for the ITS track + float mMatchChi2; + float mTopoChi2; + + void setClusterSize(int l, int size) + { + if (l >= 8) { + return; + } + if (size > 15) { + size = 15; + } + mClusterSizes &= ~(0xf << (l * 4)); + mClusterSizes |= (size << (l * 4)); + } + + int getClusterSize(int l) const + { + if (l >= 7) { + return 0; + } + return (mClusterSizes >> (l * 4)) & 0xf; + } + + int getClusterSizes() const + { + return mClusterSizes; + } + + float getAverageClusterSize() const + { + int nClusters = 0; + int nClustersSize = 0; + for (int i = 0; i < 7; ++i) { + int size = getClusterSize(i); + if (size > 0) { + nClustersSize += size; + nClusters++; + } + } + return nClusters > 0 ? static_cast(nClustersSize) / nClusters : 0.f; + } +}; + +} // namespace dataformats +} // namespace o2 + +#endif // _ALICEO2_STRANGETRACK_ diff --git a/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackFwd.h b/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackFwd.h index 01c873496b5a6..50ed36d466d25 100644 --- a/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackFwd.h +++ b/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackFwd.h @@ -161,6 +161,7 @@ class TrackParCovFwd : public TrackParFwd void propagateToZquadratic(double zEnd, double zField); void propagateToZhelix(double zEnd, double zField); void propagateToZ(double zEnd, double zField); // Parameters: helix; errors: quadratic + void propagateToDCAhelix(double zField, const std::array& p, std::array& dca); // Add Multiple Coulomb Scattering effects void addMCSEffect(double x2X0); @@ -168,6 +169,12 @@ class TrackParCovFwd : public TrackParFwd // Kalman filter/fitting bool update(const std::array& p, const std::array& cov); + // Propagate fwd track to vertex including MCS effects + bool propagateToVtxhelixWithMCS(double z, const std::array& p, const std::array& cov, double field, double x_over_X0); + bool propagateToVtxlinearWithMCS(double z, const std::array& p, const std::array& cov, double x_over_X0); + + bool getCovXYZPxPyPzGlo(std::array& cv) const; + private: /// Covariance matrix of track parameters, ordered as follows: 
   ///                                     
diff --git a/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackHMP.h b/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackHMP.h
new file mode 100644
index 0000000000000..8bef3a40b603a
--- /dev/null
+++ b/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackHMP.h
@@ -0,0 +1,54 @@
+// Copyright 2020-2022 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef ALICEO2_TRACKHMP_H
+#define ALICEO2_TRACKHMP_H
+
+#include "TMath.h"
+#include "TVector2.h"
+
+#include "ReconstructionDataFormats/TrackParametrizationWithError.h"
+#include "ReconstructionDataFormats/TrackParametrization.h"
+#include "ReconstructionDataFormats/Track.h"
+#include "ReconstructionDataFormats/TrackLTIntegral.h"
+#include "ReconstructionDataFormats/GlobalTrackID.h"
+
+// #include "HMPIDBase/Param.h"                 // for param
+
+namespace o2
+{
+namespace dataformats
+{
+
+class TrackHMP : public o2::track::TrackParCov
+{
+ public:
+  TrackHMP();
+  TrackHMP(const TrackHMP& t) = default;
+  TrackHMP(const o2::track::TrackParCov& t) : o2::track::TrackParCov{t} {};
+  //  TrackHMP(const o2::track::TrackParCov& t);
+  TrackHMP& operator=(const o2::track::TrackParCov& t);
+  ~TrackHMP() = default;
+
+  Bool_t intersect(Double_t pnt[3], Double_t norm[3], double bz) const; //
+  void propagate(Double_t len, std::array& x, std::array& p, double bz) const;
+
+  // protected:
+  //    Bool_t   Update(const AliCluster */*c*/, Double_t /*chi2*/, Int_t /*idx*/) {return 0;}
+  //    Double_t GetPredictedChi2(const AliCluster */*c*/) const {return 0.;}
+ private:
+  ClassDef(TrackHMP, 1) // HMPID reconstructed tracks
+};
+
+} // namespace dataformats
+} // namespace o2
+
+#endif
diff --git a/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackLTIntegral.h b/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackLTIntegral.h
index 95d17d8b28d5b..e799804805972 100644
--- a/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackLTIntegral.h
+++ b/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackLTIntegral.h
@@ -51,7 +51,19 @@ class TrackLTIntegral
     }
   }
 
-  GPUd() void addStep(float dL, float p2Inv);
+  GPUd() void clearFast()
+  {
+    mL = 0.f;
+    mX2X0 = 0.f;
+    mXRho = 0.f;
+    if (!isTimeNotNeeded()) {
+      for (int i = getNTOFs(); i--;) {
+        mT[i] = 0.f;
+      }
+    }
+  }
+
+  GPUd() void addStep(float dL, float q2p2);
   GPUd() void addX2X0(float d) { mX2X0 += d; }
   GPUd() void addXRho(float d) { mXRho += d; }
 
diff --git a/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackParametrization.h b/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackParametrization.h
index eff61cf48b6d7..1d6c4d9f0e4ea 100644
--- a/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackParametrization.h
+++ b/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackParametrization.h
@@ -29,7 +29,6 @@
 #include "GPUCommonDef.h"
 #include "GPUCommonRtypes.h"
 #include "GPUCommonMath.h"
-#include "GPUCommonArray.h"
 #include "GPUROOTCartesianFwd.h"
 
 #ifndef GPUCA_GPUCODE_DEVICE
@@ -39,9 +38,10 @@
 #include 
 #include 
 #include 
+#include 
 #endif
 
-#ifndef GPUCA_ALIGPUCODE //Used only by functions that are hidden on the GPU
+#ifndef GPUCA_ALIGPUCODE // Used only by functions that are hidden on the GPU
 #include "ReconstructionDataFormats/BaseCluster.h"
 #include 
 #endif
@@ -115,6 +115,15 @@ GPUconstexpr() int CovarMap[kNParams][kNParams] = {{0, 1, 3, 6, 10},
 GPUconstexpr() int DiagMap[kNParams] = {0, 2, 5, 9, 14};
 
 constexpr float HugeF = o2::constants::math::VeryBig;
+constexpr float MaxPT = 100000.;                  // do not allow pTs exceeding this value (to avoid NANs)
+constexpr float MinPTInv = 1. / MaxPT;            // do not allow q/pTs less this value (to avoid NANs)
+constexpr float ELoss2EKinThreshInv = 1. / 0.025; // do not allow E.Loss correction step with dE/Ekin above the inverse of this value
+constexpr int MaxELossIter = 50;                  // max number of iteration for the ELoss to account for BB dependence on beta*gamma
+constexpr float DefaultDCA = 999.f;               // default DCA value
+constexpr float DefaultDCACov = 999.f;            // default DCA cov value
+
+// uncomment this to enable correction for BB dependence on beta*gamma via BB derivative
+// #define _BB_NONCONST_CORR_
 
 template 
 class TrackParametrization
@@ -122,9 +131,9 @@ class TrackParametrization
 
  public:
   using value_t = value_T;
-  using dim2_t = gpu::gpustd::array;
-  using dim3_t = gpu::gpustd::array;
-  using params_t = gpu::gpustd::array;
+  using dim2_t = std::array;
+  using dim3_t = std::array;
+  using params_t = std::array;
 
   struct yzerr_t { // 2 measurement with error
     dim2_t yz;
@@ -140,8 +149,8 @@ class TrackParametrization
   GPUd() TrackParametrization(const dim3_t& xyz, const dim3_t& pxpypz, int charge, bool sectorAlpha = true, const PID pid = PID::Pion);
   GPUdDefault() TrackParametrization(const TrackParametrization&) = default;
   GPUdDefault() TrackParametrization(TrackParametrization&&) = default;
-  GPUdDefault() TrackParametrization& operator=(const TrackParametrization& src) = default;
-  GPUdDefault() TrackParametrization& operator=(TrackParametrization&& src) = default;
+  GPUhdDefault() TrackParametrization& operator=(const TrackParametrization& src) = default;
+  GPUhdDefault() TrackParametrization& operator=(TrackParametrization&& src) = default;
   GPUdDefault() ~TrackParametrization() = default;
 
   GPUd() void set(value_t x, value_t alpha, const params_t& par, int charge = 1, const PID pid = PID::Pion);
@@ -154,11 +163,11 @@ class TrackParametrization
   GPUd() value_t getZ() const;
   GPUd() value_t getSnp() const;
   GPUd() value_t getTgl() const;
-  GPUd() value_t getQ2Pt() const;
+  GPUhd() value_t getQ2Pt() const;
   GPUd() value_t getCharge2Pt() const;
   GPUd() int getAbsCharge() const;
   GPUd() PID getPID() const;
-  GPUd() void setPID(const PID pid);
+  GPUd() void setPID(const PID pid, bool passCharge = false);
 
   /// calculate cos^2 and cos of track direction in rphi-tracking
   GPUd() value_t getCsp2() const;
@@ -185,35 +194,52 @@ class TrackParametrization
   GPUd() value_t getPhi() const;
   GPUd() value_t getPhiPos() const;
 
+  GPUd() value_t getQ2P2() const;
   GPUd() value_t getPtInv() const;
   GPUd() value_t getP2Inv() const;
   GPUd() value_t getP2() const;
   GPUd() value_t getPInv() const;
   GPUd() value_t getP() const;
   GPUd() value_t getPt() const;
+  GPUd() value_t getE2() const;
+  GPUd() value_t getE() const;
+  GPUdi() static value_t getdEdxBB(value_t betagamma) { return BetheBlochSolid(betagamma); }
+  GPUdi() static value_t getdEdxBBOpt(value_t betagamma) { return BetheBlochSolidOpt(betagamma); }
+  GPUdi() static value_t getBetheBlochSolidDerivativeApprox(value_T dedx, value_T bg) { return BetheBlochSolidDerivative(dedx, bg); }
 
   GPUd() value_t getTheta() const;
   GPUd() value_t getEta() const;
   GPUd() math_utils::Point3D getXYZGlo() const;
   GPUd() void getXYZGlo(dim3_t& xyz) const;
   GPUd() bool getPxPyPzGlo(dim3_t& pxyz) const;
-  GPUd() bool getPosDirGlo(gpu::gpustd::array& posdirp) const;
+  GPUd() bool getPosDirGlo(std::array& posdirp) const;
 
   // methods for track params estimate at other point
   GPUd() bool getYZAt(value_t xk, value_t b, value_t& y, value_t& z) const;
   GPUd() value_t getZAt(value_t xk, value_t b) const;
   GPUd() value_t getYAt(value_t xk, value_t b) const;
+  GPUd() value_t getSnpAt(value_t xk, value_t b) const;
+  GPUd() value_t getSnpAt(value_t alpha, value_t xk, value_t b) const;
+  GPUd() value_t getPhiAt(value_t xk, value_t b) const;
+  GPUd() value_t getPhiPosAt(value_t xk, value_t b) const;
+  GPUd() value_t getDCAYtoMV(value_t b, value_t xmv = 0.f, value_t ymv = 0.f, value_t zmv = 0.f) const;
+  GPUd() value_t getDCAZtoMV(value_t b, value_t xmv = 0.f, value_t ymv = 0.f, value_t zmv = 0.f) const;
   GPUd() math_utils::Point3D getXYZGloAt(value_t xk, value_t b, bool& ok) const;
 
   // parameters manipulation
-  GPUd() bool correctForELoss(value_t xrho, bool anglecorr = false, value_t dedx = kCalcdEdxAuto);
+  GPUd() bool correctForELoss(value_t xrho, bool anglecorr = false);
   GPUd() bool rotateParam(value_t alpha);
+  GPUd() bool rotateParam(value_t& alpha, value_t& ca, value_t& sa);
   GPUd() bool propagateParamTo(value_t xk, value_t b);
   GPUd() bool propagateParamTo(value_t xk, const dim3_t& b);
-
-  GPUd() bool propagateParamToDCA(const math_utils::Point3D& vtx, value_t b, dim2_t* dca = nullptr, value_t maxD = 999.f);
-
   GPUd() void invertParam();
+  GPUd() bool propagateParamToDCA(const math_utils::Point3D& vtx, value_t b, dim2_t* dca = nullptr, value_t maxD = 999.f);
+  // aliases
+  GPUd() bool rotate(value_t alpha) { return rotateParam(alpha); }
+  GPUd() bool propagateTo(value_t xk, value_t b) { return propagateParamTo(xk, b); }
+  GPUd() bool propagateTo(value_t xk, const dim3_t& b) { return propagateParamTo(xk, b); }
+  GPUd() void invert() { invertParam(); }
+  GPUd() bool propagateToDCA(const math_utils::Point3D& vtx, value_t b, dim2_t* dca = nullptr, value_t maxD = 999.f) { return propagateParamToDCA(vtx, b, dca, maxD); }
 
   GPUd() bool isValid() const;
   GPUd() void invalidate();
@@ -222,8 +248,12 @@ class TrackParametrization
   GPUhd() void setUserField(uint16_t v);
 
   GPUd() void printParam() const;
+  GPUd() void printParamHexadecimal();
 #ifndef GPUCA_ALIGPUCODE
   std::string asString() const;
+  std::string asStringHexadecimal();
+  size_t hash() const { return hash(getX(), getAlpha(), getY(), getZ(), getSnp(), getTgl(), getQ2Pt()); }
+  static size_t hash(float x, float alp, float y, float z, float snp, float tgl, float q2pt);
 #endif
 
   GPUd() void updateParam(value_t delta, int i);
@@ -251,6 +281,7 @@ GPUdi() TrackParametrization::TrackParametrization(value_t x, value_t a
   : mX{x}, mAlpha{alpha}, mAbsCharge{char(gpu::CAMath::Abs(charge))}, mPID{pid}
 {
   // explicit constructor
+  math_utils::detail::bringToPMPi(mAlpha);
   for (int i = 0; i < kNParams; i++) {
     mP[i] = par[i];
   }
@@ -269,6 +300,7 @@ GPUdi() void TrackParametrization::set(value_t x, value_t alpha, const
 {
   mX = x;
   mAlpha = alpha;
+  math_utils::detail::bringToPMPi(mAlpha);
   mAbsCharge = char(gpu::CAMath::Abs(charge));
   for (int i = 0; i < kNParams; i++) {
     mP[i] = par[i];
@@ -334,7 +366,7 @@ GPUdi() auto TrackParametrization::getTgl() const -> value_t
 
 //____________________________________________________________
 template 
-GPUdi() auto TrackParametrization::getQ2Pt() const -> value_t
+GPUhdi() auto TrackParametrization::getQ2Pt() const -> value_t
 {
   return mP[kQ2Pt];
 }
@@ -362,10 +394,12 @@ GPUdi() PID TrackParametrization::getPID() const
 
 //____________________________________________________________
 template 
-GPUdi() void TrackParametrization::setPID(const PID pid)
+GPUdi() void TrackParametrization::setPID(const PID pid, bool passCharge)
 {
   mPID = pid;
-  //  setAbsCharge(pid.getCharge()); // If needed, user should change the charge via corr. setter
+  if (passCharge) {
+    setAbsCharge(pid.getCharge()); // If needed, user should change the charge via corr. setter
+  }
 }
 
 //____________________________________________________________
@@ -402,6 +436,7 @@ template 
 GPUdi() void TrackParametrization::setAlpha(value_t v)
 {
   mAlpha = v;
+  math_utils::detail::bringToPMPi(mAlpha);
 }
 
 //____________________________________________________________
@@ -530,12 +565,27 @@ GPUdi() auto TrackParametrization::getPhiPos() const -> value_t
   return phi;
 }
 
+//____________________________________________________________
+template 
+GPUdi() auto TrackParametrization::getQ2P2() const -> value_t
+{
+  // return the (q/p)^2
+  value_t q2pt2 = mP[kQ2Pt] * mP[kQ2Pt];
+  if (q2pt2 < MinPTInv * MinPTInv) {
+    q2pt2 = MinPTInv * MinPTInv;
+  }
+  return q2pt2 / (1.f + getTgl() * getTgl());
+}
+
 //____________________________________________________________
 template 
 GPUdi() auto TrackParametrization::getPtInv() const -> value_t
 {
   // return the inverted track pT
-  const value_t ptInv = gpu::CAMath::Abs(mP[kQ2Pt]);
+  value_t ptInv = gpu::CAMath::Abs(mP[kQ2Pt]);
+  if (ptInv < MinPTInv) {
+    ptInv = MinPTInv;
+  }
   return (mAbsCharge > 1) ? ptInv / mAbsCharge : ptInv;
 }
 
@@ -544,8 +594,8 @@ template 
 GPUdi() auto TrackParametrization::getP2Inv() const -> value_t
 {
   // return the inverted track momentum^2
-  const value_t p2 = mP[kQ2Pt] * mP[kQ2Pt] / (1.f + getTgl() * getTgl());
-  return (mAbsCharge > 1) ? p2 * mAbsCharge * mAbsCharge : p2;
+  value_t p2 = getPtInv();
+  return p2 * p2 / (1.f + getTgl() * getTgl());
 }
 
 //____________________________________________________________
@@ -553,17 +603,15 @@ template 
 GPUdi() auto TrackParametrization::getP2() const -> value_t
 {
   // return the track momentum^2
-  const value_t p2inv = getP2Inv();
-  return (p2inv > o2::constants::math::Almost0) ? 1.f / p2inv : o2::constants::math::VeryBig;
+  return 1.f / getP2Inv(); // getP2Inv is protected against being 0, full charge accounted
 }
 
 //____________________________________________________________
 template 
 GPUdi() auto TrackParametrization::getPInv() const -> value_t
 {
-  // return the inverted track momentum^2
-  const value_t pInv = gpu::CAMath::Abs(mP[kQ2Pt]) / gpu::CAMath::Sqrt(1.f + getTgl() * getTgl());
-  return (mAbsCharge > 1) ? pInv / mAbsCharge : pInv;
+  // return the inverted track momentum
+  return getPtInv() / gpu::CAMath::Sqrt(1.f + getTgl() * getTgl()); // getPtInv() is protected against being 0, full charge accounted
 }
 
 //____________________________________________________________
@@ -571,8 +619,23 @@ template 
 GPUdi() auto TrackParametrization::getP() const -> value_t
 {
   // return the track momentum
-  const value_t pInv = getPInv();
-  return (pInv > o2::constants::math::Almost0) ? 1.f / pInv : o2::constants::math::VeryBig;
+  return 1.f / getPInv(); // getPInv is already protected against being 0
+}
+
+//____________________________________________________________
+template 
+GPUdi() auto TrackParametrization::getE2() const -> value_t
+{
+  // return the track energy^2
+  return getP2() + getPID().getMass2();
+}
+
+//____________________________________________________________
+template 
+GPUdi() auto TrackParametrization::getE() const -> value_t
+{
+  // return the track energy
+  return gpu::CAMath::Sqrt(getE2());
 }
 
 //____________________________________________________________
@@ -580,11 +643,7 @@ template 
 GPUdi() auto TrackParametrization::getPt() const -> value_t
 {
   // return the track transverse momentum
-  value_t ptI = gpu::CAMath::Abs(mP[kQ2Pt]);
-  if (mAbsCharge > 1) {
-    ptI /= mAbsCharge;
-  }
-  return (ptI > o2::constants::math::Almost0) ? 1.f / ptI : o2::constants::math::VeryBig;
+  return 1.f / getPtInv(); // getPtInv is already protected against being 0
 }
 
 //____________________________________________________________
@@ -610,7 +669,7 @@ GPUdi() auto TrackParametrization::getXYZGlo() const -> math_utils::Poi
 #else // mockup on GPU without ROOT
   float sina, cosa;
   gpu::CAMath::SinCos(getAlpha(), sina, cosa);
-  return math_utils::Point3D(cosa * getX() + sina * getY(), cosa * getY() - sina * getX(), getZ());
+  return math_utils::Point3D(cosa * getX() - sina * getY(), cosa * getY() + sina * getX(), getZ());
 #endif
 }
 
@@ -640,7 +699,7 @@ GPUdi() auto TrackParametrization::getXYZGloAt(value_t xk, value_t b, b
 #else // mockup on GPU without ROOT
     float sina, cosa;
     gpu::CAMath::SinCos(getAlpha(), sina, cosa);
-    return math_utils::Point3D(cosa * xk + sina * y, cosa * y - sina * xk, z);
+    return math_utils::Point3D(cosa * xk - sina * y, cosa * y + sina * xk, z);
 #endif
   } else {
     return math_utils::Point3D();
@@ -694,7 +753,28 @@ GPUdi() void TrackParametrization::updateParams(const value_t* delta)
   for (int i = kNParams; i--;) {
     mP[i] += delta[i];
   }
+  // make sure that snp is in the valid range
+  if (mP[kSnp] > constants::math::Almost1) {
+    mP[kSnp] = constants::math::Almost1;
+  } else if (mP[kSnp] < -constants::math::Almost1) {
+    mP[kSnp] = -constants::math::Almost1;
+  }
+}
+
+#ifndef GPUCA_ALIGPUCODE
+template 
+size_t TrackParametrization::hash(float x, float alp, float y, float z, float snp, float tgl, float q2pt)
+{
+  size_t h = std::hash{}(o2::math_utils::detail::truncateFloatFraction(x, 0xFFFFFFF0));
+  h ^= std::hash{}(o2::math_utils::detail::truncateFloatFraction(alp, 0xFFFFFFF0)) << 1;
+  h ^= std::hash{}(o2::math_utils::detail::truncateFloatFraction(y, 0xFFFFFFF0)) << 1;
+  h ^= std::hash{}(o2::math_utils::detail::truncateFloatFraction(z, 0xFFFFFFF0)) << 1;
+  h ^= std::hash{}(o2::math_utils::detail::truncateFloatFraction(snp, 0xFFFFFF00)) << 1;
+  h ^= std::hash{}(o2::math_utils::detail::truncateFloatFraction(tgl, 0xFFFFFF00)) << 1;
+  h ^= std::hash{}(o2::math_utils::detail::truncateFloatFraction(q2pt, 0xFFFFFC00)) << 1;
+  return h;
 }
+#endif
 
 } // namespace track
 } // namespace o2
diff --git a/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackParametrizationWithError.h b/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackParametrizationWithError.h
index f60e03feeda36..0fc01e6db61a2 100644
--- a/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackParametrizationWithError.h
+++ b/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackParametrizationWithError.h
@@ -18,6 +18,7 @@
 #define INCLUDE_RECONSTRUCTIONDATAFORMATS_TRACKPARAMETRIZATIONWITHERROR_H_
 
 #include "ReconstructionDataFormats/TrackParametrization.h"
+#include 
 
 namespace o2
 {
@@ -37,26 +38,26 @@ class TrackParametrizationWithError : public TrackParametrization
   static_assert(std::is_floating_point_v);
 #endif
 
-  using covMat_t = gpu::gpustd::array;
-  using MatrixDSym5 = ROOT::Math::SMatrix>;
-  using MatrixD5 = ROOT::Math::SMatrix>;
+  using covMat_t = std::array;
+  using MatrixDSym5 = o2::math_utils::SMatrix>;
+  using MatrixD5 = o2::math_utils::SMatrix>;
 
-  GPUd() TrackParametrizationWithError();
+  GPUhd() TrackParametrizationWithError();
   GPUd() TrackParametrizationWithError(value_t x, value_t alpha, const params_t& par, const covMat_t& cov, int charge = 1, const PID pid = PID::Pion);
   GPUd() TrackParametrizationWithError(const dim3_t& xyz, const dim3_t& pxpypz,
-                                       const gpu::gpustd::array& cv, int sign, bool sectorAlpha = true, const PID pid = PID::Pion);
+                                       const std::array& cv, int sign, bool sectorAlpha = true, const PID pid = PID::Pion);
 
-  GPUdDefault() TrackParametrizationWithError(const TrackParametrizationWithError& src) = default;
+  GPUhdDefault() TrackParametrizationWithError(const TrackParametrizationWithError& src) = default;
   GPUdDefault() TrackParametrizationWithError(TrackParametrizationWithError&& src) = default;
-  GPUdDefault() TrackParametrizationWithError& operator=(const TrackParametrizationWithError& src) = default;
-  GPUdDefault() TrackParametrizationWithError& operator=(TrackParametrizationWithError&& src) = default;
+  GPUhdDefault() TrackParametrizationWithError& operator=(const TrackParametrizationWithError& src) = default;
+  GPUhdDefault() TrackParametrizationWithError& operator=(TrackParametrizationWithError&& src) = default;
   GPUdDefault() ~TrackParametrizationWithError() = default;
   using TrackParametrization::TrackParametrization;
 
   using TrackParametrization::set;
   GPUd() void set(value_t x, value_t alpha, const params_t& par, const covMat_t& cov, int charge = 1, const PID pid = PID::Pion);
   GPUd() void set(value_t x, value_t alpha, const value_t* par, const value_t* cov, int charge = 1, const PID pid = PID::Pion);
-  GPUd() void set(const dim3_t& xyz, const dim3_t& pxpypz, const gpu::gpustd::array& cv, int sign, bool sectorAlpha = true, const PID pid = PID::Pion);
+  GPUd() void set(const dim3_t& xyz, const dim3_t& pxpypz, const std::array& cv, int sign, bool sectorAlpha = true, const PID pid = PID::Pion);
   GPUd() const covMat_t& getCov() const;
   GPUd() value_t getSigmaY2() const;
   GPUd() value_t getSigmaZY() const;
@@ -76,31 +77,43 @@ class TrackParametrizationWithError : public TrackParametrization
   GPUd() value_t getCovarElem(int i, int j) const;
   GPUd() value_t getDiagError2(int i) const;
 
-  GPUd() bool getCovXYZPxPyPzGlo(gpu::gpustd::array& c) const;
+  GPUd() bool getCovXYZPxPyPzGlo(std::array& c) const;
 
   GPUd() void print() const;
+  GPUd() void printHexadecimal();
 #ifndef GPUCA_ALIGPUCODE
   std::string asString() const;
+  std::string asStringHexadecimal();
 #endif
 
   // parameters + covmat manipulation
+  GPUd() bool testRotate(value_t alpha) const;
   GPUd() bool rotate(value_t alpha);
-  GPUd() bool propagateTo(value_t xk, value_t b);
+  GPUd() bool rotate(value_t alpha, TrackParametrization& linRef, value_t bz);
+  GPUd() bool propagateTo(value_t xk, value_t bz);
+  GPUd() bool propagateTo(value_t xk, TrackParametrization& linRef, value_t bz);
+  GPUd() bool propagateTo(value_t xk, value_t bz, TrackParametrization* linRef) { return linRef ? propagateTo(xk, *linRef, bz) : propagateTo(xk, bz); }
   GPUd() bool propagateTo(value_t xk, const dim3_t& b);
-  GPUd() bool propagateToDCA(const o2::dataformats::VertexBase& vtx, value_t b, o2::dataformats::DCA* dca = nullptr, value_t maxD = 999.f);
+  GPUd() bool propagateTo(value_t xk, TrackParametrization& linRef, const dim3_t& b);
+  GPUd() bool propagateTo(value_t xk, const dim3_t& b, TrackParametrization* linRef) { return linRef ? propagateTo(xk, *linRef, b) : propagateTo(xk, b); }
+  GPUd() bool propagateToDCA(const o2::dataformats::VertexBase& vtx, value_t bz, o2::dataformats::DCA* dca = nullptr, value_t maxD = 999.f);
   GPUd() void invert();
-
   GPUd() value_t getPredictedChi2(const dim2_t& p, const dim3_t& cov) const;
+  GPUd() value_t getPredictedChi2Quiet(const dim2_t& p, const dim3_t& cov) const;
   GPUd() value_t getPredictedChi2(const value_t* p, const value_t* cov) const;
+  GPUd() value_t getPredictedChi2Quiet(const value_t* p, const value_t* cov) const;
 
   template 
   GPUd() value_t getPredictedChi2(const BaseCluster& p) const;
+  template 
+  GPUd() value_t getPredictedChi2Quiet(const BaseCluster& p) const;
 
-  void buildCombinedCovMatrix(const TrackParametrizationWithError& rhs, MatrixDSym5& cov) const;
-  value_t getPredictedChi2(const TrackParametrizationWithError& rhs, MatrixDSym5& covToSet) const;
-  value_t getPredictedChi2(const TrackParametrizationWithError& rhs) const;
-  bool update(const TrackParametrizationWithError& rhs, const MatrixDSym5& covInv);
-  bool update(const TrackParametrizationWithError& rhs);
+  GPUd() void buildCombinedCovMatrix(const TrackParametrizationWithError& rhs, MatrixDSym5& cov) const;
+  GPUd() value_t getPredictedChi2(const TrackParametrizationWithError& rhs, MatrixDSym5& covToSet) const;
+  GPUd() value_t getPredictedChi2(const TrackParametrizationWithError& rhs) const;
+  GPUd() value_t getPredictedChi2Quiet(const TrackParametrizationWithError& rhs) const;
+  GPUd() bool update(const TrackParametrizationWithError& rhs, const MatrixDSym5& covInv);
+  GPUd() bool update(const TrackParametrizationWithError& rhs);
 
   GPUd() bool update(const dim2_t& p, const dim3_t& cov);
   GPUd() bool update(const value_t* p, const value_t* cov);
@@ -109,10 +122,11 @@ class TrackParametrizationWithError : public TrackParametrization
   template 
   GPUd() bool update(const BaseCluster& p);
 
-  GPUd() bool correctForMaterial(value_t x2x0, value_t xrho, bool anglecorr = false, value_t dedx = kCalcdEdxAuto);
-
+  GPUd() bool correctForMaterial(value_t x2x0, value_t xrho, bool anglecorr = false);
+  GPUd() bool correctForMaterial(TrackParametrization& linRef, value_t x2x0, value_t xrho, bool anglecorr = false);
   GPUd() void resetCovariance(value_t s2 = 0);
   GPUd() void checkCovariance();
+  GPUd() void checkCorrelations();
   GPUd() void setCov(value_t v, size_t i, size_t j);
   GPUd() void setCov(value_t v, int i);
   GPUd() void setCov(const covMat_t& mat);
@@ -121,6 +135,15 @@ class TrackParametrizationWithError : public TrackParametrization
   GPUd() void updateCov(value_t delta, size_t i, size_t j);
   GPUd() void updateCov(value_t delta, size_t i);
 
+  GPUd() void updateCov(const params_t delta2, bool preserveCorrelations);
+  GPUd() void updateCov(const value_t* delta2, bool preserveCorrelations);
+
+  GPUd() void updateCovCorr(const params_t delta2);
+  GPUd() void updateCovCorr(const value_t* delta2);
+
+  GPUd() void updateCov(const params_t delta2);
+  GPUd() void updateCov(const value_t* delta2);
+
  protected:
   covMat_t mC{0.f}; // 15 covariance matrix elements
 
@@ -129,7 +152,7 @@ class TrackParametrizationWithError : public TrackParametrization
 
 //__________________________________________________________________________
 template 
-GPUdi() TrackParametrizationWithError::TrackParametrizationWithError() : TrackParametrization{}
+GPUhdi() TrackParametrizationWithError::TrackParametrizationWithError() : TrackParametrization{}
 {
 }
 
@@ -137,7 +160,8 @@ GPUdi() TrackParametrizationWithError::TrackParametrizationWithError()
 template 
 GPUdi() TrackParametrizationWithError::TrackParametrizationWithError(value_t x, value_t alpha, const params_t& par,
                                                                               const covMat_t& cov, int charge, const PID pid)
-  : TrackParametrization{x, alpha, par, charge, pid}
+  : TrackParametrization{
+      x, alpha, par, charge, pid}
 {
   // explicit constructor
   for (int i = 0; i < kCovMatSize; i++) {
@@ -293,11 +317,21 @@ template 
 template 
 GPUdi() auto TrackParametrizationWithError::getPredictedChi2(const BaseCluster& p) const -> value_t
 {
-  const dim2_t pyz = {p.getY(), p.getZ()};
-  const dim3_t cov = {p.getSigmaY2(), p.getSigmaYZ(), p.getSigmaZ2()};
+  const dim2_t pyz = {value_T(p.getY()), value_T(p.getZ())};
+  const dim3_t cov = {value_T(p.getSigmaY2()), value_T(p.getSigmaYZ()), value_T(p.getSigmaZ2())};
   return getPredictedChi2(pyz, cov);
 }
 
+//__________________________________________________________________________
+template 
+template 
+GPUdi() auto TrackParametrizationWithError::getPredictedChi2Quiet(const BaseCluster& p) const -> value_t
+{
+  const dim2_t pyz = {value_T(p.getY()), value_T(p.getZ())};
+  const dim3_t cov = {value_T(p.getSigmaY2()), value_T(p.getSigmaYZ()), value_T(p.getSigmaZ2())};
+  return getPredictedChi2Quiet(pyz, cov);
+}
+
 //______________________________________________
 template 
 GPUdi() auto TrackParametrizationWithError::getPredictedChi2(const dim2_t& p, const dim3_t& cov) const -> value_t
@@ -305,6 +339,13 @@ GPUdi() auto TrackParametrizationWithError::getPredictedChi2(const dim2
   return getPredictedChi2(p.data(), cov.data());
 }
 
+//______________________________________________
+template 
+GPUdi() auto TrackParametrizationWithError::getPredictedChi2Quiet(const dim2_t& p, const dim3_t& cov) const -> value_t
+{
+  return getPredictedChi2Quiet(p.data(), cov.data());
+}
+
 //______________________________________________
 template 
 GPUdi() bool TrackParametrizationWithError::update(const dim2_t& p, const dim3_t& cov)
@@ -317,8 +358,8 @@ template 
 template 
 GPUdi() bool TrackParametrizationWithError::update(const BaseCluster& p)
 {
-  const dim2_t pyz = {p.getY(), p.getZ()};
-  const dim3_t cov = {p.getSigmaY2(), p.getSigmaYZ(), p.getSigmaZ2()};
+  const dim2_t pyz = {value_T(p.getY()), value_T(p.getZ())};
+  const dim3_t cov = {value_T(p.getSigmaY2()), value_T(p.getSigmaYZ()), value_T(p.getSigmaZ2())};
   return update(pyz, cov);
 }
 
@@ -365,6 +406,71 @@ GPUdi() void TrackParametrizationWithError::updateCov(const covMat_t& d
   }
 }
 
+//__________________________________________________________________________
+template 
+GPUdi() void TrackParametrizationWithError::updateCov(const params_t delta2)
+{
+  // Increment cov.matrix diagonal elements by the vector of squared deltas
+  updateCov(delta2.data());
+}
+
+//__________________________________________________________________________
+template 
+GPUdi() void TrackParametrizationWithError::updateCov(const value_t* delta2)
+{
+  // Increment cov.matrix diagonal elements by the vector of squared deltas
+  for (int i = 0; i < kNParams; i++) {
+    mC[DiagMap[i]] += delta2[i];
+  }
+}
+
+//__________________________________________________________________________
+template 
+GPUdi() void TrackParametrizationWithError::updateCovCorr(const params_t delta2)
+{
+  // Increment cov.matrix diagonal elements by the vector of squared deltas, modify non-diagonal elements to preserve correlations
+  updateCovCorr(delta2.data());
+}
+
+//__________________________________________________________________________
+template 
+GPUdi() void TrackParametrizationWithError::updateCovCorr(const value_t* delta2)
+{
+  // Increment cov.matrix diagonal elements by the vector of squared deltas, modify non-diagonal elements to preserve correlations
+#pragma GCC diagnostic push // FIXME: remove in the future, GCC compiler bug reports incorrect uninitialized warning for oldDiag
+#pragma GCC diagnostic ignored "-Wuninitialized"
+  value_t oldDiag[kNParams];
+  for (int i = 0; i < kNParams; i++) {
+    auto diagI = DiagMap[i];
+    oldDiag[i] = mC[diagI];
+    mC[diagI] += delta2[i];
+    for (int j = 0; j < i; j++) {
+      mC[CovarMap[i][j]] *= gpu::CAMath::Sqrt(mC[diagI] * mC[DiagMap[j]] / (oldDiag[i] * oldDiag[j]));
+    }
+  }
+#pragma GCC diagnostic pop
+}
+
+//__________________________________________________________________________
+template 
+GPUdi() void TrackParametrizationWithError::updateCov(const params_t delta2, bool preserveCorrelations)
+{
+  // Increment cov.matrix diagonal elements by the vector of squared deltas. If requested, modify non-diagonal elements to preserve correlations
+  updateCov(delta2.data(), preserveCorrelations);
+}
+
+//__________________________________________________________________________
+template 
+GPUdi() void TrackParametrizationWithError::updateCov(const value_t* delta2, bool preserveCorrelations)
+{
+  // Increment cov.matrix diagonal elements by the vector of squared deltas. If requested, modify non-diagonal elements to preserve correlations
+  if (preserveCorrelations) {
+    updateCovCorr(delta2);
+  } else {
+    updateCov(delta2);
+  }
+}
+
 } // namespace track
 } // namespace o2
 #endif /* INCLUDE_RECONSTRUCTIONDATAFORMATS_TRACKPARAMETRIZATIONWITHERROR_H_ */
diff --git a/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackUtils.h b/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackUtils.h
index 8e8dd9033bdf8..8a79130d64eda 100644
--- a/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackUtils.h
+++ b/DataFormats/Reconstruction/include/ReconstructionDataFormats/TrackUtils.h
@@ -18,9 +18,9 @@
 #define INCLUDE_RECONSTRUCTIONDATAFORMATS_TRACKUTILS_H_
 
 #include "GPUCommonRtypes.h"
-#include "GPUCommonArray.h"
 
 #ifndef GPUCA_GPUCODE_DEVICE
+#include 
 #include 
 #endif
 
@@ -33,14 +33,17 @@ namespace track
 {
 // helper function
 template 
-GPUd() value_T BetheBlochSolid(value_T bg, value_T rho = 2.33, value_T kp1 = 0.20, value_T kp2 = 3.00, value_T meanI = 173e-9,
-                               value_T meanZA = 0.49848);
+GPUd() value_T BetheBlochSolid(value_T bg, value_T rho = 2.33, value_T kp1 = 0.20, value_T kp2 = 3.00, value_T meanI = 173e-9, value_T meanZA = 0.49848);
+
+template 
+GPUd() value_T BetheBlochSolidOpt(value_T bg);
+
 template 
-GPUd() void g3helx3(value_T qfield, value_T step, gpu::gpustd::array& vect);
+GPUd() void g3helx3(value_T qfield, value_T step, std::array& vect);
 
 //____________________________________________________
 template 
-GPUd() void g3helx3(value_T qfield, value_T step, gpu::gpustd::array& vect)
+GPUd() void g3helx3(value_T qfield, value_T step, std::array& vect)
 {
   /******************************************************************
    *                                                                *
@@ -121,11 +124,11 @@ GPUd() value_T BetheBlochSolid(value_T bg, value_T rho, value_T kp1, value_T kp2
   static_assert(std::is_floating_point_v);
 #endif
 
-  constexpr value_T mK = 0.307075e-3f; // [GeV*cm^2/g]
-  constexpr value_T me = 0.511e-3f;    // [GeV/c^2]
+  constexpr value_T mK = 0.307075e-3; // [GeV*cm^2/g]
+  constexpr value_T me = 0.511e-3;    // [GeV/c^2]
   kp1 *= 2.303f;
   kp2 *= 2.303f;
-  value_T bg2 = bg * bg;
+  value_T bg2 = bg * bg, beta2 = bg2 / (1 + bg2);
   value_T maxT = 2.f * me * bg2; // neglecting the electron mass
 
   //*** Density effect
@@ -138,7 +141,75 @@ GPUd() value_T BetheBlochSolid(value_T bg, value_T rho, value_T kp1, value_T kp2
     double r = (kp2 - x) / (kp2 - kp1);
     d2 = lhwI + x - 0.5f + (0.5f - lhwI - kp1) * r * r * r;
   }
-  return mK * meanZA * (1 + bg2) / bg2 * (0.5f * gpu::CAMath::Log(2 * me * bg2 * maxT / (meanI * meanI)) - bg2 / (1 + bg2) - d2);
+  auto dedx = mK * meanZA / beta2 * (0.5f * gpu::CAMath::Log(2 * me * bg2 * maxT / (meanI * meanI)) - beta2 - d2);
+  return dedx > 0. ? dedx : 0.;
+}
+
+//____________________________________________________
+template 
+GPUd() value_T BetheBlochSolidOpt(value_T bg)
+{
+  //
+  // This is the parameterization of the Bethe-Bloch formula inspired by Geant with hardcoded constants and better optimization
+  //
+  // bg  - beta*gamma
+  // rho - density [g/cm^3]
+  // kp1 - density effect first junction point
+  // kp2 - density effect second junction point
+  // meanI - mean excitation energy [GeV]
+  // meanZA - mean Z/A
+  //
+  // The default values for the kp* parameters are for silicon.
+  // The returned value is in [GeV/(g/cm^2)].
+  //
+  //  constexpr value_T rho = 2.33;
+  //  constexpr value_T meanI = 173e-9;
+  //  constexpr value_T me = 0.511e-3;    // [GeV/c^2]
+
+  constexpr value_T mK = 0.307075e-3; // [GeV*cm^2/g]
+  constexpr value_T kp1 = 0.20 * 2.303;
+  constexpr value_T kp2 = 3.00 * 2.303;
+  constexpr value_T meanZA = 0.49848;
+  constexpr value_T lhwI = -1.7175226;         // gpu::CAMath::Log(28.816 * 1e-9 * gpu::CAMath::Sqrt(rho * meanZA) / meanI);
+  constexpr value_T log2muTomeanI = 8.6839805; // gpu::CAMath::Log( 2. * me / meanI);
+
+  value_T bg2 = bg * bg, beta2 = bg2 / (1. + bg2);
+
+  //*** Density effect
+  value_T d2 = 0.;
+  const value_T x = gpu::CAMath::Log(bg);
+  if (x > kp2) {
+    d2 = lhwI - 0.5f + x;
+  } else if (x > kp1) {
+    value_T r = (kp2 - x) / (kp2 - kp1);
+    d2 = lhwI - 0.5 + x + (0.5 - lhwI - kp1) * r * r * r;
+  }
+  auto dedx = mK * meanZA / beta2 * (log2muTomeanI + x + x - beta2 - d2);
+  return dedx > 0. ? dedx : 0.;
+}
+
+//____________________________________________________
+template 
+GPUdi() value_T BetheBlochSolidDerivative(value_T dedx, value_T bg)
+{
+  //
+  // This is approximate derivative of the BB over betagamm, NO check for the consistency of the provided dedx and bg is done
+  // Charge 1 particle is assumed for the provied dedx. For charge > 1 particles dedx/q^2 should be provided and obtained value must be scaled by q^2
+  // The call should be usually done as
+  // auto dedx = BetheBlochSolidOpt(bg);
+  // // if derivative needed
+  // auto ddedx = BetheBlochSolidDerivative(dedx, bg, bg*bg)
+  //
+  // dedx - precalculate dedx for bg
+  // bg  - beta*gamma
+  //
+  constexpr value_T mK = 0.307075e-3; // [GeV*cm^2/g]
+  constexpr value_T meanZA = 0.49848;
+  auto bg2 = bg * bg;
+  auto t1 = 1 + bg2;
+  //  auto derH = (mK * meanZA * (t1+bg2) - dedx*bg2)/(bg*t1);
+  auto derH = (mK * meanZA * (t1 + 1. / bg2) - dedx) / (bg * t1);
+  return derH + derH;
 }
 
 } // namespace track
diff --git a/DataFormats/Reconstruction/include/ReconstructionDataFormats/V0.h b/DataFormats/Reconstruction/include/ReconstructionDataFormats/V0.h
index 9686a7af98056..a96ba1dd1e5a0 100644
--- a/DataFormats/Reconstruction/include/ReconstructionDataFormats/V0.h
+++ b/DataFormats/Reconstruction/include/ReconstructionDataFormats/V0.h
@@ -15,6 +15,7 @@
 #include "ReconstructionDataFormats/VtxTrackIndex.h"
 #include "ReconstructionDataFormats/Track.h"
 #include "ReconstructionDataFormats/PID.h"
+#include "ReconstructionDataFormats/DecayNBodyIndex.h" // RS Remove after dropping indices in O2Physics
 #include 
 #include 
 
@@ -31,12 +32,9 @@ class V0 : public o2::track::TrackParCov
   using PID = o2::track::PID;
 
   V0() = default;
-  V0(const std::array& xyz, const std::array& pxyz, const std::array& covxyz,
-     const o2::track::TrackParCov& trPos, const o2::track::TrackParCov& trNeg,
-     GIndex trPosID, GIndex trNegID, o2::track::PID pid = o2::track::PID::K0);
 
-  GIndex getProngID(int i) const { return mProngIDs[i]; }
-  void setProngID(int i, GIndex gid) { mProngIDs[i] = gid; }
+  V0(const std::array& xyz, const std::array& pxyz, const std::array& covxyz,
+     const o2::track::TrackParCov& trPos, const o2::track::TrackParCov& trNeg, o2::track::PID pid = o2::track::PID::K0);
 
   const Track& getProng(int i) const { return mProngs[i]; }
   Track& getProng(int i) { return mProngs[i]; }
@@ -48,23 +46,28 @@ class V0 : public o2::track::TrackParCov
   float getDCA() const { return mDCA; }
   void setDCA(float d) { mDCA = d; }
 
-  int getVertexID() const { return mVertexID; }
-  void setVertexID(int id) { mVertexID = id; }
-
-  float calcMass2() const { return calcMass2(mProngs[0].getPID(), mProngs[1].getPID()); }
-  float calcMass2(PID pidPos, PID pidNeg) const { return calcMass2(pidPos.getMass2(), pidNeg.getMass2()); }
+  float calcMass2() const { return calcMass2PID(mProngs[0].getPID(), mProngs[1].getPID()); }
+  float calcMass2PID(PID pidPos, PID pidNeg) const { return calcMass2(PID::getMass2(pidPos), PID::getMass2(pidNeg)); }
   float calcMass2(float massPos2, float massNeg2) const;
 
+  float calcMass2AsPhoton() const { return calcMass2PID(PID::Electron, PID::Electron); }
+  float calcMass2AsK0() const { return calcMass2PID(PID::Pion, PID::Pion); }
+  float calcMass2AsLambda() const { return calcMass2PID(PID::Proton, PID::Pion); }
+  float calcMass2AsAntiLambda() const { return calcMass2PID(PID::Pion, PID::Proton); }
+  float calcMass2AsHyperTriton() const { return calcMass2PID(PID::Helium3, PID::Pion); }
+  float calcMass2AsAntiHyperTriton() const { return calcMass2PID(PID::Pion, PID::Helium3); }
+  float calcMass2AsHyperhydrog4() const { return calcMass2PID(PID::Alpha, PID::Pion); }
+  float calcMass2AsAntiHyperhydrog4() const { return calcMass2PID(PID::Pion, PID::Alpha); }
+  float calcAPQt() const;
+  float calcAPAlpha() const;
   float calcR2() const { return getX() * getX() + getY() * getY(); }
 
  protected:
-  std::array mProngIDs; // global IDs of prongs
   std::array mProngs;    // prongs kinematics at vertex
   float mCosPA = 0;                // cos of pointing angle
   float mDCA = 9990;               // distance of closest approach of prongs
-  int mVertexID = -1;              // id of parent vertex
 
-  ClassDefNV(V0, 1);
+  ClassDefNV(V0, 2);
 };
 
 } // namespace dataformats
diff --git a/DataFormats/Reconstruction/include/ReconstructionDataFormats/Vertex.h b/DataFormats/Reconstruction/include/ReconstructionDataFormats/Vertex.h
index da68048420f4d..cb1c9d5d87c7f 100644
--- a/DataFormats/Reconstruction/include/ReconstructionDataFormats/Vertex.h
+++ b/DataFormats/Reconstruction/include/ReconstructionDataFormats/Vertex.h
@@ -14,14 +14,17 @@
 
 #include "GPUCommonDef.h"
 #include "GPUCommonMath.h"
-#include "GPUCommonArray.h"
 #include 
 
 #include "CommonDataFormat/TimeStamp.h"
 #ifndef GPUCA_GPUCODE_DEVICE
-#include 
-#include 
 #include 
+#include 
+#ifndef GPUCA_NO_FMT
+#include 
+#include 
+#include 
+#endif
 #endif
 
 namespace o2
@@ -42,9 +45,17 @@ class VertexBase
   static constexpr int kNCov = 6;
   GPUhdDefault() VertexBase() = default;
   GPUhdDefault() ~VertexBase() = default;
-  GPUhd() VertexBase(const math_utils::Point3D& pos, const gpu::gpustd::array& cov) : mPos(pos), mCov(cov)
+  GPUhd() VertexBase(const float* pos, const float* cov)
   {
+    mPos = math_utils::Point3D(pos[0], pos[1], pos[2]);
+    mCov[kCovXX] = cov[kCovXX];
+    mCov[kCovXY] = cov[kCovXY];
+    mCov[kCovXZ] = cov[kCovXZ];
+    mCov[kCovYY] = cov[kCovYY];
+    mCov[kCovYZ] = cov[kCovYZ];
+    mCov[kCovZZ] = cov[kCovZZ];
   }
+  GPUhd() VertexBase(const math_utils::Point3D& pos, const std::array& cov) : mPos(pos), mCov(cov) {}
 
 #if !defined(GPUCA_NO_FMT) && !defined(GPUCA_GPUCODE_DEVICE)
   void print() const;
@@ -55,13 +66,19 @@ class VertexBase
   GPUhd() float getX() const { return mPos.X(); }
   GPUhd() float getY() const { return mPos.Y(); }
   GPUhd() float getZ() const { return mPos.Z(); }
+  GPUhd() float getR() const { return gpu::CAMath::Hypot(mPos.X(), mPos.Y()); }
   GPUd() float getSigmaX2() const { return mCov[kCovXX]; }
   GPUd() float getSigmaY2() const { return mCov[kCovYY]; }
   GPUd() float getSigmaZ2() const { return mCov[kCovZZ]; }
   GPUd() float getSigmaXY() const { return mCov[kCovXY]; }
   GPUd() float getSigmaXZ() const { return mCov[kCovXZ]; }
   GPUd() float getSigmaYZ() const { return mCov[kCovYZ]; }
-  GPUd() const gpu::gpustd::array& getCov() const { return mCov; }
+  GPUd() float getSigmaX() const { return gpu::CAMath::Sqrt(getSigmaX2()); }
+  GPUd() float getSigmaY() const { return gpu::CAMath::Sqrt(getSigmaY2()); }
+  GPUd() float getSigmaZ() const { return gpu::CAMath::Sqrt(getSigmaZ2()); }
+
+  GPUd() const std::array& getCov() const { return mCov; }
+  GPUd() float getCov(int e) const { return mCov[e]; }
 
   GPUd() math_utils::Point3D getXYZ() const { return mPos; }
   GPUd() math_utils::Point3D& getXYZ() { return mPos; }
@@ -84,6 +101,10 @@ class VertexBase
   GPUd() void setSigmaXY(float v) { mCov[kCovXY] = v; }
   GPUd() void setSigmaXZ(float v) { mCov[kCovXZ] = v; }
   GPUd() void setSigmaYZ(float v) { mCov[kCovYZ] = v; }
+  GPUd() void setSigmaX(float val) { setSigmaX2(val * val); }
+  GPUd() void setSigmaY(float val) { setSigmaY2(val * val); }
+  GPUd() void setSigmaZ(float val) { setSigmaZ2(val * val); }
+
   GPUd() void setCov(float sxx, float sxy, float syy, float sxz, float syz, float szz)
   {
     setSigmaX2(sxx);
@@ -93,14 +114,15 @@ class VertexBase
     setSigmaXZ(sxz);
     setSigmaYZ(syz);
   }
-  GPUd() void setCov(const gpu::gpustd::array& cov) { mCov = cov; }
+  GPUd() void setCov(const std::array& cov) { mCov = cov; }
+  GPUd() void setCov(float c, int e) { mCov[e] = c; }
 
   bool operator==(const VertexBase& other) const;
   bool operator!=(const VertexBase& other) const { return !(*this == other); }
 
  protected:
   math_utils::Point3D mPos{0., 0., 0.}; ///< cartesian position
-  gpu::gpustd::array mCov{};     ///< errors, see CovElems enum
+  std::array mCov{};             ///< errors, see CovElems enum
 
   ClassDefNV(VertexBase, 1);
 };
@@ -116,15 +138,19 @@ class Vertex : public VertexBase
   using ushort = unsigned short;
   enum Flags : ushort {
     TimeValidated = 0x1 << 0, // Flag that the vertex was validated by external time measurement (e.g. FIT)
+    UPCMode = 0x1 << 1,       // vertex is found in the UPC mode ITS ROF
     FlagsMask = 0xffff
   };
 
   GPUhdDefault() Vertex() = default;
   GPUhdDefault() ~Vertex() = default;
-  GPUhd() Vertex(const math_utils::Point3D& pos, const gpu::gpustd::array& cov, ushort nCont, float chi2)
-    : VertexBase(pos, cov), mChi2(chi2), mNContributors(nCont)
-  {
-  }
+  GPUhd() Vertex(const float* pos, const float* cov, ushort nCont, float chi2) : VertexBase(pos, cov), mChi2(chi2), mNContributors(nCont) {}
+  GPUhd() Vertex(const math_utils::Point3D& pos, const std::array& cov, ushort nCont, float chi2) : VertexBase(pos, cov), mChi2(chi2), mNContributors(nCont) {}
+
+#if !defined(GPUCA_NO_FMT) && !defined(GPUCA_GPUCODE_DEVICE)
+  void print() const;
+  std::string asString() const;
+#endif
 
   GPUd() ushort getNContributors() const { return mNContributors; }
   GPUd() void setNContributors(ushort v) { mNContributors = v; }
@@ -133,13 +159,13 @@ class Vertex : public VertexBase
   GPUd() ushort getFlags() const { return mBits; }
   GPUd() bool isFlagSet(uint f) const { return mBits & (FlagsMask & f); }
   GPUd() void setFlags(ushort f) { mBits |= FlagsMask & f; }
-  GPUd() void resetFrags(ushort f = FlagsMask) { mBits &= ~(FlagsMask & f); }
+  GPUd() void resetFlags(ushort f = FlagsMask) { mBits &= ~(FlagsMask & f); }
 
   GPUd() void setChi2(float v) { mChi2 = v; }
   GPUd() float getChi2() const { return mChi2; }
 
-  GPUd() const Stamp& getTimeStamp() const { return mTimeStamp; }
-  GPUd() Stamp& getTimeStamp() { return mTimeStamp; }
+  GPUhd() const Stamp& getTimeStamp() const { return mTimeStamp; }
+  GPUhd() Stamp& getTimeStamp() { return mTimeStamp; }
   GPUd() void setTimeStamp(const Stamp& v) { mTimeStamp = v; }
 
  protected:
@@ -153,6 +179,49 @@ class Vertex : public VertexBase
 
 #if !defined(GPUCA_GPUCODE_DEVICE) && !defined(GPUCA_NO_FMT)
 std::ostream& operator<<(std::ostream& os, const o2::dataformats::VertexBase& v);
+
+namespace detail
+{
+template 
+concept Streamable = requires(std::ostream& os, const T& a) {
+  { os << a } -> std::same_as;
+};
+
+template 
+concept HasFormattableTimeStamp = requires(const T& t) {
+  { fmt::format("{}", t.getTimeStamp()) } -> std::convertible_to;
+};
+} // namespace detail
+
+template 
+inline std::string Vertex::asString() const
+{
+  const std::string stamp = [&]() -> std::string {
+    if constexpr (detail::Streamable) {
+      std::ostringstream oss;
+      oss << mTimeStamp;
+      return oss.str();
+    } else if constexpr (detail::HasFormattableTimeStamp) {
+      return fmt::format("{}", mTimeStamp.getTimeStamp());
+    } else {
+      return "X";
+    }
+  }();
+  return fmt::format("{} NContrib:{} Chi2:{:.2f} Flags:{:b} Stamp:{}", VertexBase::asString(), mNContributors, mChi2, mBits, stamp);
+}
+
+template 
+inline std::ostream& operator<<(std::ostream& os, const o2::dataformats::Vertex& v)
+{
+  os << v.asString();
+  return os;
+}
+
+template 
+inline void Vertex::print() const
+{
+  std::cout << *this << '\n';
+}
 #endif
 
 } // namespace dataformats
@@ -170,9 +239,6 @@ template <>
 struct is_messageable>> : std::true_type {
 };
 template <>
-struct is_messageable>> : std::true_type {
-};
-template <>
 struct is_messageable>> : std::true_type {
 };
 } // namespace framework
diff --git a/DataFormats/Reconstruction/src/Cascade.cxx b/DataFormats/Reconstruction/src/Cascade.cxx
index dbc2c4cbae011..f5d7a017a89f8 100644
--- a/DataFormats/Reconstruction/src/Cascade.cxx
+++ b/DataFormats/Reconstruction/src/Cascade.cxx
@@ -12,23 +12,21 @@
 #include "ReconstructionDataFormats/Cascade.h"
 
 using namespace o2::dataformats;
-
+/*
 Cascade::Cascade(const std::array& xyz, const std::array& pxyz, const std::array& covxyz,
-                 const o2::track::TrackParCov& v0, const o2::track::TrackParCov& bachelor,
-                 int v0ID, GIndex bachelorID, o2::track::PID pid)
+                 const o2::track::TrackParCov& v0, const o2::track::TrackParCov& bachelor, o2::track::PID pid) : mProngs{v0, bachelor}
 {
-  std::array covV{}, covB{};
+  std::array covC{0.}, covV{}, covB{};
   v0.getCovXYZPxPyPzGlo(covV);
   bachelor.getCovXYZPxPyPzGlo(covB);
-  for (int i = 0; i < 21; i++) {
-    covV[i] += covB[i];
-  }
+  constexpr int MomInd[6] = {9, 13, 14, 18, 19, 20}; // cov matrix elements for momentum component
   for (int i = 0; i < 6; i++) {
-    covV[i] = covxyz[i];
+    covC[i] = covxyz[i];
+    covC[MomInd[i]] = covV[MomInd[i]] + covB[MomInd[i]];
   }
-  this->set(xyz, pxyz, covV, v0.getCharge() + bachelor.getCharge(), true, pid);
-  setV0ID(v0ID);
-  setBachelorID(bachelorID);
+  this->set(xyz, pxyz, covC, v0.getCharge() + bachelor.getCharge(), true, pid);
+  this->checkCorrelations();
   setV0Track(v0);
   setBachelorTrack(bachelor);
 }
+*/
diff --git a/DataFormats/Reconstruction/src/Decay3Body.cxx b/DataFormats/Reconstruction/src/Decay3Body.cxx
new file mode 100644
index 0000000000000..aa071cea675cd
--- /dev/null
+++ b/DataFormats/Reconstruction/src/Decay3Body.cxx
@@ -0,0 +1,37 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "ReconstructionDataFormats/Decay3Body.h"
+
+using namespace o2::dataformats;
+
+Decay3Body::Decay3Body(const std::array& xyz, const std::array& pxyz, const std::array& covxyz, const Track& tr0, const Track& tr1, const Track& tr2, o2::track::PID pid)
+  : mProngs{tr0, tr1, tr2}
+{
+  std::array cov{}, cov1{}, cov2{};
+  tr0.getCovXYZPxPyPzGlo(cov);
+  tr1.getCovXYZPxPyPzGlo(cov1);
+  tr2.getCovXYZPxPyPzGlo(cov2);
+  for (int i = 0; i < 21; i++) {
+    cov[i] += cov1[i] + cov2[i];
+  }
+  for (int i = 0; i < 6; i++) {
+    cov[i] = covxyz[i];
+  }
+  this->set(xyz, pxyz, cov, tr0.getCharge() + tr1.getCharge() + tr2.getCharge(), true, pid);
+}
+
+float Decay3Body::calcMass2(float mass0, float mass1, float mass2) const
+{
+  auto p2 = getP2();
+  auto energy = std::sqrt(mass0 + mProngs[0].getP2()) + std::sqrt(mass1 + mProngs[1].getP2()) + std::sqrt(mass1 + mProngs[2].getP2());
+  return energy * energy - p2;
+}
diff --git a/DataFormats/Reconstruction/src/DecayNBodyIndex.cxx b/DataFormats/Reconstruction/src/DecayNBodyIndex.cxx
new file mode 100644
index 0000000000000..bd012091aad3f
--- /dev/null
+++ b/DataFormats/Reconstruction/src/DecayNBodyIndex.cxx
@@ -0,0 +1,18 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "ReconstructionDataFormats/DecayNBodyIndex.h"
+
+namespace o2::dataformats
+{
+template class DecayNBodyIndex<2>;
+template class DecayNBodyIndex<3>;
+} // namespace o2::dataformats
diff --git a/DataFormats/Reconstruction/src/MatchInfoHMP.cxx b/DataFormats/Reconstruction/src/MatchInfoHMP.cxx
index 1058d39752ed7..77983c4355584 100644
--- a/DataFormats/Reconstruction/src/MatchInfoHMP.cxx
+++ b/DataFormats/Reconstruction/src/MatchInfoHMP.cxx
@@ -20,5 +20,6 @@ ClassImp(o2::dataformats::MatchInfoHMP);
 
 void MatchInfoHMP::print() const
 {
-  printf("Match of GlobalID %s and HMPID cl %d ", getTrackRef().asString().c_str(), getIdxHMPClus());
+  // printf("Match of GlobalID %s and HMPID cl %d ", getTrackRef().asString().c_str(), getIdxHMPClus());
+  printf("Match of GlobalID and HMPID cl");
 }
diff --git a/DataFormats/Reconstruction/src/PID.cxx b/DataFormats/Reconstruction/src/PID.cxx
index 2e6f7b7fc7483..a7fbc811c1c81 100644
--- a/DataFormats/Reconstruction/src/PID.cxx
+++ b/DataFormats/Reconstruction/src/PID.cxx
@@ -15,7 +15,7 @@
 
 #include "ReconstructionDataFormats/PID.h"
 #include 
-#include "FairLogger.h"
+#include 
 
 using namespace o2::track;
 
diff --git a/DataFormats/Reconstruction/src/PrimaryVertexExt.cxx b/DataFormats/Reconstruction/src/PrimaryVertexExt.cxx
new file mode 100644
index 0000000000000..6065f04a3bc1a
--- /dev/null
+++ b/DataFormats/Reconstruction/src/PrimaryVertexExt.cxx
@@ -0,0 +1,51 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "ReconstructionDataFormats/PrimaryVertexExt.h"
+#include 
+#include 
+#include "CommonUtils/StringUtils.h"
+
+namespace o2
+{
+namespace dataformats
+{
+
+#ifndef GPUCA_ALIGPUCODE
+using GTrackID = o2::dataformats::GlobalTrackID;
+
+std::string PrimaryVertexExt::asString() const
+{
+  auto str = o2::utils::Str::concat_string(PrimaryVertex::asString(), fmt::format("VtxID={} FT0A/C={}/{} FT0T={}", VtxID, FT0A, FT0C, FT0Time));
+  for (int i = 0; i < GTrackID::Source::NSources; i++) {
+    if (getNSrc(i) > 0) {
+      str += fmt::format(" {}={}", GTrackID::getSourceName(i), getNSrc(i));
+    }
+  }
+  return str;
+}
+
+std::ostream& operator<<(std::ostream& os, const o2::dataformats::PrimaryVertexExt& v)
+{
+  // stream itself
+  os << v.asString();
+  return os;
+}
+
+void PrimaryVertexExt::print() const
+{
+  std::cout << *this << std::endl;
+}
+
+#endif
+
+} // namespace dataformats
+} // namespace o2
diff --git a/DataFormats/Reconstruction/src/ReconstructionDataFormatsLinkDef.h b/DataFormats/Reconstruction/src/ReconstructionDataFormatsLinkDef.h
index 1cbbbd0087595..b386830d9872d 100644
--- a/DataFormats/Reconstruction/src/ReconstructionDataFormatsLinkDef.h
+++ b/DataFormats/Reconstruction/src/ReconstructionDataFormatsLinkDef.h
@@ -24,10 +24,13 @@
 #pragma link C++ class o2::track::TrackParCovD + ;
 #pragma link C++ class o2::track::TrackParCov + ;
 #pragma link C++ class o2::track::TrackParametrizationWithError < float> + ;
+#pragma link C++ class std::vector < o2::track::TrackParametrizationWithError < float>> + ;
+
 #pragma link C++ class o2::track::TrackParametrizationWithError < double> + ;
 #pragma link C++ class o2::track::TrackParFwd + ;
 #pragma link C++ class o2::track::PID + ;
 #pragma link C++ class o2::track::TrackLTIntegral + ;
+#pragma link C++ class std::vector < o2::track::TrackLTIntegral> + ;
 
 #pragma link C++ class o2::track::TrackParCovFwd + ;
 #pragma link C++ class std::vector < o2::track::TrackParCovFwd> + ;
@@ -44,6 +47,9 @@
 #pragma link C++ class o2::dataformats::TrackTPCTOF + ;
 #pragma link C++ class std::vector < o2::dataformats::TrackTPCTOF> + ;
 
+#pragma link C++ class o2::dataformats::TrackHMP + ;
+#pragma link C++ class std::vector < o2::dataformats::TrackHMP> + ;
+
 #pragma link C++ class o2::dataformats::TrackCosmics + ;
 #pragma link C++ class std::vector < o2::dataformats::TrackCosmics> + ;
 
@@ -68,10 +74,12 @@
 #pragma link C++ class o2::dataformats::Vertex < o2::dataformats::TimeStamp < int>> + ;
 #pragma link C++ class o2::dataformats::Vertex < o2::dataformats::TimeStampWithError < float, float>> + ;
 #pragma link C++ class o2::dataformats::PrimaryVertex + ;
+#pragma link C++ class o2::dataformats::PrimaryVertexExt + ;
 
 #pragma link C++ class std::vector < o2::dataformats::Vertex < o2::dataformats::TimeStamp < int>>> + ;
 #pragma link C++ class std::vector < o2::dataformats::Vertex < o2::dataformats::TimeStampWithError < float, float>>> + ;
 #pragma link C++ class std::vector < o2::dataformats::PrimaryVertex> + ;
+#pragma link C++ class std::vector < o2::dataformats::PrimaryVertexExt> + ;
 
 #pragma link C++ class o2::dataformats::GlobalTrackID + ;
 #pragma link C++ class std::vector < o2::dataformats::GlobalTrackID> + ;
@@ -85,10 +93,31 @@
 
 #pragma link C++ class o2::dataformats::DCA + ;
 
+#pragma link C++ class o2::dataformats::DecayNBodyIndex < 2> + ;
+#pragma link C++ class o2::dataformats::DecayNBodyIndex < 3> + ;
+#pragma link C++ class o2::dataformats::Decay3BodyIndex + ;
+#pragma link C++ class o2::dataformats::V0Index + ;
+#pragma link C++ class o2::dataformats::CascadeIndex + ;
+
+#pragma link C++ class std::vector < o2::dataformats::DecayNBodyIndex < 2>> + ;
+#pragma link C++ class std::vector < o2::dataformats::DecayNBodyIndex < 3>> + ;
+#pragma link C++ class std::vector < o2::dataformats::Decay3BodyIndex> + ;
+#pragma link C++ class std::vector < o2::dataformats::V0Index> + ;
+#pragma link C++ class std::vector < o2::dataformats::CascadeIndex> + ;
+
 #pragma link C++ class o2::dataformats::V0 + ;
 #pragma link C++ class std::vector < o2::dataformats::V0> + ;
 
 #pragma link C++ class o2::dataformats::Cascade + ;
 #pragma link C++ class std::vector < o2::dataformats::Cascade> + ;
 
+#pragma link C++ class o2::dataformats::Decay3Body + ;
+#pragma link C++ class std::vector < o2::dataformats::Decay3Body> + ;
+
+#pragma link C++ class o2::dataformats::StrangeTrack + ;
+#pragma link C++ class std::vector < o2::dataformats::StrangeTrack> + ;
+
+#pragma link C++ class o2::track::TrackAuxPar + ;
+#pragma link C++ class o2::track::CrossInfo + ;
+
 #endif
diff --git a/DataFormats/Reconstruction/src/StrangeTrack.cxx b/DataFormats/Reconstruction/src/StrangeTrack.cxx
new file mode 100644
index 0000000000000..70c3c8d1c2d80
--- /dev/null
+++ b/DataFormats/Reconstruction/src/StrangeTrack.cxx
@@ -0,0 +1,19 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file StrangeTrack.cxx
+/// \brief
+
+#include "ReconstructionDataFormats/StrangeTrack.h"
+
+using namespace o2::dataformats;
+
+ClassImp(o2::dataformats::StrangeTrack);
diff --git a/DataFormats/Reconstruction/src/TrackFwd.cxx b/DataFormats/Reconstruction/src/TrackFwd.cxx
index f567b47f18038..dfe72c5b2ccc4 100644
--- a/DataFormats/Reconstruction/src/TrackFwd.cxx
+++ b/DataFormats/Reconstruction/src/TrackFwd.cxx
@@ -11,6 +11,7 @@
 
 #include "ReconstructionDataFormats/TrackFwd.h"
 #include "Math/MatrixFunctions.h"
+#include 
 
 namespace o2
 {
@@ -244,6 +245,12 @@ void TrackParCovFwd::propagateToZhelix(double zEnd, double zField)
 //__________________________________________________________________________
 void TrackParCovFwd::propagateToZ(double zEnd, double zField)
 {
+  // Security for zero B field
+  if (zField == 0.0) {
+    propagateToZlinear(zEnd);
+    return;
+  }
+
   // Extrapolate track parameters and covariances matrix to "zEnd"
   // Parameters: helix track model; Error propagation: Quadratic
 
@@ -433,6 +440,137 @@ void TrackParFwd::getCircleParams(float bz, o2::math_utils::CircleXY& c,
     c.yC = getY();
   }
 }
+//________________________________________________________________
+bool TrackParCovFwd::propagateToVtxhelixWithMCS(double z, const std::array& p, const std::array& cov, double field, double x_over_X0)
+{
+  // Propagate fwd track to vertex using helix model, adding MCS effects
+  addMCSEffect(x_over_X0);
+  propagateToZhelix(z, field);
+  return update(p, cov);
+}
+//________________________________________________________________
+bool TrackParCovFwd::propagateToVtxlinearWithMCS(double z, const std::array& p, const std::array& cov, double x_over_X0)
+{
+  // Propagate fwd track to vertex using linear model, adding MCS effects
+  addMCSEffect(x_over_X0);
+  propagateToZlinear(z);
+  return update(p, cov);
+}
+
+bool TrackParCovFwd::getCovXYZPxPyPzGlo(std::array& cv) const
+{
+  //---------------------------------------------------------------------
+  // This function returns the global covariance matrix of the fwdtrack params
+  //
+  // Cov(x,x) ... :   cv[0]
+  // Cov(y,x) ... :   cv[1]  cv[2]
+  // Cov(z,x) ... :   cv[3]  cv[4]  cv[5]
+  // Cov(px,x)... :   cv[6]  cv[7]  cv[8]  cv[9]
+  // Cov(py,x)... :   cv[10] cv[11] cv[12] cv[13] cv[14]
+  // Cov(pz,x)... :   cv[15] cv[16] cv[17] cv[18] cv[19] cv[20]
+  //---------------------------------------------------------------------
+  auto pt = getPt();
+  auto cp = std::sqrt((1. - getSnp()) * (1. + getSnp()));
+  auto sp = getSnp();
+  auto tgl = getTgl();
+
+  auto px = pt * std::sqrt((1. - getSnp()) * (1. + getSnp()));
+  auto py = pt * getSnp();
+  auto pz = pt * getTgl();
+  auto q = getCharge();
+
+  cv[0] = mCovariances(0, 0);
+  cv[1] = mCovariances(1, 0);
+  cv[2] = mCovariances(1, 1);
+  cv[3] = 0;
+  cv[4] = 0;
+  cv[5] = 0;
+  cv[6] = -mCovariances(0, 2) * py - mCovariances(0, 4) * px * pt * q;
+  cv[7] = -mCovariances(1, 2) * py - mCovariances(1, 4) * px * pt * q;
+  cv[8] = 0;
+  cv[9] = 2 * mCovariances(2, 4) * px * py * q * pt + mCovariances(2, 2) * py * py + mCovariances(4, 4) * px * px * pt * pt;
+  cv[10] = mCovariances(0, 2) * px - mCovariances(0, 4) * py * pt * q;
+  cv[11] = mCovariances(1, 2) * px - mCovariances(1, 4) * py * pt * q;
+  cv[12] = 0;
+  cv[13] = mCovariances(2, 4) * (py * py - px * px) * q * pt - mCovariances(2, 2) * px * py + mCovariances(4, 4) * px * py * pt * pt;
+  cv[14] = -2 * mCovariances(2, 4) * px * py * q * pt + mCovariances(2, 2) * px * px + mCovariances(4, 4) * py * py * pt * pt;
+  cv[15] = mCovariances(0, 3) * pt - mCovariances(0, 4) * pt * pz * q;
+  cv[16] = mCovariances(1, 3) * pt - mCovariances(1, 4) * pt * pz * q;
+  cv[17] = 0;
+  cv[18] = -mCovariances(2, 3) * py * pt - mCovariances(3, 4) * px * q * pt * pt + mCovariances(2, 4) * py * pz * q * pt + mCovariances(4, 4) * px * pz * pt * pt;
+  cv[19] = mCovariances(2, 3) * px * pt - mCovariances(3, 4) * q * pt * pt * py - mCovariances(2, 4) * px * pz * q * pt + mCovariances(4, 4) * py * pz * pt * pt;
+  cv[20] = -2 * mCovariances(3, 4) * pz * q * pt * pt + mCovariances(3, 3) * pt * pt + mCovariances(4, 4) * pz * pz * pt * pt;
+
+  return true;
+}
+
+//________________________________________________________________
+
+void TrackParCovFwd::propagateToDCAhelix(double zField, const std::array& p, std::array& dca)
+{
+  // Computing DCA of fwd track w.r.t vertex in helix track model, using Newton-Raphson minimization
+
+  auto x0 = mParameters(0);
+  auto y0 = mParameters(1);
+  auto z0 = mZ;
+  auto phi0 = mParameters(2);
+  auto tanl = mParameters(3);
+  auto qOverPt = mParameters(4);
+  auto k = TMath::Abs(o2::constants::math::B2C * zField);
+  auto qpt = 1.0 / qOverPt;
+  auto qR = qpt / std::fabs(k);
+  auto invtanl = 1.0 / tanl;
+  auto Hz = std::copysign(1, zField);
+
+  auto xPV = p[0];
+  auto yPV = p[1];
+  auto zPV = p[2];
+
+  auto qRtanl = qR * tanl;
+  auto invqRtanl = 1.0 / qRtanl;
+  auto [sinp, cosp] = o2::math_utils::sincosd(phi0);
+
+  auto z = zPV;
+  double tol = 1e-4;
+  int max_iter = 10;
+  int iter = 0;
+
+  while (iter++ < max_iter) {
+    double theta = (z0 - z) * invqRtanl;
+    double phi_theta = phi0 + Hz * theta;
+    double sin_phi_theta = sin(phi_theta);
+    double cos_phi_theta = cos(phi_theta);
+
+    double DX = x0 - Hz * qR * (sin_phi_theta - sinp) - xPV;
+    double DY = y0 + Hz * qR * (cos_phi_theta - cosp) - yPV;
+    double DZ = z - zPV;
+
+    double dD2_dZ =
+      2 * DX * cos_phi_theta * invtanl +
+      2 * DY * sin_phi_theta * invtanl +
+      2 * DZ;
+
+    double d2D2_dZ2 =
+      2 * invtanl * invtanl +
+      2 * invtanl * (DX * Hz * sin_phi_theta - DY * Hz * cos_phi_theta) * invqRtanl +
+      2;
+
+    double z_new = z - dD2_dZ / d2D2_dZ2;
+
+    if (std::abs(z_new - z) < tol) {
+      z = z_new;
+      this->propagateToZhelix(z, zField);
+      dca[0] = this->getX() - xPV;
+      dca[1] = this->getY() - yPV;
+      dca[2] = this->getZ() - zPV;
+      LOG(debug) << "Converged after " << iter << " iterations for vertex X=" << p[0] << ", Y=" << p[1] << ", Z = " << p[2];
+      return;
+    }
+    z = z_new;
+  }
+  LOG(debug) << "Failed to converge after " << iter << " iterations for vertex X=" << p[0] << ", Y=" << p[1] << ", Z = " << p[2];
+  return;
+}
 
 } // namespace track
 } // namespace o2
diff --git a/DataFormats/Reconstruction/src/TrackHMP.cxx b/DataFormats/Reconstruction/src/TrackHMP.cxx
new file mode 100644
index 0000000000000..dd5865d5bc363
--- /dev/null
+++ b/DataFormats/Reconstruction/src/TrackHMP.cxx
@@ -0,0 +1,146 @@
+// Copyright 2020-2022 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "ReconstructionDataFormats/TrackParametrizationWithError.h"
+#include "ReconstructionDataFormats/TrackParametrization.h"
+#include "ReconstructionDataFormats/Track.h"
+#include "ReconstructionDataFormats/TrackLTIntegral.h"
+#include "ReconstructionDataFormats/GlobalTrackID.h"
+#include "ReconstructionDataFormats/TrackHMP.h"
+#include "Field/MagneticField.h"
+// #include "DetectorsBase/Propagator.h"
+
+// #include "Field/MagFieldFast.h"
+
+#include 
+#include 
+
+#include  //fields
+// #include "Math/Vector3D.h" //fields
+
+const Double_t kB2C = -0.299792458e-3;
+
+ClassImp(o2::dataformats::TrackHMP)
+
+  using XYZVector = ROOT::Math::XYZVector;
+
+namespace o2
+{
+namespace dataformats
+{
+
+//++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+TrackHMP::TrackHMP() = default;
+//++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+TrackHMP& TrackHMP::operator=(const o2::track::TrackParCov& t)
+{
+  // ass. op.
+
+  return *this;
+}
+//++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+bool TrackHMP::intersect(Double_t pnt[3], Double_t norm[3], double bz) const
+{
+  //+++++++++++++++++++++++++++++++++++++++++
+  // Origin: K. Shileev (Kirill.Shileev@cern.ch)
+  // Finds point of intersection (if exists) of the helix with the plane.
+  // Stores result in fX and fP.
+  // Arguments: planePoint,planeNorm - the plane defined by any plane's point
+  // and vector, normal to the plane
+  // Returns: kTrue if helix intersects the plane, kFALSE otherwise.
+  //+++++++++++++++++++++++++++++++++++++++++
+
+  std::array x0;
+  getXYZGlo(x0); // get track position in MARS
+
+  // estimates initial helix length up to plane
+  Double_t s = (pnt[0] - x0[0]) * norm[0] + (pnt[1] - x0[1]) * norm[1] + (pnt[2] - x0[2]) * norm[2];
+
+  Double_t dist = 99999, distPrev = dist;
+  // Double_t p[3];
+
+  std::array x, p;
+
+  while (TMath::Abs(dist) > 0.00001) {
+
+    // calculates helix at the distance s from x0 ALONG the helix
+
+    propagate(s, x, p, bz);
+    // distance between current helix position and plane
+
+    dist = (x[0] - pnt[0]) * norm[0] + (x[1] - pnt[1]) * norm[1] + (x[2] - pnt[2]) * norm[2];
+    if (TMath::Abs(dist) >= TMath::Abs(distPrev)) { /*Printf("***********************dist > distPrev******************");*/
+      return kFALSE;
+    }
+    distPrev = dist;
+    s -= dist;
+  }
+  // on exit pnt is intersection point,norm is track vector at that point,
+  // all in MARS
+  for (Int_t i = 0; i < 3; i++) {
+    pnt[i] = x.at(i);
+    norm[i] = p.at(i);
+  }
+
+  return kTRUE;
+} // Intersect()
+//++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+void TrackHMP::propagate(Double_t len, std::array& x, std::array& p, double bz) const
+{
+  //+++++++++++++++++++++++++++++++++++++++++
+  // Origin: K. Shileev (Kirill.Shileev@cern.ch)
+  // Extrapolate track along simple helix in magnetic field
+  // Arguments: len -distance alogn helix, [cm]
+  //            bz  - mag field, [kGaus]
+  // Returns: x and p contain extrapolated positon and momentum
+  // The momentum returned for straight-line tracks is meaningless !
+  //+++++++++++++++++++++++++++++++++++++++++
+
+  std::array x0;
+  getXYZGlo(x0); // get track position in MARS
+
+  x.at(0) = x0.at(0);
+  x.at(1) = x0.at(1);
+  x.at(2) = x0.at(2);
+
+  if (getPtInv() < o2::constants::math::Almost0 || TMath::Abs(bz) < o2::constants::math::Almost0) { // straight-line tracks
+
+    TVector3 trackDirection(1., 1., 1.);
+    trackDirection.SetMag(1);
+    trackDirection.SetTheta(getTheta());
+    trackDirection.SetPhi(getPhi());
+
+    // Double_t unit[3]; GetDirection(unit);
+    x.at(0) += trackDirection.X() * len;
+    x.at(1) += trackDirection.Y() * len;
+    x.at(2) += trackDirection.Z() * len;
+
+    p.at(0) = trackDirection.X() / o2::constants::math::Almost0;
+    p.at(1) = trackDirection.Y() / o2::constants::math::Almost0;
+    p.at(2) = trackDirection.Z() / o2::constants::math::Almost0;
+  } else {
+
+    getPxPyPzGlo(p);
+    Double_t pp = getP();
+
+    Double_t a = -kB2C * bz * getSign(); ////////// what is kB2C
+    Double_t rho = a / pp;
+    x.at(0) += p.at(0) * TMath::Sin(rho * len) / a - p.at(1) * (1 - TMath::Cos(rho * len)) / a;
+    x.at(1) += p.at(1) * TMath::Sin(rho * len) / a + p.at(0) * (1 - TMath::Cos(rho * len)) / a;
+    x.at(2) += p.at(2) * len / pp;
+    Double_t p0 = p.at(0);
+    p.at(0) = p0 * TMath::Cos(rho * len) - p.at(1) * TMath::Sin(rho * len);
+    p.at(1) = p.at(1) * TMath::Cos(rho * len) + p0 * TMath::Sin(rho * len);
+  }
+
+} // Propagate()
+} // namespace dataformats
+} // namespace o2
diff --git a/DataFormats/Reconstruction/src/TrackLTIntegral.cxx b/DataFormats/Reconstruction/src/TrackLTIntegral.cxx
index 52a77ce92a4da..426c3da04726c 100644
--- a/DataFormats/Reconstruction/src/TrackLTIntegral.cxx
+++ b/DataFormats/Reconstruction/src/TrackLTIntegral.cxx
@@ -9,6 +9,10 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 
+#ifndef GPUCA_GPUCODE_DEVICE
+#include 
+#endif
+
 #include "ReconstructionDataFormats/TrackLTIntegral.h"
 #include "CommonConstants/PhysicsConstants.h"
 #include "MathUtils/Utils.h"
@@ -35,9 +39,9 @@ GPUd() void TrackLTIntegral::print() const
 }
 
 //_____________________________________________________
-GPUd() void TrackLTIntegral::addStep(float dL, float p2Inv)
+GPUd() void TrackLTIntegral::addStep(float dL, float q2p2)
 {
-  ///< add step in cm to integrals
+  ///< add step in cm to integrals, q2p2 is (q/p)^2.
   mL += dL;
   if (isTimeNotNeeded()) {
     return;
@@ -45,7 +49,7 @@ GPUd() void TrackLTIntegral::addStep(float dL, float p2Inv)
   const float dTns = dL * 1000.f / o2::constants::physics::LightSpeedCm2NS; // time change in ps for beta = 1 particle
   for (int id = 0; id < getNTOFs(); id++) {
     const float m2z = track::PID::getMass2Z(id);
-    const float betaInv = math_utils::sqrt(1.f + m2z * m2z * p2Inv);
+    const float betaInv = math_utils::sqrt(1.f + m2z * m2z * q2p2);
     mT[id] += dTns * betaInv;
   }
 }
diff --git a/DataFormats/Reconstruction/src/TrackMCHMID.cxx b/DataFormats/Reconstruction/src/TrackMCHMID.cxx
index 71deda22bc844..2b7b0ebf7983b 100644
--- a/DataFormats/Reconstruction/src/TrackMCHMID.cxx
+++ b/DataFormats/Reconstruction/src/TrackMCHMID.cxx
@@ -55,7 +55,7 @@ std::pair TrackMCHMID::getTimeMUS(const InteractionReco
   bool isInTF = bcDiff >= 0 && bcDiff < nOrbits * o2::constants::lhc::LHCMaxBunches;
   if (printError && !isInTF) {
     LOGP(alarm, "ATTENTION: wrong bunches diff. {} for current IR {} wrt 1st TF orbit {}, source:MCH-MID",
-         bcDiff, mIR, startIR);
+         bcDiff, mIR.asString(), startIR.asString());
   }
   return std::make_pair(Time(tMean, tErr), isInTF);
 }
diff --git a/DataFormats/Reconstruction/src/TrackParametrization.cxx b/DataFormats/Reconstruction/src/TrackParametrization.cxx
index b60a99d543049..7fe677a6e1c7a 100644
--- a/DataFormats/Reconstruction/src/TrackParametrization.cxx
+++ b/DataFormats/Reconstruction/src/TrackParametrization.cxx
@@ -14,17 +14,6 @@
 /// @since  Oct 1, 2020
 /// @brief
 
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
 #include "ReconstructionDataFormats/TrackParametrization.h"
 #include "ReconstructionDataFormats/Vertex.h"
 #include "ReconstructionDataFormats/DCA.h"
@@ -59,9 +48,9 @@ GPUd() TrackParametrization::TrackParametrization(const dim3_t& xyz, co
   value_t radPos2 = xyz[0] * xyz[0] + xyz[1] * xyz[1];
   value_t alp = 0;
   if (sectorAlpha || radPos2 < 1) {
-    alp = math_utils::detail::atan2(pxpypz[1], pxpypz[0]);
+    alp = gpu::CAMath::ATan2(pxpypz[1], pxpypz[0]);
   } else {
-    alp = math_utils::detail::atan2(xyz[1], xyz[0]);
+    alp = gpu::CAMath::ATan2(xyz[1], xyz[0]);
   }
   if (sectorAlpha) {
     alp = math_utils::detail::angle2Alpha(alp);
@@ -72,7 +61,7 @@ GPUd() TrackParametrization::TrackParametrization(const dim3_t& xyz, co
   // protection against cosp<0
   if (cs * pxpypz[0] + sn * pxpypz[1] < 0) {
     LOG(debug) << "alpha from phiPos() will invalidate this track parameters, overriding to alpha from phi()";
-    alp = math_utils::detail::atan2(pxpypz[1], pxpypz[0]);
+    alp = gpu::CAMath::ATan2(pxpypz[1], pxpypz[0]);
     if (sectorAlpha) {
       alp = math_utils::detail::angle2Alpha(alp);
     }
@@ -80,14 +69,14 @@ GPUd() TrackParametrization::TrackParametrization(const dim3_t& xyz, co
   }
 
   // protection:  avoid alpha being too close to 0 or +-pi/2
-  if (math_utils::detail::abs(sn) < 2 * kSafe) {
+  if (gpu::CAMath::Abs(sn) < 2 * kSafe) {
     if (alp > 0) {
       alp += alp < constants::math::PIHalf ? 2 * kSafe : -2 * kSafe;
     } else {
       alp += alp > -constants::math::PIHalf ? -2 * kSafe : 2 * kSafe;
     }
     math_utils::detail::sincos(alp, sn, cs);
-  } else if (math_utils::detail::abs(cs) < 2 * kSafe) {
+  } else if (gpu::CAMath::Abs(cs) < 2 * kSafe) {
     if (alp > 0) {
       alp += alp > constants::math::PIHalf ? 2 * kSafe : -2 * kSafe;
     } else {
@@ -103,20 +92,20 @@ GPUd() TrackParametrization::TrackParametrization(const dim3_t& xyz, co
   math_utils::detail::rotateZ(ver, -alp);
   math_utils::detail::rotateZ(mom, -alp);
   //
-  value_t ptI = 1.f / sqrt(mom[0] * mom[0] + mom[1] * mom[1]);
+  value_t ptI = 1.f / gpu::CAMath::Sqrt(mom[0] * mom[0] + mom[1] * mom[1]);
   mX = ver[0];
   mAlpha = alp;
   mP[kY] = ver[1];
   mP[kZ] = ver[2];
   mP[kSnp] = mom[1] * ptI;
   mP[kTgl] = mom[2] * ptI;
-  mAbsCharge = math_utils::detail::abs(charge);
+  mAbsCharge = gpu::CAMath::Abs(charge);
   mP[kQ2Pt] = charge ? ptI * charge : ptI;
   mPID = pid;
   //
-  if (math_utils::detail::abs(1 - getSnp()) < kSafe) {
+  if (gpu::CAMath::Abs(1 - getSnp()) < kSafe) {
     mP[kSnp] = 1.f - kSafe; // Protection
-  } else if (math_utils::detail::abs(-1 - getSnp()) < kSafe) {
+  } else if (gpu::CAMath::Abs(-1 - getSnp()) < kSafe) {
     mP[kSnp] = -1.f + kSafe; // Protection
   }
   //
@@ -127,11 +116,11 @@ template 
 GPUd() bool TrackParametrization::getPxPyPzGlo(dim3_t& pxyz) const
 {
   // track momentum
-  if (math_utils::detail::abs(getQ2Pt()) < constants::math::Almost0 || math_utils::detail::abs(getSnp()) > constants::math::Almost1) {
+  if (gpu::CAMath::Abs(getQ2Pt()) < constants::math::Almost0 || gpu::CAMath::Abs(getSnp()) > constants::math::Almost1) {
     return false;
   }
   value_t cs, sn, pt = getPt();
-  value_t r = math_utils::detail::sqrt((1.f - getSnp()) * (1.f + getSnp()));
+  value_t r = gpu::CAMath::Sqrt((1.f - getSnp()) * (1.f + getSnp()));
   math_utils::detail::sincos(getAlpha(), sn, cs);
   pxyz[0] = pt * (r * cs - getSnp() * sn);
   pxyz[1] = pt * (getSnp() * cs + r * sn);
@@ -141,17 +130,17 @@ GPUd() bool TrackParametrization::getPxPyPzGlo(dim3_t& pxyz) const
 
 //____________________________________________________
 template 
-GPUd() bool TrackParametrization::getPosDirGlo(gpu::gpustd::array& posdirp) const
+GPUd() bool TrackParametrization::getPosDirGlo(std::array& posdirp) const
 {
   // fill vector with lab x,y,z,px/p,py/p,pz/p,p,sinAlpha,cosAlpha
-  value_t ptI = math_utils::detail::abs(getQ2Pt());
+  value_t ptI = getPtInv();
   value_t snp = getSnp();
-  if (ptI < constants::math::Almost0 || math_utils::detail::abs(snp) > constants::math::Almost1) {
+  if (gpu::CAMath::Abs(snp) > constants::math::Almost1) {
     return false;
   }
   value_t &sn = posdirp[7], &cs = posdirp[8];
-  value_t csp = math_utils::detail::sqrt((1.f - snp) * (1.f + snp));
-  value_t cstht = math_utils::detail::sqrt(1.f + getTgl() * getTgl());
+  value_t csp = gpu::CAMath::Sqrt((1.f - snp) * (1.f + snp));
+  value_t cstht = gpu::CAMath::Sqrt(1.f + getTgl() * getTgl());
   value_t csthti = 1.f / cstht;
   math_utils::detail::sincos(getAlpha(), sn, cs);
   posdirp[0] = getX() * cs - getY() * sn;
@@ -169,7 +158,7 @@ template 
 GPUd() bool TrackParametrization::rotateParam(value_t alpha)
 {
   // rotate to alpha frame
-  if (math_utils::detail::abs(getSnp()) > constants::math::Almost1) {
+  if (gpu::CAMath::Abs(getSnp()) > constants::math::Almost1) {
     LOGP(debug, "Precondition is not satisfied: |sin(phi)|>1 ! {:f}", getSnp());
     return false;
   }
@@ -178,7 +167,7 @@ GPUd() bool TrackParametrization::rotateParam(value_t alpha)
   //
   value_t ca = 0, sa = 0;
   math_utils::detail::sincos(alpha - getAlpha(), sa, ca);
-  value_t snp = getSnp(), csp = math_utils::detail::sqrt((1.f - snp) * (1.f + snp)); // Improve precision
+  value_t snp = getSnp(), csp = gpu::CAMath::Sqrt((1.f - snp) * (1.f + snp)); // Improve precision
   // RS: check if rotation does no invalidate track model (cos(local_phi)>=0, i.e. particle
   // direction in local frame is along the X axis
   if ((csp * ca + snp * sa) < 0) {
@@ -187,7 +176,39 @@ GPUd() bool TrackParametrization::rotateParam(value_t alpha)
   }
   //
   value_t tmp = snp * ca - csp * sa;
-  if (math_utils::detail::abs(tmp) > constants::math::Almost1) {
+  if (gpu::CAMath::Abs(tmp) > constants::math::Almost1) {
+    LOGP(debug, "Rotation failed: new snp {:.2f}", tmp);
+    return false;
+  }
+  value_t xold = getX(), yold = getY();
+  mAlpha = alpha;
+  mX = xold * ca + yold * sa;
+  mP[kY] = -xold * sa + yold * ca;
+  mP[kSnp] = tmp;
+  return true;
+}
+
+//______________________________________________________________
+template 
+GPUd() bool TrackParametrization::rotateParam(value_t& alpha, value_t& ca, value_t& sa)
+{
+  // rotate to alpha frame
+  if (gpu::CAMath::Abs(getSnp()) > constants::math::Almost1) {
+    LOGP(debug, "Precondition is not satisfied: |sin(phi)|>1 ! {:f}", getSnp());
+    return false;
+  }
+  //
+  math_utils::detail::bringToPMPi(alpha);
+  math_utils::detail::sincos(alpha - getAlpha(), sa, ca);
+  value_t snp = getSnp(), csp = gpu::CAMath::Sqrt((1.f - snp) * (1.f + snp)); // Improve precision
+  // RS: check if rotation does no invalidate track model (cos(local_phi)>=0, i.e. particle direction in local frame is along the X axis
+  if ((csp * ca + snp * sa) < 0) {
+    // LOGF(warning,"Rotation failed: local cos(phi) would become {:.2f}", csp * ca + snp * sa);
+    return false;
+  }
+  //
+  value_t tmp = snp * ca - csp * sa;
+  if (gpu::CAMath::Abs(tmp) > constants::math::Almost1) {
     LOGP(debug, "Rotation failed: new snp {:.2f}", tmp);
     return false;
   }
@@ -209,14 +230,13 @@ GPUd() bool TrackParametrization::propagateParamTo(value_t xk, const di
   // X [cm] is in the "tracking coordinate system" of this track.
   // b[]={Bx,By,Bz} [kG] is in the Global coordidate system.
   //----------------------------------------------------------------
-
   value_t dx = xk - getX();
-  if (math_utils::detail::abs(dx) < constants::math::Almost0) {
+  if (gpu::CAMath::Abs(dx) < constants::math::Almost0) {
     return true;
   }
   // Do not propagate tracks outside the ALICE detector
-  if (math_utils::detail::abs(dx) > 1e5 || math_utils::detail::abs(getY()) > 1e5 || math_utils::detail::abs(getZ()) > 1e5) {
-    LOGP(warning, "Anomalous track, target X:{:f}", xk);
+  if (gpu::CAMath::Abs(dx) > 1e5 || gpu::CAMath::Abs(getY()) > 1e5 || gpu::CAMath::Abs(getZ()) > 1e5) {
+    LOG(warning) << "Anomalous track, traget X:" << xk;
     return false;
   }
   value_t crv = getCurvature(b[2]);
@@ -226,43 +246,43 @@ GPUd() bool TrackParametrization::propagateParamTo(value_t xk, const di
 
   value_t x2r = crv * dx;
   value_t f1 = getSnp(), f2 = f1 + x2r;
-  if (math_utils::detail::abs(f1) > constants::math::Almost1 || math_utils::detail::abs(f2) > constants::math::Almost1) {
+  if (gpu::CAMath::Abs(f1) > constants::math::Almost1 || gpu::CAMath::Abs(f2) > constants::math::Almost1) {
     return false;
   }
-  value_t r1 = math_utils::detail::sqrt((1.f - f1) * (1.f + f1));
-  if (math_utils::detail::abs(r1) < constants::math::Almost0) {
+  value_t r1 = gpu::CAMath::Sqrt((1.f - f1) * (1.f + f1));
+  if (gpu::CAMath::Abs(r1) < constants::math::Almost0) {
     return false;
   }
-  value_t r2 = math_utils::detail::sqrt((1.f - f2) * (1.f + f2));
-  if (math_utils::detail::abs(r2) < constants::math::Almost0) {
+  value_t r2 = gpu::CAMath::Sqrt((1.f - f2) * (1.f + f2));
+  if (gpu::CAMath::Abs(r2) < constants::math::Almost0) {
     return false;
   }
   value_t dy2dx = (f1 + f2) / (r1 + r2);
-  value_t step = (math_utils::detail::abs(x2r) < 0.05f) ? dx * math_utils::detail::abs(r2 + f2 * dy2dx)                                              // chord
-                                                                 : 2.f * CAMath::ASin(0.5f * dx * math_utils::detail::sqrt(1.f + dy2dx * dy2dx) * crv) / crv; // arc
-  step *= math_utils::detail::sqrt(1.f + getTgl() * getTgl());
+  value_t step = (gpu::CAMath::Abs(x2r) < 0.05f) ? dx * gpu::CAMath::Abs(r2 + f2 * dy2dx)                                              // chord
+                                                 : 2.f * CAMath::ASin(0.5f * dx * gpu::CAMath::Sqrt(1.f + dy2dx * dy2dx) * crv) / crv; // arc
+  step *= gpu::CAMath::Sqrt(1.f + getTgl() * getTgl());
   //
   // get the track x,y,z,px/p,py/p,pz/p,p,sinAlpha,cosAlpha in the Global System
-  gpu::gpustd::array vecLab{0.f};
+  std::array vecLab{0.f};
   if (!getPosDirGlo(vecLab)) {
     return false;
   }
 
   // rotate to the system where Bx=By=0.
   value_t bxy2 = b[0] * b[0] + b[1] * b[1];
-  value_t bt = math_utils::detail::sqrt(bxy2);
+  value_t bt = gpu::CAMath::Sqrt(bxy2);
   value_t cosphi = 1.f, sinphi = 0.f;
   if (bt > constants::math::Almost0) {
     cosphi = b[0] / bt;
     sinphi = b[1] / bt;
   }
-  value_t bb = math_utils::detail::sqrt(bxy2 + b[2] * b[2]);
+  value_t bb = gpu::CAMath::Sqrt(bxy2 + b[2] * b[2]);
   value_t costet = 1.f, sintet = 0.f;
   if (bb > constants::math::Almost0) {
     costet = b[2] / bb;
     sintet = bt / bb;
   }
-  gpu::gpustd::array vect{costet * cosphi * vecLab[0] + costet * sinphi * vecLab[1] - sintet * vecLab[2],
+  std::array vect{costet * cosphi * vecLab[0] + costet * sinphi * vecLab[1] - sintet * vecLab[2],
                                       -sinphi * vecLab[0] + cosphi * vecLab[1],
                                       sintet * cosphi * vecLab[0] + sintet * sinphi * vecLab[1] + costet * vecLab[2],
                                       costet * cosphi * vecLab[3] + costet * sinphi * vecLab[4] - sintet * vecLab[5],
@@ -294,8 +314,8 @@ GPUd() bool TrackParametrization::propagateParamTo(value_t xk, const di
 
   // Do the final correcting step to the target plane (linear approximation)
   value_t x = vecLab[0], y = vecLab[1], z = vecLab[2];
-  if (math_utils::detail::abs(dx) > constants::math::Almost0) {
-    if (math_utils::detail::abs(vecLab[3]) < constants::math::Almost0) {
+  if (gpu::CAMath::Abs(dx) > constants::math::Almost0) {
+    if (gpu::CAMath::Abs(vecLab[3]) < constants::math::Almost0) {
       return false;
     }
     dx = xk - vecLab[0];
@@ -305,7 +325,7 @@ GPUd() bool TrackParametrization::propagateParamTo(value_t xk, const di
   }
 
   // Calculate the track parameters
-  t = 1.f / math_utils::detail::sqrt(vecLab[3] * vecLab[3] + vecLab[4] * vecLab[4]);
+  t = 1.f / gpu::CAMath::Sqrt(vecLab[3] * vecLab[3] + vecLab[4] * vecLab[4]);
   mX = xk;
   mP[kY] = y;
   mP[kZ] = z;
@@ -326,30 +346,26 @@ GPUd() bool TrackParametrization::propagateParamTo(value_t xk, value_t
   // distances only ((dx) < constants::math::Almost0) {
+  if (gpu::CAMath::Abs(dx) < constants::math::Almost0) {
     return true;
   }
-  value_t crv = (math_utils::detail::abs(b) < constants::math::Almost0) ? 0.f : getCurvature(b);
+  value_t crv = (gpu::CAMath::Abs(b) < constants::math::Almost0) ? 0.f : getCurvature(b);
   value_t x2r = crv * dx;
   value_t f1 = getSnp(), f2 = f1 + x2r;
-  if ((math_utils::detail::abs(f1) > constants::math::Almost1) || (math_utils::detail::abs(f2) > constants::math::Almost1)) {
+  if ((gpu::CAMath::Abs(f1) > constants::math::Almost1) || (gpu::CAMath::Abs(f2) > constants::math::Almost1)) {
     return false;
   }
-  value_t r1 = math_utils::detail::sqrt((1.f - f1) * (1.f + f1));
-  if (math_utils::detail::abs(r1) < constants::math::Almost0) {
+  value_t r1 = gpu::CAMath::Sqrt((1.f - f1) * (1.f + f1));
+  if (gpu::CAMath::Abs(r1) < constants::math::Almost0) {
     return false;
   }
-  value_t r2 = math_utils::detail::sqrt((1.f - f2) * (1.f + f2));
-  if (math_utils::detail::abs(r2) < constants::math::Almost0) {
+  value_t r2 = gpu::CAMath::Sqrt((1.f - f2) * (1.f + f2));
+  if (gpu::CAMath::Abs(r2) < constants::math::Almost0) {
     return false;
   }
-  mX = xk;
   double dy2dx = (f1 + f2) / (r1 + r2);
-  mP[kY] += dx * dy2dx;
-  mP[kSnp] += x2r;
-  if (math_utils::detail::abs(x2r) < 0.05f) {
-    mP[kZ] += dx * (r2 + f2 * dy2dx) * getTgl();
-  } else {
+  bool arcz = gpu::CAMath::Abs(x2r) > 0.05f;
+  if (arcz) {
     // for small dx/R the linear apporximation of the arc by the segment is OK,
     // but at large dx/R the error is very large and leads to incorrect Z propagation
     // angle traversed delta = 2*asin(dist_start_end / R / 2), hence the arc is: R*deltaPhi
@@ -358,7 +374,11 @@ GPUd() bool TrackParametrization::propagateParamTo(value_t xk, value_t
     //    double rot = 2*TMath::ASin(0.5*chord*crv); // angular difference seen from the circle center
     //    track1 += rot/crv*track3;
     //
-    value_t rot = CAMath::ASin(r1 * f2 - r2 * f1);  // more economic version from Yura.
+    auto arg = r1 * f2 - r2 * f1;
+    if (gpu::CAMath::Abs(arg) > constants::math::Almost1) {
+      return false;
+    }
+    value_t rot = CAMath::ASin(arg);                // more economic version from Yura.
     if (f1 * f1 + f2 * f2 > 1.f && f1 * f2 < 0.f) { // special cases of large rotations or large abs angles
       if (f2 > 0.f) {
         rot = constants::math::PI - rot; //
@@ -367,7 +387,12 @@ GPUd() bool TrackParametrization::propagateParamTo(value_t xk, value_t
       }
     }
     mP[kZ] += getTgl() / crv * rot;
+  } else {
+    mP[kZ] += dx * (r2 + f2 * dy2dx) * getTgl();
   }
+  mX = xk;
+  mP[kY] += dx * dy2dx;
+  mP[kSnp] += x2r;
   return true;
 }
 
@@ -378,27 +403,31 @@ GPUd() bool TrackParametrization::propagateParamToDCA(const math_utils:
   // propagate track to DCA to the vertex
   value_t sn, cs, alp = getAlpha();
   math_utils::detail::sincos(alp, sn, cs);
-  value_t x = getX(), y = getY(), snp = getSnp(), csp = math_utils::detail::sqrt((1.f - snp) * (1.f + snp));
+  value_t x = getX(), y = getY(), snp = getSnp(), csp = gpu::CAMath::Sqrt((1.f - snp) * (1.f + snp));
   value_t xv = vtx.X() * cs + vtx.Y() * sn, yv = -vtx.X() * sn + vtx.Y() * cs, zv = vtx.Z();
   x -= xv;
   y -= yv;
   // Estimate the impact parameter neglecting the track curvature
-  value_t d = math_utils::detail::abs(x * snp - y * csp);
+  value_t d = gpu::CAMath::Abs(x * snp - y * csp);
   if (d > maxD) {
+    if (dca) { // provide default DCA for failed propag
+      (*dca)[0] = o2::track::DefaultDCA;
+      (*dca)[1] = o2::track::DefaultDCA;
+    }
     return false;
   }
   value_t crv = getCurvature(b);
   value_t tgfv = -(crv * x - snp) / (crv * y + csp);
-  sn = tgfv / math_utils::detail::sqrt(1.f + tgfv * tgfv);
-  cs = math_utils::detail::sqrt((1.f - sn) * (1.f + sn));
-  cs = (math_utils::detail::abs(tgfv) > constants::math::Almost0) ? sn / tgfv : constants::math::Almost1;
+  sn = tgfv / gpu::CAMath::Sqrt(1.f + tgfv * tgfv);
+  cs = gpu::CAMath::Sqrt((1.f - sn) * (1.f + sn));
+  cs = (gpu::CAMath::Abs(tgfv) > constants::math::Almost0) ? sn / tgfv : constants::math::Almost1;
 
   x = xv * cs + yv * sn;
   yv = -xv * sn + yv * cs;
   xv = x;
 
   auto tmpT(*this); // operate on the copy to recover after the failure
-  alp += math_utils::detail::asin(sn);
+  alp += gpu::CAMath::ASin(sn);
   if (!tmpT.rotateParam(alp) || !tmpT.propagateParamTo(xv, b)) {
 #ifndef GPUCA_ALIGPUCODE
     LOG(debug) << "failed to propagate to alpha=" << alp << " X=" << xv << " for vertex "
@@ -406,6 +435,10 @@ GPUd() bool TrackParametrization::propagateParamToDCA(const math_utils:
 #else
     LOG(debug) << "failed to propagate to alpha=" << alp << " X=" << xv << " for vertex " << vtx.X() << ' ' << vtx.Y() << ' ' << vtx.Z();
 #endif
+    if (dca) { // provide default DCA for failed propag
+      (*dca)[0] = o2::track::DefaultDCA;
+      (*dca)[1] = o2::track::DefaultDCA;
+    }
     return false;
   }
   *this = tmpT;
@@ -426,26 +459,26 @@ GPUd() bool TrackParametrization::getYZAt(value_t xk, value_t b, value_
   value_t dx = xk - getX();
   y = mP[kY];
   z = mP[kZ];
-  if (math_utils::detail::abs(dx) < constants::math::Almost0) {
+  if (gpu::CAMath::Abs(dx) < constants::math::Almost0) {
     return true;
   }
   value_t crv = getCurvature(b);
   value_t x2r = crv * dx;
   value_t f1 = getSnp(), f2 = f1 + x2r;
-  if ((math_utils::detail::abs(f1) > constants::math::Almost1) || (math_utils::detail::abs(f2) > constants::math::Almost1)) {
+  if ((gpu::CAMath::Abs(f1) > constants::math::Almost1) || (gpu::CAMath::Abs(f2) > constants::math::Almost1)) {
     return false;
   }
-  value_t r1 = math_utils::detail::sqrt((1.f - f1) * (1.f + f1));
-  if (math_utils::detail::abs(r1) < constants::math::Almost0) {
+  value_t r1 = gpu::CAMath::Sqrt((1.f - f1) * (1.f + f1));
+  if (gpu::CAMath::Abs(r1) < constants::math::Almost0) {
     return false;
   }
-  value_t r2 = math_utils::detail::sqrt((1.f - f2) * (1.f + f2));
-  if (math_utils::detail::abs(r2) < constants::math::Almost0) {
+  value_t r2 = gpu::CAMath::Sqrt((1.f - f2) * (1.f + f2));
+  if (gpu::CAMath::Abs(r2) < constants::math::Almost0) {
     return false;
   }
   double dy2dx = (f1 + f2) / (r1 + r2);
   y += dx * dy2dx;
-  if (math_utils::detail::abs(x2r) < 0.05f) {
+  if (gpu::CAMath::Abs(x2r) < 0.05f) {
     z += dx * (r2 + f2 * dy2dx) * getTgl();
   } else {
     // for small dx/R the linear apporximation of the arc by the segment is OK,
@@ -502,6 +535,80 @@ GPUd() typename TrackParametrization::value_t TrackParametrization
+GPUd() typename TrackParametrization::value_t TrackParametrization::getSnpAt(value_t xk, value_t b) const
+{
+  ///< this method is just an alias for obtaining snp @ X in the tree->Draw()
+  value_t dx = xk - getX();
+  if (gpu::CAMath::Abs(dx) < constants::math::Almost0) {
+    return getSnp();
+  }
+  value_t crv = (gpu::CAMath::Abs(b) < constants::math::Almost0) ? 0.f : getCurvature(b);
+  value_t x2r = crv * dx;
+  return mP[kSnp] + x2r;
+}
+
+//______________________________________________________________
+template 
+GPUd() typename TrackParametrization::value_t TrackParametrization::getPhiAt(value_t xk, value_t b) const
+{
+  ///< this method is just an alias for obtaining phi @ X in the tree->Draw()
+  value_t dx = xk - getX();
+  if (gpu::CAMath::Abs(dx) < constants::math::Almost0) {
+    return getPhi();
+  }
+  value_t crv = (gpu::CAMath::Abs(b) < constants::math::Almost0) ? 0.f : getCurvature(b);
+  value_t x2r = crv * dx;
+  value_t snp = mP[kSnp] + x2r;
+  value_t phi = 999.;
+  if (gpu::CAMath::Abs(snp) < constants::math::Almost1) {
+    phi = gpu::CAMath::ASin(snp) + getAlpha();
+    math_utils::detail::bringTo02Pi(phi);
+  }
+  return phi;
+}
+
+//______________________________________________________________
+template 
+GPUd() typename TrackParametrization::value_t TrackParametrization::getPhiPosAt(value_t xk, value_t b) const
+{
+  ///< this method is just an alias for obtaining phiPos @ X in the tree->Draw()
+  value_t phi = 999.;
+  auto y = getYAt(xk, b);
+  if (y > -9998.) {
+    phi = gpu::CAMath::ATan2(y, xk) + getAlpha();
+    math_utils::detail::bringTo02Pi(phi);
+  }
+  return phi;
+}
+
+//______________________________________________________________
+template 
+GPUd() typename TrackParametrization::value_t TrackParametrization::getSnpAt(value_t alpha, value_t xk, value_t b) const
+{
+  ///< this method is just an alias for obtaining snp @ alpha, X in the tree->Draw()
+  math_utils::detail::bringToPMPi(alpha);
+  value_t ca = 0, sa = 0;
+  math_utils::detail::sincos(alpha - getAlpha(), sa, ca);
+  value_t snp = getSnp(), csp = gpu::CAMath::Sqrt((1.f - snp) * (1.f + snp)); // Improve precision
+  // RS: check if rotation does no invalidate track model (cos(local_phi)>=0, i.e. particle direction in local frame is along the X axis
+  if ((csp * ca + snp * sa) < 0.) {
+    // LOGF(warning,"Rotation failed: local cos(phi) would become {:.2f}", csp * ca + snp * sa);
+    return -999;
+  }
+  value_t tmp = snp * ca - csp * sa;
+  if (gpu::CAMath::Abs(tmp) > constants::math::Almost1) {
+    LOGP(debug, "Rotation failed: new snp {:.2f}", tmp);
+    return -999;
+  }
+  value_t xrot = getX() * ca + getY() * sa;
+  value_t dx = xk - xrot;
+  value_t crv = (gpu::CAMath::Abs(b) < constants::math::Almost0) ? 0.f : getCurvature(b);
+  value_t x2r = crv * dx;
+  return tmp + x2r;
+}
+
 #ifndef GPUCA_ALIGPUCODE
 //_____________________________________________________________
 template 
@@ -511,6 +618,31 @@ std::string TrackParametrization::asString() const
   return fmt::format("X:{:+.4e} Alp:{:+.3e} Par: {:+.4e} {:+.4e} {:+.4e} {:+.4e} {:+.4e} |Q|:{:d} {:s}",
                      getX(), getAlpha(), getY(), getZ(), getSnp(), getTgl(), getQ2Pt(), getAbsCharge(), getPID().getName());
 }
+
+//_____________________________________________________________
+template 
+std::string TrackParametrization::asStringHexadecimal()
+{
+  auto _X = getX();
+  auto _Alpha = getAlpha();
+  auto _Y = getY();
+  auto _Z = getZ();
+  auto _Snp = getSnp();
+  auto _Tgl = getTgl();
+  float _Q2Pt = getQ2Pt();
+  float _AbsCharge = getAbsCharge();
+  // print parameters as string
+  return fmt::format("X:{:x} Alp:{:x} Par: {:x} {:x} {:x} {:x} {:x} |Q|:{:x} {:s}\n",
+                     reinterpret_cast(_X),
+                     reinterpret_cast(_Alpha),
+                     reinterpret_cast(_Y),
+                     reinterpret_cast(_Z),
+                     reinterpret_cast(_Snp),
+                     reinterpret_cast(_Tgl),
+                     reinterpret_cast(_Q2Pt),
+                     reinterpret_cast(_AbsCharge),
+                     getPID().getName());
+}
 #endif
 
 //______________________________________________________________
@@ -520,6 +652,30 @@ GPUd() void TrackParametrization::printParam() const
   // print parameters
 #ifndef GPUCA_ALIGPUCODE
   printf("%s\n", asString().c_str());
+#elif !defined(GPUCA_GPUCODE_DEVICE) || (!defined(__OPENCL__) && defined(GPUCA_GPU_DEBUG_PRINT))
+  printf("X:%+.4e Alp:%+.3e Par: %+.4e %+.4e %+.4e %+.4e %+.4e |Q|:%d %s\n",
+         getX(), getAlpha(), getY(), getZ(), getSnp(), getTgl(), getQ2Pt(), getAbsCharge(), getPID().getName());
+#endif
+}
+
+//______________________________________________________________
+template 
+GPUd() void TrackParametrization::printParamHexadecimal()
+{
+  // print parameters
+#ifndef GPUCA_ALIGPUCODE
+  printf("%s\n", asStringHexadecimal().c_str());
+#elif !defined(GPUCA_GPUCODE_DEVICE) || (!defined(__OPENCL__) && defined(GPUCA_GPU_DEBUG_PRINT))
+  printf("X:%x Alp:%x Par: %x %x %x %x %x |Q|:%x %s\n",
+         gpu::CAMath::Float2UIntReint(getX()),
+         gpu::CAMath::Float2UIntReint(getAlpha()),
+         gpu::CAMath::Float2UIntReint(getY()),
+         gpu::CAMath::Float2UIntReint(getZ()),
+         gpu::CAMath::Float2UIntReint(getSnp()),
+         gpu::CAMath::Float2UIntReint(getTgl()),
+         gpu::CAMath::Float2UIntReint(getQ2Pt()),
+         gpu::CAMath::Float2UIntReint(getAbsCharge()),
+         getPID().getName());
 #endif
 }
 
@@ -535,58 +691,66 @@ GPUd() bool TrackParametrization::getXatLabR(value_t r, value_t& x, val
   // DirOutward (==1) - go along the track (increasing mX)
   // DirInward (==-1) - go backward (decreasing mX)
   //
-  const auto fy = mP[0], sn = mP[2];
+  const double fy = mP[0], sn = mP[2];
   const value_t kEps = 1.e-6;
   //
+  if (gpu::CAMath::Abs(getSnp()) > constants::math::Almost1) {
+    return false;
+  }
   auto crv = getCurvature(bz);
-  if (math_utils::detail::abs(crv) > constants::math::Almost0) { // helix
+  while (gpu::CAMath::Abs(crv) > constants::math::Almost0) { // helix ?
     // get center of the track circle
-    math_utils::CircleXY circle;
+    math_utils::CircleXY circle{};
     getCircleParamsLoc(bz, circle);
-    value_t r0 = math_utils::detail::sqrt(circle.getCenterD2());
+    if (circle.rC == 0.) {
+      crv = 0.;
+      break;
+    }
+    value_t r0 = gpu::CAMath::Sqrt(circle.getCenterD2());
     if (r0 <= constants::math::Almost0) {
       return false; // the track is concentric to circle
     }
-    value_t tR2r0 = 1.f, g = 0.f, tmp = 0.f;
-    if (math_utils::detail::abs(circle.rC - r0) > kEps) {
+    double tR2r0 = 1., g = 0., tmp = 0.;
+    if (gpu::CAMath::Abs(circle.rC - r0) > kEps) {
       tR2r0 = circle.rC / r0;
       g = 0.5f * (r * r / (r0 * circle.rC) - tR2r0 - 1.f / tR2r0);
       tmp = 1.f + g * tR2r0;
     } else {
       tR2r0 = 1.0;
-      g = 0.5f * r * r / (r0 * circle.rC) - 1.f;
-      tmp = 0.5f * r * r / (r0 * r0);
+      g = 0.5 * r * r / (r0 * circle.rC) - 1.;
+      tmp = 0.5 * r * r / (r0 * r0);
     }
-    value_t det = (1.f - g) * (1.f + g);
-    if (det < 0.f) {
+    auto det = (1. - g) * (1. + g);
+    if (det < 0.) {
       return false; // does not reach raduis r
     }
-    det = math_utils::detail::sqrt(det);
+    det = gpu::CAMath::Sqrt(det);
     //
     // the intersection happens in 2 points: {circle.xC+tR*C,circle.yC+tR*S}
     // with C=f*c0+-|s0|*det and S=f*s0-+c0 sign(s0)*det
     // where s0 and c0 make direction for the circle center (=circle.xC/r0 and circle.yC/r0)
     //
     x = circle.xC * tmp;
-    value_t y = circle.yC * tmp;
-    if (math_utils::detail::abs(circle.yC) > constants::math::Almost0) { // when circle.yC==0 the x,y is unique
-      value_t dfx = tR2r0 * math_utils::detail::abs(circle.yC) * det;
-      value_t dfy = tR2r0 * circle.xC * (circle.yC > 0.f ? det : -det);
+    auto y = circle.yC * tmp;
+    if (gpu::CAMath::Abs(circle.yC) > constants::math::Almost0) { // when circle.yC==0 the x,y is unique
+      auto dfx = tR2r0 * gpu::CAMath::Abs(circle.yC) * det;
+      auto dfy = tR2r0 * circle.xC * (circle.yC > 0. ? det : -det);
       if (dir == DirAuto) {                              // chose the one which corresponds to smallest step
-        value_t delta = (x - mX) * dfx - (y - fy) * dfy; // the choice of + in C will lead to smaller step if delta<0
-        x += delta < 0.f ? dfx : -dfx;
+        auto delta = (x - mX) * dfx - (y - fy) * dfy;    // the choice of + in C will lead to smaller step if delta<0
+        x += delta < 0. ? dfx : -dfx;
       } else if (dir == DirOutward) { // along track direction: x must be > mX
         x -= dfx;                     // try the smallest step (dfx is positive)
-        value_t dfeps = mX - x;       // handle special case of very small step
+        auto dfeps = mX - x;          // handle special case of very small step
         if (dfeps < -kEps) {
           return true;
         }
-        if (math_utils::detail::abs(dfeps) < kEps && math_utils::detail::abs(mX * mX + fy * fy - r * r) < kEps) { // are we already in right r?
-          return mX;
+        if (gpu::CAMath::Abs(dfeps) < kEps && gpu::CAMath::Abs(mX * mX + fy * fy - r * r) < kEps) { // are we already in right r?
+          x = mX;
+          return true;
         }
         x += dfx + dfx;
-        value_t dxm = x - mX;
-        if (dxm > 0.f) {
+        auto dxm = x - mX;
+        if (dxm > 0.) {
           return true;
         } else if (dxm < -kEps) {
           return false;
@@ -594,16 +758,17 @@ GPUd() bool TrackParametrization::getXatLabR(value_t r, value_t& x, val
         x = mX;                 // don't move
       } else {                  // backward: x must be < mX
         x += dfx;               // try the smallest step (dfx is positive)
-        value_t dfeps = x - mX; // handle special case of very small step
+        auto dfeps = x - mX;    // handle special case of very small step
         if (dfeps < -kEps) {
           return true;
         }
-        if (math_utils::detail::abs(dfeps) < kEps && math_utils::detail::abs(mX * mX + fy * fy - r * r) < kEps) { // are we already in right r?
-          return mX;
+        if (gpu::CAMath::Abs(dfeps) < kEps && gpu::CAMath::Abs(mX * mX + fy * fy - r * r) < kEps) { // are we already in right r?
+          x = mX;
+          return true;
         }
         x -= dfx + dfx;
-        value_t dxm = x - mX;
-        if (dxm < 0.f) {
+        auto dxm = x - mX;
+        if (dxm < 0.) {
           return true;
         }
         if (dxm > kEps) {
@@ -616,85 +781,86 @@ GPUd() bool TrackParametrization::getXatLabR(value_t r, value_t& x, val
         return false;
       }
     }
-  } else {                                                                  // this is a straight track
-    if (math_utils::detail::abs(sn) >= constants::math::Almost1) { // || to Y axis
-      value_t det = (r - mX) * (r + mX);
-      if (det < 0.f) {
-        return false; // does not reach raduis r
-      }
-      x = mX;
-      if (dir == DirAuto) {
-        return true;
-      }
-      det = math_utils::detail::sqrt(det);
-      if (dir == DirOutward) { // along the track direction
-        if (sn > 0.f) {
-          if (fy > det) {
-            return false; // track is along Y axis and above the circle
-          }
-        } else {
-          if (fy < -det) {
-            return false; // track is against Y axis amd belo the circle
-          }
+    return true;
+  }
+  // this is a straight track
+  if (gpu::CAMath::Abs(sn) >= constants::math::Almost1) { // || to Y axis
+    double det = (r - mX) * (r + mX);
+    if (det < 0.f) {
+      return false; // does not reach raduis r
+    }
+    x = mX;
+    if (dir == DirAuto) {
+      return true;
+    }
+    det = gpu::CAMath::Sqrt(det);
+    if (dir == DirOutward) { // along the track direction
+      if (sn > 0.) {
+        if (fy > det) {
+          return false; // track is along Y axis and above the circle
         }
-      } else if (dir == DirInward) { // against track direction
-        if (sn > 0.f) {
-          if (fy < -det) {
-            return false; // track is along Y axis
-          }
-        } else if (fy > det) {
-          return false; // track is against Y axis
+      } else {
+        if (fy < -det) {
+          return false; // track is against Y axis amd belo the circle
         }
       }
-    } else if (math_utils::detail::abs(sn) <= constants::math::Almost0) { // || to X axis
-      value_t det = (r - fy) * (r + fy);
-      if (det < 0.f) {
-        return false; // does not reach raduis r
-      }
-      det = math_utils::detail::sqrt(det);
-      if (dir == DirAuto) {
-        x = mX > 0.f ? det : -det; // choose the solution requiring the smalest step
-        return true;
-      } else if (dir == DirOutward) { // along the track direction
-        if (mX > det) {
-          return false; // current point is in on the right from the circle
-        } else {
-          x = (mX < -det) ? -det : det; // on the left : within the circle
-        }
-      } else { // against the track direction
-        if (mX < -det) {
-          return false;
-        } else {
-          x = mX > det ? det : -det;
+    } else if (dir == DirInward) { // against track direction
+      if (sn > 0.) {
+        if (fy < -det) {
+          return false; // track is along Y axis
         }
+      } else if (fy > det) {
+        return false; // track is against Y axis
+      }
+    }
+  } else if (gpu::CAMath::Abs(sn) <= constants::math::Almost0) { // || to X axis
+    double det = (r - fy) * (r + fy);
+    if (det < 0.) {
+      return false; // does not reach raduis r
+    }
+    det = gpu::CAMath::Sqrt(det);
+    if (dir == DirAuto) {
+      x = mX > 0. ? det : -det; // choose the solution requiring the smalest step
+      return true;
+    } else if (dir == DirOutward) { // along the track direction
+      if (mX > det) {
+        return false; // current point is in on the right from the circle
+      } else {
+        x = (mX < -det) ? -det : det; // on the left : within the circle
       }
-    } else { // general case of straight line
-      value_t cs = math_utils::detail::sqrt((1.f - sn) * (1.f + sn));
-      value_t xsyc = mX * sn - fy * cs;
-      value_t det = (r - xsyc) * (r + xsyc);
-      if (det < 0.f) {
-        return false; // does not reach raduis r
+    } else { // against the track direction
+      if (mX < -det) {
+        return false;
+      } else {
+        x = mX > det ? det : -det;
       }
-      det = math_utils::detail::sqrt(det);
-      value_t xcys = mX * cs + fy * sn;
-      value_t t = -xcys;
-      if (dir == DirAuto) {
-        t += t > 0.f ? -det : det; // chose the solution requiring the smalest step
-      } else if (dir > 0) {        // go in increasing mX direction. ( t+-det > 0)
-        if (t >= -det) {
-          t += det; // take minimal step giving t>0
-        } else {
-          return false; // both solutions have negative t
-        }
-      } else { // go in decreasing mX direction. (t+-det < 0)
-        if (t < det) {
-          t -= det; // take minimal step giving t<0
-        } else {
-          return false; // both solutions have positive t
-        }
+    }
+  } else { // general case of straight line
+    auto cs = gpu::CAMath::Sqrt((1. - sn) * (1. + sn));
+    auto xsyc = mX * sn - fy * cs;
+    auto det = (r - xsyc) * (r + xsyc);
+    if (det < 0.) {
+      return false; // does not reach raduis r
+    }
+    det = gpu::CAMath::Sqrt(det);
+    auto xcys = mX * cs + fy * sn;
+    auto t = -xcys;
+    if (dir == DirAuto) {
+      t += t > 0. ? -det : det;  // chose the solution requiring the smalest step
+    } else if (dir > 0) {        // go in increasing mX direction. ( t+-det > 0)
+      if (t >= -det) {
+        t += det; // take minimal step giving t>0
+      } else {
+        return false; // both solutions have negative t
+      }
+    } else { // go in decreasing mX direction. (t+-det < 0)
+      if (t < det) {
+        t -= det; // take minimal step giving t<0
+      } else {
+        return false; // both solutions have positive t
       }
-      x = mX + cs * t;
     }
+    x = mX + cs * t;
   }
   //
   return true;
@@ -702,7 +868,7 @@ GPUd() bool TrackParametrization::getXatLabR(value_t r, value_t& x, val
 
 //______________________________________________
 template 
-GPUd() bool TrackParametrization::correctForELoss(value_t xrho, bool anglecorr, value_t dedx)
+GPUd() bool TrackParametrization::correctForELoss(value_t xrho, bool anglecorr)
 {
   //------------------------------------------------------------------
   // This function corrects the track parameters for the energy loss in crossed material.
@@ -712,41 +878,77 @@ GPUd() bool TrackParametrization::correctForELoss(value_t xrho, bool an
   // "dedx" - mean enery loss (GeV/(g/cm^2), if <=kCalcdEdxAuto : calculate on the fly
   // "anglecorr" - switch for the angular correction
   //------------------------------------------------------------------
-  constexpr value_t kMaxELossFrac = 0.3f; // max allowed fractional eloss
-  constexpr value_t kMinP = 0.01f;        // kill below this momentum
-
-  // Apply angle correction, if requested
-  if (anglecorr) {
-    value_t csp2 = (1.f - getSnp()) * (1.f + getSnp()); // cos(phi)^2
-    value_t cst2I = (1.f + getTgl() * getTgl());        // 1/cos(lambda)^2
-    value_t angle = math_utils::detail::sqrt(cst2I / (csp2));
-    xrho *= angle;
-  }
-  value_t p = getP();
-  value_t p2 = p * p;
-  value_t e2 = p2 + getPID().getMass2();
-  value_t beta2 = p2 / e2;
-
-  // Calculating the energy loss corrections************************
-  if ((xrho != 0.f) && (beta2 < 1.f)) {
-    if (dedx < kCalcdEdxAuto + constants::math::Almost1) { // request to calculate dedx on the fly
-      dedx = BetheBlochSolid(p / getPID().getMass());
-      if (mAbsCharge != 1) {
-        dedx *= mAbsCharge * mAbsCharge;
-      }
+  constexpr value_t kMinP = 0.01f; // kill below this momentum
+
+  auto m = getPID().getMass();
+  if (m > 0 && xrho != 0.f) {
+    // Apply angle correction, if requested
+    if (anglecorr) {
+      value_t csp2 = (1.f - getSnp()) * (1.f + getSnp()); // cos(phi)^2
+      value_t cst2I = (1.f + getTgl() * getTgl());        // 1/cos(lambda)^2
+      value_t angle = gpu::CAMath::Sqrt(cst2I / (csp2));
+      xrho *= angle;
+    }
+    int charge2 = getAbsCharge() * getAbsCharge();
+    value_t p = getP(), p0 = p, p2 = p * p, e2 = p2 + getPID().getMass2(), massInv = 1. / m, bg = p * massInv;
+    value_t e = gpu::CAMath::Sqrt(e2), ekin = e - m, dedx = getdEdxBBOpt(bg);
+#ifdef _BB_NONCONST_CORR_
+    value_t dedxDer = 0., dedx1 = dedx;
+#endif
+    if (charge2 != 1) {
+      dedx *= charge2;
     }
-
     value_t dE = dedx * xrho;
-    value_t e = math_utils::detail::sqrt(e2);
-    if (math_utils::detail::abs(dE) > kMaxELossFrac * e) {
-      return false; // 30% energy loss is too much!
+    int na = 1 + int(gpu::CAMath::Abs(dE) / ekin * ELoss2EKinThreshInv);
+    if (na > MaxELossIter) {
+      na = MaxELossIter;
+    }
+    if (na > 1) {
+      dE /= na;
+      xrho /= na;
+#ifdef _BB_NONCONST_CORR_
+      dedxDer = getBetheBlochSolidDerivativeApprox(dedx1, bg); // require correction for non-constantness of dedx vs betagamma
+      if (charge2 != 1) {
+        dedxDer *= charge2;
+      }
+#endif
+    }
+    while (na--) {
+#ifdef _BB_NONCONST_CORR_
+      if (dedxDer != 0.) { // correction for non-constantness of dedx vs beta*gamma (in linear approximation): for a single step dE -> dE * [(exp(dedxDer) - 1)/dedxDer]
+        if (xrho < 0) {
+          dedxDer = -dedxDer; // E.loss ( -> positive derivative)
+        }
+        auto corrC = (gpu::CAMath::Exp(dedxDer) - 1.) / dedxDer;
+        dE *= corrC;
+      }
+#endif
+      e += dE;
+      if (e > m) { // stopped
+        p = gpu::CAMath::Sqrt(e * e - getPID().getMass2());
+      } else {
+        return false;
+      }
+      if (na) {
+        bg = p * massInv;
+        dedx = getdEdxBBOpt(bg);
+#ifdef _BB_NONCONST_CORR_
+        dedxDer = getBetheBlochSolidDerivativeApprox(dedx, bg);
+#endif
+        if (charge2 != 1) {
+          dedx *= charge2;
+#ifdef _BB_NONCONST_CORR_
+          dedxDer *= charge2;
+#endif
+        }
+        dE = dedx * xrho;
+      }
     }
-    value_t eupd = e + dE;
-    value_t pupd2 = eupd * eupd - getPID().getMass2();
-    if (pupd2 < kMinP * kMinP) {
+
+    if (p < kMinP) {
       return false;
     }
-    setQ2Pt(getQ2Pt() * p / math_utils::detail::sqrt(pupd2));
+    setQ2Pt(getQ2Pt() * p0 / p);
   }
 
   return true;
@@ -765,10 +967,30 @@ GPUd() typename o2::track::TrackParametrization::yzerr_t TrackParametri
           {v.getSigmaX2() * sn2 + dsxysncs + v.getSigmaY2() * cs2, (sn + cs) * v.getSigmaYZ(), v.getSigmaZ2()}};
 }
 
+//______________________________________________
+template 
+GPUd() typename TrackParametrization::value_t TrackParametrization::getDCAYtoMV(value_t b, value_t xmv, value_t ymv, value_t zmv) const
+{
+  auto ttmp = *this;
+  dim2_t dca;
+  return ttmp.propagateParamToDCA({xmv, ymv, zmv}, b, &dca) ? dca[0] : -9999.;
+}
+
+//______________________________________________
+template 
+GPUd() typename TrackParametrization::value_t TrackParametrization::getDCAZtoMV(value_t b, value_t xmv, value_t ymv, value_t zmv) const
+{
+  auto ttmp = *this;
+  dim2_t dca;
+  return ttmp.propagateParamToDCA({xmv, ymv, zmv}, b, &dca) ? dca[1] : -9999.;
+}
+
 namespace o2::track
 {
+#if !defined(GPUCA_GPUCODE) || defined(GPUCA_GPUCODE_DEVICE) // FIXME: DR: WORKAROUND to avoid CUDA bug creating host symbols for device code.
 template class TrackParametrization;
-#ifndef GPUCA_GPUCODE_DEVICE
+#endif
+#ifndef GPUCA_GPUCODE
 template class TrackParametrization;
 #endif
 } // namespace o2::track
diff --git a/DataFormats/Reconstruction/src/TrackParametrizationWithError.cxx b/DataFormats/Reconstruction/src/TrackParametrizationWithError.cxx
index d3a1c58a59dbd..2f8f15f783c60 100644
--- a/DataFormats/Reconstruction/src/TrackParametrizationWithError.cxx
+++ b/DataFormats/Reconstruction/src/TrackParametrizationWithError.cxx
@@ -9,32 +9,14 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 
-/// @file   TrackparametrizationWithError.cxx
-/// @author ruben.shahoyan@cern.ch, michael.lettrich@cern.ch
-/// @since  Oct 1, 2020
-/// @brief
-
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
 #include "ReconstructionDataFormats/TrackParametrizationWithError.h"
 #include "ReconstructionDataFormats/Vertex.h"
 #include "ReconstructionDataFormats/DCA.h"
+#include "CommonConstants/MathConstants.h"
 #include 
 
 #ifndef GPUCA_GPUCODE_DEVICE
 #include 
-#ifndef GPUCA_STANDALONE
-#include "Math/SMatrix.h"
-#endif
 #endif
 
 #ifndef GPUCA_ALIGPUCODE
@@ -61,7 +43,7 @@ GPUd() void TrackParametrizationWithError::invert()
 
 //______________________________________________________________
 template 
-GPUd() bool TrackParametrizationWithError::propagateTo(value_t xk, value_t b)
+GPUd() bool TrackParametrizationWithError::propagateTo(value_t xk, value_t bz)
 {
   //----------------------------------------------------------------
   // propagate this track to the plane X=xk (cm) in the field "b" (kG)
@@ -70,7 +52,7 @@ GPUd() bool TrackParametrizationWithError::propagateTo(value_t xk, valu
   if (gpu::CAMath::Abs(dx) < constants::math::Almost0) {
     return true;
   }
-  value_t crv = this->getCurvature(b);
+  value_t crv = this->getCurvature(bz);
   value_t x2r = crv * dx;
   value_t f1 = this->getSnp(), f2 = f1 + x2r;
   if ((gpu::CAMath::Abs(f1) > constants::math::Almost1) || (gpu::CAMath::Abs(f2) > constants::math::Almost1)) {
@@ -84,24 +66,25 @@ GPUd() bool TrackParametrizationWithError::propagateTo(value_t xk, valu
   if (gpu::CAMath::Abs(r2) < constants::math::Almost0) {
     return false;
   }
-  this->setX(xk);
-  double dy2dx = (f1 + f2) / (r1 + r2);
+  double r1pr2Inv = 1. / (r1 + r2);
+  double dy2dx = (f1 + f2) * r1pr2Inv;
+  bool arcz = gpu::CAMath::Abs(x2r) > 0.05f;
   params_t dP{0.f};
-  dP[kY] = dx * dy2dx;
-  dP[kSnp] = x2r;
-  if (gpu::CAMath::Abs(x2r) < 0.05f) {
-    dP[kZ] = dx * (r2 + f2 * dy2dx) * this->getTgl();
-  } else {
+  if (arcz) {
     // for small dx/R the linear apporximation of the arc by the segment is OK,
     // but at large dx/R the error is very large and leads to incorrect Z propagation
     // angle traversed delta = 2*asin(dist_start_end / R / 2), hence the arc is: R*deltaPhi
     // The dist_start_end is obtained from sqrt(dx^2+dy^2) = x/(r1+r2)*sqrt(2+f1*f2+r1*r2)
     //    double chord = dx*TMath::Sqrt(1+dy2dx*dy2dx);   // distance from old position to new one
     //    double rot = 2*TMath::ASin(0.5*chord*crv); // angular difference seen from the circle center
-    //    mP1 += rot/crv*mP3;
+    //    track1 += rot/crv*track3;
     //
-    value_t rot = gpu::CAMath::ASin(r1 * f2 - r2 * f1); // more economic version from Yura.
-    if (f1 * f1 + f2 * f2 > 1.f && f1 * f2 < 0.f) {     // special cases of large rotations or large abs angles
+    auto arg = r1 * f2 - r2 * f1;
+    if (gpu::CAMath::Abs(arg) > constants::math::Almost1) {
+      return false;
+    }
+    value_t rot = gpu::CAMath::ASin(arg);           // more economic version from Yura.
+    if (f1 * f1 + f2 * f2 > 1.f && f1 * f2 < 0.f) { // special cases of large rotations or large abs angles
       if (f2 > 0.f) {
         rot = constants::math::PI - rot; //
       } else {
@@ -109,7 +92,12 @@ GPUd() bool TrackParametrizationWithError::propagateTo(value_t xk, valu
       }
     }
     dP[kZ] = this->getTgl() / crv * rot;
+  } else {
+    dP[kZ] = dx * (r2 + f2 * dy2dx) * this->getTgl();
   }
+  this->setX(xk);
+  dP[kY] = dx * dy2dx;
+  dP[kSnp] = x2r;
 
   this->updateParams(dP); // apply corrections
 
@@ -119,14 +107,17 @@ GPUd() bool TrackParametrizationWithError::propagateTo(value_t xk, valu
           &c44 = mC[kSigQ2Pt2];
 
   // evaluate matrix in double prec.
-  double rinv = 1. / r1;
-  double r3inv = rinv * rinv * rinv;
-  double f24 = dx * b * constants::math::B2C; // x2r/mP[kQ2Pt];
-  double f02 = dx * r3inv;
-  double f04 = 0.5 * f24 * f02;
-  double f12 = f02 * this->getTgl() * f1;
-  double f14 = 0.5 * f24 * f12; // 0.5*f24*f02*getTgl()*f1;
-  double f13 = dx * rinv;
+  value_t kb = bz * constants::math::B2C;
+  double r2inv = 1. / r2, r1inv = 1. / r1;
+  double dx2r1pr2 = dx * r1pr2Inv;
+
+  double hh = dx2r1pr2 * r2inv * (1. + r1 * r2 + f1 * f2), jj = dx * (dy2dx - f2 * r2inv);
+  double f02 = hh * r1inv;
+  double f04 = hh * dx2r1pr2 * kb;
+  double f24 = dx * kb; // x2r/mP[kQ2Pt];
+  double f12 = this->getTgl() * (f02 * f2 + jj);
+  double f13 = dx * (r2 + f2 * dy2dx);
+  double f14 = this->getTgl() * (f04 * f2 + jj * f24);
 
   // b = C*ft
   double b00 = f02 * c20 + f04 * c40, b01 = f12 * c20 + f14 * c40 + f13 * c30;
@@ -164,6 +155,107 @@ GPUd() bool TrackParametrizationWithError::propagateTo(value_t xk, valu
   return true;
 }
 
+//______________________________________________________________
+template 
+GPUd() bool TrackParametrizationWithError::propagateTo(value_t xk, TrackParametrization& linRef0, value_t bz)
+{
+  //----------------------------------------------------------------
+  // propagate this track to the plane X=xk (cm) in the field "b" (kG), using linRef as linearization point
+  //----------------------------------------------------------------
+  if (this->getAbsCharge() == 0) {
+    bz = 0;
+  }
+  value_t dx = xk - this->getX();
+  if (gpu::CAMath::Abs(dx) < constants::math::Almost0) {
+    this->setX(xk);
+    linRef0.setX(xk);
+    return true;
+  }
+  // propagate reference track
+  TrackParametrization linRef1 = linRef0;
+  if (!linRef1.propagateTo(xk, bz)) {
+    return false;
+  }
+  value_t kb = bz * constants::math::B2C;
+  // evaluate in double prec.
+  double snpRef0 = linRef0.getSnp(), cspRef0 = gpu::CAMath::Sqrt((1 - snpRef0) * (1 + snpRef0));
+  double snpRef1 = linRef1.getSnp(), cspRef1 = gpu::CAMath::Sqrt((1 - snpRef1) * (1 + snpRef1));
+  double cspRef0Inv = 1 / cspRef0, cspRef1Inv = 1 / cspRef1, cc = cspRef0 + cspRef1, ccInv = 1 / cc, dy2dx = (snpRef0 + snpRef1) * ccInv;
+  double dxccInv = dx * ccInv, hh = dxccInv * cspRef1Inv * (1 + cspRef0 * cspRef1 + snpRef0 * snpRef1), jj = dx * (dy2dx - snpRef1 * cspRef1Inv);
+
+  double f02 = hh * cspRef0Inv;
+  double f04 = hh * dxccInv * kb;
+  double f24 = dx * kb;
+  double f12 = linRef0.getTgl() * (f02 * snpRef1 + jj);
+  double f13 = dx * (cspRef1 + snpRef1 * dy2dx); // dS
+  double f14 = linRef0.getTgl() * (f04 * snpRef1 + jj * f24);
+
+  // difference between the current and reference state
+  value_t diff[5];
+  for (int i = 0; i < 5; i++) {
+    diff[i] = this->getParam(i) - linRef0.getParam(i);
+  }
+  value_t snpUpd = snpRef1 + diff[kSnp] + f24 * diff[kQ2Pt];
+  if (gpu::CAMath::Abs(snpUpd) > constants::math::Almost1) {
+    return false;
+  }
+  linRef0 = linRef1; // update reference track
+  this->setX(xk);
+  this->setY(linRef1.getY() + diff[kY] + f02 * diff[kSnp] + f04 * diff[kQ2Pt]);
+  this->setZ(linRef1.getZ() + diff[kZ] + f13 * diff[kTgl] + f14 * diff[kQ2Pt]);
+  this->setSnp(snpUpd);
+  this->setTgl(linRef1.getTgl() + diff[kTgl]);
+  this->setQ2Pt(linRef1.getQ2Pt() + diff[kQ2Pt]);
+
+  value_t &c00 = mC[kSigY2], &c10 = mC[kSigZY], &c11 = mC[kSigZ2], &c20 = mC[kSigSnpY], &c21 = mC[kSigSnpZ],
+          &c22 = mC[kSigSnp2], &c30 = mC[kSigTglY], &c31 = mC[kSigTglZ], &c32 = mC[kSigTglSnp], &c33 = mC[kSigTgl2],
+          &c40 = mC[kSigQ2PtY], &c41 = mC[kSigQ2PtZ], &c42 = mC[kSigQ2PtSnp], &c43 = mC[kSigQ2PtTgl],
+          &c44 = mC[kSigQ2Pt2];
+
+  // b = C*ft
+  double b00 = f02 * c20 + f04 * c40, b01 = f12 * c20 + f14 * c40 + f13 * c30;
+  double b02 = f24 * c40;
+  double b10 = f02 * c21 + f04 * c41, b11 = f12 * c21 + f14 * c41 + f13 * c31;
+  double b12 = f24 * c41;
+  double b20 = f02 * c22 + f04 * c42, b21 = f12 * c22 + f14 * c42 + f13 * c32;
+  double b22 = f24 * c42;
+  double b40 = f02 * c42 + f04 * c44, b41 = f12 * c42 + f14 * c44 + f13 * c43;
+  double b42 = f24 * c44;
+  double b30 = f02 * c32 + f04 * c43, b31 = f12 * c32 + f14 * c43 + f13 * c33;
+  double b32 = f24 * c43;
+
+  // a = f*b = f*C*ft
+  double a00 = f02 * b20 + f04 * b40, a01 = f02 * b21 + f04 * b41, a02 = f02 * b22 + f04 * b42;
+  double a11 = f12 * b21 + f14 * b41 + f13 * b31, a12 = f12 * b22 + f14 * b42 + f13 * b32;
+  double a22 = f24 * b42;
+
+  // F*C*Ft = C + (b + bt + a)
+  c00 += b00 + b00 + a00;
+  c10 += b10 + b01 + a01;
+  c20 += b20 + b02 + a02;
+  c30 += b30;
+  c40 += b40;
+  c11 += b11 + b11 + a11;
+  c21 += b21 + b12 + a12;
+  c31 += b31;
+  c41 += b41;
+  c22 += b22 + b22 + a22;
+  c32 += b32;
+  c42 += b42;
+
+  checkCovariance();
+
+  return true;
+}
+
+//______________________________________________________________
+template 
+GPUd() bool TrackParametrizationWithError::testRotate(value_t) const
+{
+  // no ops
+  return true;
+}
+
 //______________________________________________________________
 template 
 GPUd() bool TrackParametrizationWithError::rotate(value_t alpha)
@@ -182,7 +274,7 @@ GPUd() bool TrackParametrizationWithError::rotate(value_t alpha)
   // RS: check if rotation does no invalidate track model (cos(local_phi)>=0, i.e. particle
   // direction in local frame is along the X axis
   if ((csp * ca + snp * sa) < 0) {
-    //LOGP(warning,"Rotation failed: local cos(phi) would become {:.2f}", csp * ca + snp * sa);
+    // LOGP(warning,"Rotation failed: local cos(phi) would become {:.2f}", csp * ca + snp * sa);
     return false;
   }
   //
@@ -219,6 +311,101 @@ GPUd() bool TrackParametrizationWithError::rotate(value_t alpha)
   return true;
 }
 
+//______________________________________________________________
+template 
+GPUd() bool TrackParametrizationWithError::rotate(value_t alpha, TrackParametrization& linRef0, value_t bz)
+{
+  // RS: similar to int32_t GPUTPCGMPropagator::RotateToAlpha(float newAlpha), i.e. rotate the track to new frame alpha, using linRef as linearization point
+  // rotate to alpha frame the reference (linearization point) trackParam, then align the current track to it
+  if (gpu::CAMath::Abs(this->getSnp()) > constants::math::Almost1) {
+    LOGP(debug, "Precondition is not satisfied: |sin(phi)|>1 ! {:f}", this->getSnp());
+    return false;
+  }
+  //
+  math_utils::detail::bringToPMPi(alpha);
+  //
+  value_t ca = 0, sa = 0;
+  TrackParametrization linRef1 = linRef0;
+  // rotate the reference, adjusting alpha to +-pi, return precalculated cos and sin of alpha - alphaOld
+  if (!linRef1.rotateParam(alpha, ca, sa)) {
+    return false;
+  }
+
+  value_t trackX = this->getX() * ca + this->getY() * sa; // X of the rotated current track
+  if (!linRef1.propagateParamTo(trackX, bz)) {
+    return false;
+  }
+
+  // now rotate the current track
+  value_t snp = this->getSnp(), csp = gpu::CAMath::Sqrt((1.f - snp) * (1.f + snp)), updSnp = snp * ca - csp * sa;
+  if ((csp * ca + snp * sa) < 0 || gpu::CAMath::Abs(updSnp) > constants::math::Almost1) {
+    // LOGP(warning,"Rotation failed: local cos(phi) would become {:.2f}", csp * ca + snp * sa);
+    return false;
+  }
+  this->setY(-sa * this->getX() + ca * this->getY());
+  this->setX(trackX);
+  this->setSnp(updSnp);
+  this->setAlpha(alpha);
+
+  // rotate covariance, accounting for the extra error from the rotated X
+  value_t snpRef0 = linRef0.getSnp(), cspRef0 = gpu::CAMath::Sqrt((value_t(1) - snpRef0) * (value_t(1) + snpRef0)); // original reference
+  value_t snpRef1 = linRef1.getSnp(), cspRef1 = ca * cspRef0 + sa * snpRef0;                                        // rotated reference
+  value_t rr = cspRef1 / cspRef0;                                                                                   // cos1_ref / cos0_ref
+
+  // "extra row" of the lower triangle of cov. matrix
+  value_t cXSigY = mC[kSigY2] * ca * sa;
+  value_t cXSigZ = mC[kSigZY] * sa;
+  value_t cXSigSnp = mC[kSigSnpY] * rr * sa;
+  value_t cXSigTgl = mC[kSigTglY] * sa;
+  value_t cXSigQ2Pt = mC[kSigQ2PtY] * sa;
+  value_t cSigX2 = mC[kSigY2] * sa * sa;
+
+  // plane rotation of existing cov matrix
+  mC[kSigY2] *= ca * ca;
+  mC[kSigZY] *= ca;
+  mC[kSigSnpY] *= ca * rr;
+  mC[kSigSnpZ] *= rr;
+  mC[kSigSnp2] *= rr * rr;
+  mC[kSigTglY] *= ca;
+  mC[kSigTglSnp] *= rr;
+  mC[kSigQ2PtY] *= ca;
+  mC[kSigQ2PtSnp] *= rr;
+
+  // transport covariance from pseudo 6x6 matrix to usual 5x5, Jacobian (trust to Sergey):
+  auto cspRef1Inv = value_t(1) / cspRef1;
+  auto j3 = -snpRef1 * cspRef1Inv;          // -pYmod/pXmod = -tg_pho = -sin_phi_mod / cos_phi_mod
+  auto j4 = -linRef1.getTgl() * cspRef1Inv; // -pZmod/pXmod = -tgl_mod / cos_phi_mod
+  auto j5 = linRef1.getCurvature(bz);
+  //       Y  Z Sin DzDs q/p  X
+  //  { {  1, 0, 0,  0,  0,  j3 }, // Y
+  //    {  0, 1, 0,  0,  0,  j4 }, // Z
+  //    {  0, 0, 1,  0,  0,  j5 }, // snp
+  //    {  0, 0, 0,  1,  0,   0 }, // tgl
+  //    {  0, 0, 0,  0,  1,   0 } }; // q/pt
+  auto hXSigY = cXSigY + cSigX2 * j3;
+  auto hXSigZ = cXSigZ + cSigX2 * j4;
+  auto hXSigSnp = cXSigSnp + cSigX2 * j5;
+
+  mC[kSigY2] += j3 * (cXSigY + hXSigY);
+  mC[kSigZ2] += j4 * (cXSigZ + hXSigZ);
+  mC[kSigSnpY] += cXSigSnp * j3 + hXSigY * j5;
+  mC[kSigSnp2] += j5 * (cXSigSnp + hXSigSnp);
+  mC[kSigTglZ] += cXSigTgl * j4;
+  mC[kSigQ2PtY] += cXSigQ2Pt * j3;
+  mC[kSigQ2PtSnp] += cXSigQ2Pt * j5;
+
+  mC[kSigZY] += cXSigZ * j3 + hXSigY * j4;
+  mC[kSigSnpZ] += cXSigSnp * j4 + hXSigZ * j5;
+  mC[kSigTglY] += cXSigTgl * j3;
+  mC[kSigTglSnp] += cXSigTgl * j5;
+  mC[kSigQ2PtZ] += cXSigQ2Pt * j4;
+
+  checkCovariance();
+  linRef0 = linRef1;
+
+  return true;
+}
+
 //_______________________________________________________________________
 template 
 GPUd() bool TrackParametrizationWithError::propagateToDCA(const o2::dataformats::VertexBase& vtx, value_t b, o2::dataformats::DCA* dca, value_t maxD)
@@ -230,9 +417,13 @@ GPUd() bool TrackParametrizationWithError::propagateToDCA(const o2::dat
   value_t xv = vtx.getX() * cs + vtx.getY() * sn, yv = -vtx.getX() * sn + vtx.getY() * cs, zv = vtx.getZ();
   x -= xv;
   y -= yv;
-  //Estimate the impact parameter neglecting the track curvature
+  // Estimate the impact parameter neglecting the track curvature
   value_t d = gpu::CAMath::Abs(x * snp - y * csp);
   if (d > maxD) {
+    if (dca) { // provide default DCA for failed propag
+      dca->set(o2::track::DefaultDCA, o2::track::DefaultDCA,
+               o2::track::DefaultDCACov, o2::track::DefaultDCACov, o2::track::DefaultDCACov);
+    }
     return false;
   }
   value_t crv = this->getCurvature(b);
@@ -250,9 +441,11 @@ GPUd() bool TrackParametrizationWithError::propagateToDCA(const o2::dat
   if (!tmpT.rotate(alp) || !tmpT.propagateTo(xv, b)) {
 #if !defined(GPUCA_ALIGPUCODE)
     LOG(debug) << "failed to propagate to alpha=" << alp << " X=" << xv << vtx << " | Track is: " << tmpT.asString();
-#elif !defined(GPUCA_NO_FMT)
-    LOG(debug) << "failed to propagate to alpha=" << alp << " X=" << xv << vtx;
 #endif
+    if (dca) { // provide default DCA for failed propag
+      dca->set(o2::track::DefaultDCA, o2::track::DefaultDCA,
+               o2::track::DefaultDCACov, o2::track::DefaultDCACov, o2::track::DefaultDCACov);
+    }
     return false;
   }
   *this = tmpT;
@@ -267,7 +460,7 @@ GPUd() bool TrackParametrizationWithError::propagateToDCA(const o2::dat
 //______________________________________________________________
 template 
 GPUd() TrackParametrizationWithError::TrackParametrizationWithError(const dim3_t& xyz, const dim3_t& pxpypz,
-                                                                             const gpu::gpustd::array& cv, int charge, bool sectorAlpha, const PID pid)
+                                                                             const std::array& cv, int charge, bool sectorAlpha, const PID pid)
 {
   // construct track param and covariance from kinematics and lab errors
   set(xyz, pxpypz, cv, charge, sectorAlpha, pid);
@@ -276,7 +469,7 @@ GPUd() TrackParametrizationWithError::TrackParametrizationWithError(con
 //______________________________________________________________
 template 
 GPUd() void TrackParametrizationWithError::set(const dim3_t& xyz, const dim3_t& pxpypz,
-                                                        const gpu::gpustd::array& cv, int charge, bool sectorAlpha, const PID pid)
+                                                        const std::array& cv, int charge, bool sectorAlpha, const PID pid)
 {
   // set track param and covariance from kinematics and lab errors
 
@@ -455,7 +648,7 @@ GPUd() bool TrackParametrizationWithError::propagateTo(value_t xk, cons
   }
   // Do not propagate tracks outside the ALICE detector
   if (gpu::CAMath::Abs(dx) > 1e5 || gpu::CAMath::Abs(this->getY()) > 1e5 || gpu::CAMath::Abs(this->getZ()) > 1e5) {
-    LOGP(warning, "Anomalous track, target X:{:f}", xk);
+    LOG(warning) << "Anomalous track, target X:" << xk;
     //    print();
     return false;
   }
@@ -476,14 +669,14 @@ GPUd() bool TrackParametrizationWithError::propagateTo(value_t xk, cons
   if (gpu::CAMath::Abs(r2) < constants::math::Almost0) {
     return false;
   }
-
-  value_t dy2dx = (f1 + f2) / (r1 + r2);
+  double r1pr2Inv = 1. / (r1 + r2), r2inv = 1. / r2, r1inv = 1. / r1;
+  double dy2dx = (f1 + f2) * r1pr2Inv, dx2r1pr2 = dx * r1pr2Inv;
   value_t step = (gpu::CAMath::Abs(x2r) < 0.05f) ? dx * gpu::CAMath::Abs(r2 + f2 * dy2dx)                                                   // chord
                                                  : 2.f * gpu::CAMath::ASin(0.5f * dx * gpu::CAMath::Sqrt(1.f + dy2dx * dy2dx) * crv) / crv; // arc
   step *= gpu::CAMath::Sqrt(1.f + this->getTgl() * this->getTgl());
   //
   // get the track x,y,z,px/p,py/p,pz/p,p,sinAlpha,cosAlpha in the Global System
-  gpu::gpustd::array vecLab{0.f};
+  std::array vecLab{0.f};
   if (!this->getPosDirGlo(vecLab)) {
     return false;
   }
@@ -493,15 +686,16 @@ GPUd() bool TrackParametrizationWithError::propagateTo(value_t xk, cons
           &c22 = mC[kSigSnp2], &c30 = mC[kSigTglY], &c31 = mC[kSigTglZ], &c32 = mC[kSigTglSnp], &c33 = mC[kSigTgl2],
           &c40 = mC[kSigQ2PtY], &c41 = mC[kSigQ2PtZ], &c42 = mC[kSigQ2PtSnp], &c43 = mC[kSigQ2PtTgl],
           &c44 = mC[kSigQ2Pt2];
+
   // evaluate matrix in double prec.
-  double rinv = 1. / r1;
-  double r3inv = rinv * rinv * rinv;
-  double f24 = dx * b[2] * constants::math::B2C; // x2r/track[kQ2Pt];
-  double f02 = dx * r3inv;
-  double f04 = 0.5 * f24 * f02;
-  double f12 = f02 * this->getTgl() * f1;
-  double f14 = 0.5 * f24 * f12; // 0.5*f24*f02*getTgl()*f1;
-  double f13 = dx * rinv;
+  value_t kb = b[2] * constants::math::B2C;
+  double hh = dx2r1pr2 * r2inv * (1. + r1 * r2 + f1 * f2), jj = dx * (dy2dx - f2 * r2inv);
+  double f02 = hh * r1inv;
+  double f04 = hh * dx2r1pr2 * kb;
+  double f24 = dx * kb; // x2r/mP[kQ2Pt];
+  double f12 = this->getTgl() * (f02 * f2 + jj);
+  double f13 = dx * (r2 + f2 * dy2dx);
+  double f14 = this->getTgl() * (f04 * f2 + jj * f24);
 
   // b = C*ft
   double b00 = f02 * c20 + f04 * c40, b01 = f12 * c20 + f14 * c40 + f13 * c30;
@@ -550,7 +744,7 @@ GPUd() bool TrackParametrizationWithError::propagateTo(value_t xk, cons
     costet = b[2] / bb;
     sintet = bt / bb;
   }
-  gpu::gpustd::array vect{costet * cosphi * vecLab[0] + costet * sinphi * vecLab[1] - sintet * vecLab[2],
+  std::array vect{costet * cosphi * vecLab[0] + costet * sinphi * vecLab[1] - sintet * vecLab[2],
                                       -sinphi * vecLab[0] + cosphi * vecLab[1],
                                       sintet * cosphi * vecLab[0] + sintet * sinphi * vecLab[1] + costet * vecLab[2],
                                       costet * cosphi * vecLab[3] + costet * sinphi * vecLab[4] - sintet * vecLab[5],
@@ -559,8 +753,8 @@ GPUd() bool TrackParametrizationWithError::propagateTo(value_t xk, cons
                                       vecLab[6]};
 
   // Do the helix step
-  value_t sgn = this->getSign();
-  g3helx3(sgn * bb, step, vect);
+  value_t q = this->getCharge();
+  g3helx3(q * bb, step, vect);
 
   // Rotate back to the Global System
   vecLab[0] = cosphi * costet * vect[0] - sinphi * vect[1] + cosphi * sintet * vect[2];
@@ -599,16 +793,225 @@ GPUd() bool TrackParametrizationWithError::propagateTo(value_t xk, cons
   this->setZ(z);
   this->setSnp(vecLab[4] * t);
   this->setTgl(vecLab[5] * t);
-  this->setQ2Pt(sgn * t / vecLab[6]);
+  this->setQ2Pt(q * t / vecLab[6]);
+
+  return true;
+}
+
+//____________________________________________________________
+template 
+GPUd() bool TrackParametrizationWithError::propagateTo(value_t xk, TrackParametrization& linRef0, const dim3_t& b)
+{
+  //----------------------------------------------------------------
+  // Extrapolate this track to the plane X=xk in the field b[].
+  //
+  // X [cm] is in the "tracking coordinate system" of this track.
+  // b[]={Bx,By,Bz} [kG] is in the Global coordidate system.
+  //----------------------------------------------------------------
+
+  value_t dx = xk - this->getX();
+  if (gpu::CAMath::Abs(dx) < constants::math::Almost0) {
+    return true;
+  }
+  // Do not propagate tracks outside the ALICE detector
+  if (gpu::CAMath::Abs(dx) > 1e5 || gpu::CAMath::Abs(this->getY()) > 1e5 || gpu::CAMath::Abs(this->getZ()) > 1e5) {
+    LOG(warning) << "Anomalous track, target X:" << xk;
+    //    print();
+    return false;
+  }
+  if (gpu::CAMath::Abs(dx) < constants::math::Almost0) {
+    this->setX(xk);
+    linRef0.setX(xk);
+    return true;
+  }
+  // preliminary calculations to find the step size
+  value_t crv = (gpu::CAMath::Abs(b[2]) < constants::math::Almost0) ? 0.f : linRef0.getCurvature(b[2]);
+  if (gpu::CAMath::Abs(crv) < constants::math::Almost0) {
+    return propagateTo(xk, linRef0, 0.);
+  }
+  value_t kb = b[2] * constants::math::B2C, x2r = crv * dx;
+  // evaluate in double prec.
+  value_t snpRef0 = linRef0.getSnp(), snpRef1 = snpRef0 + x2r;
+  if ((gpu::CAMath::Abs(snpRef0) > constants::math::Almost1) || (gpu::CAMath::Abs(snpRef1) > constants::math::Almost1)) {
+    return false;
+  }
+  value_t cspRef0 = gpu::CAMath::Sqrt((1 - snpRef0) * (1 + snpRef0)), cspRef1 = gpu::CAMath::Sqrt((1 - snpRef1) * (1 + snpRef1));
+  if (gpu::CAMath::Abs(cspRef0) < constants::math::Almost0 || gpu::CAMath::Abs(cspRef1) < constants::math::Almost0) {
+    return false;
+  }
+  value_t cspRef0Inv = value_t(1) / cspRef0, cspRef1Inv = value_t(1) / cspRef1, cc = cspRef0 + cspRef1, ccInv = value_t(1) / cc, dy2dx = (snpRef0 + snpRef1) * ccInv;
+  value_t step = (gpu::CAMath::Abs(crv * dx) < 0.05f) ? dx * (cspRef1 + snpRef1 * dy2dx) : 2. * gpu::CAMath::ASin(0.5 * dx * gpu::CAMath::Sqrt(1.f + dy2dx * dy2dx) * crv) / crv; // arc
+  step *= gpu::CAMath::Sqrt(1.f + linRef0.getTgl() * linRef0.getTgl());
+
+  //
+  // get the track x,y,z,px/p,py/p,pz/p,p,sinAlpha,cosAlpha in the Global System
+  std::array vecLab{0.f};
+  if (!linRef0.getPosDirGlo(vecLab)) {
+    return false;
+  }
+  //
+  // Rotate to the system where Bx=By=0.
+  value_t bxy2 = b[0] * b[0] + b[1] * b[1];
+  value_t bt = gpu::CAMath::Sqrt(bxy2);
+  value_t cosphi = 1.f, sinphi = 0.f;
+  if (bt > constants::math::Almost0) {
+    cosphi = b[0] / bt;
+    sinphi = b[1] / bt;
+  }
+  value_t bb = gpu::CAMath::Sqrt(bxy2 + b[2] * b[2]);
+  value_t costet = 1., sintet = 0.;
+  if (bb > constants::math::Almost0) {
+    costet = b[2] / bb;
+    sintet = bt / bb;
+  }
+  std::array vect{costet * cosphi * vecLab[0] + costet * sinphi * vecLab[1] - sintet * vecLab[2],
+                              -sinphi * vecLab[0] + cosphi * vecLab[1],
+                              sintet * cosphi * vecLab[0] + sintet * sinphi * vecLab[1] + costet * vecLab[2],
+                              costet * cosphi * vecLab[3] + costet * sinphi * vecLab[4] - sintet * vecLab[5],
+                              -sinphi * vecLab[3] + cosphi * vecLab[4],
+                              sintet * cosphi * vecLab[3] + sintet * sinphi * vecLab[4] + costet * vecLab[5],
+                              vecLab[6]};
+
+  // Do the helix step
+  value_t q = this->getCharge();
+  g3helx3(q * bb, step, vect);
+
+  // Rotate back to the Global System
+  vecLab[0] = cosphi * costet * vect[0] - sinphi * vect[1] + cosphi * sintet * vect[2];
+  vecLab[1] = sinphi * costet * vect[0] + cosphi * vect[1] + sinphi * sintet * vect[2];
+  vecLab[2] = -sintet * vect[0] + costet * vect[2];
+
+  vecLab[3] = cosphi * costet * vect[3] - sinphi * vect[4] + cosphi * sintet * vect[5];
+  vecLab[4] = sinphi * costet * vect[3] + cosphi * vect[4] + sinphi * sintet * vect[5];
+  vecLab[5] = -sintet * vect[3] + costet * vect[5];
+
+  // Rotate back to the Tracking System
+  value_t sinalp = -vecLab[7], cosalp = vecLab[8];
+  value_t t = cosalp * vecLab[0] - sinalp * vecLab[1];
+  vecLab[1] = sinalp * vecLab[0] + cosalp * vecLab[1];
+  vecLab[0] = t;
+  t = cosalp * vecLab[3] - sinalp * vecLab[4];
+  vecLab[4] = sinalp * vecLab[3] + cosalp * vecLab[4];
+  vecLab[3] = t;
+
+  // Do the final correcting step to the target plane (linear approximation)
+  value_t x = vecLab[0], y = vecLab[1], z = vecLab[2];
+  if (gpu::CAMath::Abs(dx) > constants::math::Almost0) {
+    if (gpu::CAMath::Abs(vecLab[3]) < constants::math::Almost0) {
+      return false;
+    }
+    auto dxFin = xk - vecLab[0];
+    x += dxFin;
+    y += vecLab[4] / vecLab[3] * dxFin;
+    z += vecLab[5] / vecLab[3] * dxFin;
+  }
+
+  // Calculate the track parameters
+  auto linRef1 = linRef0;
+  t = 1.f / gpu::CAMath::Sqrt(vecLab[3] * vecLab[3] + vecLab[4] * vecLab[4]);
+  linRef1.setX(xk);
+  linRef1.setY(y);
+  linRef1.setZ(z);
+  linRef1.setSnp(snpRef1 = vecLab[4] * t); // reassign snpRef1
+  linRef1.setTgl(vecLab[5] * t);
+  linRef1.setQ2Pt(q * t / vecLab[6]);
+
+  // recalculate parameters of the transported ref track needed for transport of this:
+  cspRef1 = gpu::CAMath::Sqrt((1 - snpRef1) * (1 + snpRef1));
+  cspRef1Inv = value_t(1) / cspRef1;
+  cc = cspRef0 + cspRef1;
+  ccInv = value_t(1) / cc;
+  dy2dx = (snpRef0 + snpRef1) * ccInv;
+  double dxccInv = dx * ccInv, hh = dxccInv * cspRef1Inv * (1 + cspRef0 * cspRef1 + snpRef0 * snpRef1), jj = dx * (dy2dx - snpRef1 * cspRef1Inv);
+  double f02 = hh * cspRef0Inv;
+  double f04 = hh * dxccInv * kb;
+  double f24 = dx * kb;
+  double f12 = linRef0.getTgl() * (f02 * snpRef1 + jj);
+  double f13 = dx * (cspRef1 + snpRef1 * dy2dx); // dS
+  double f14 = linRef0.getTgl() * (f04 * snpRef1 + jj * f24);
+
+  // difference between the current and reference state
+  value_t diff[5];
+  for (int i = 0; i < 5; i++) {
+    diff[i] = this->getParam(i) - linRef0.getParam(i);
+  }
+  value_t snpUpd = snpRef1 + diff[kSnp] + f24 * diff[kQ2Pt];
+  if (gpu::CAMath::Abs(snpUpd) > constants::math::Almost1) {
+    return false;
+  }
+  this->setX(xk);
+  this->setY(linRef1.getY() + diff[kY] + f02 * diff[kSnp] + f04 * diff[kQ2Pt]);
+  this->setZ(linRef1.getZ() + diff[kZ] + f13 * diff[kTgl] + f14 * diff[kQ2Pt]);
+  this->setSnp(snpUpd);
+  this->setTgl(linRef1.getTgl() + diff[kTgl]);
+  this->setQ2Pt(linRef1.getQ2Pt() + diff[kQ2Pt]);
+
+  linRef0 = linRef1; // update reference track
+
+  // matrix transformed with Bz component only
+  value_t &c00 = mC[kSigY2], &c10 = mC[kSigZY], &c11 = mC[kSigZ2], &c20 = mC[kSigSnpY], &c21 = mC[kSigSnpZ],
+          &c22 = mC[kSigSnp2], &c30 = mC[kSigTglY], &c31 = mC[kSigTglZ], &c32 = mC[kSigTglSnp], &c33 = mC[kSigTgl2],
+          &c40 = mC[kSigQ2PtY], &c41 = mC[kSigQ2PtZ], &c42 = mC[kSigQ2PtSnp], &c43 = mC[kSigQ2PtTgl],
+          &c44 = mC[kSigQ2Pt2];
+
+  // b = C*ft
+  double b00 = f02 * c20 + f04 * c40, b01 = f12 * c20 + f14 * c40 + f13 * c30;
+  double b02 = f24 * c40;
+  double b10 = f02 * c21 + f04 * c41, b11 = f12 * c21 + f14 * c41 + f13 * c31;
+  double b12 = f24 * c41;
+  double b20 = f02 * c22 + f04 * c42, b21 = f12 * c22 + f14 * c42 + f13 * c32;
+  double b22 = f24 * c42;
+  double b40 = f02 * c42 + f04 * c44, b41 = f12 * c42 + f14 * c44 + f13 * c43;
+  double b42 = f24 * c44;
+  double b30 = f02 * c32 + f04 * c43, b31 = f12 * c32 + f14 * c43 + f13 * c33;
+  double b32 = f24 * c43;
+
+  // a = f*b = f*C*ft
+  double a00 = f02 * b20 + f04 * b40, a01 = f02 * b21 + f04 * b41, a02 = f02 * b22 + f04 * b42;
+  double a11 = f12 * b21 + f14 * b41 + f13 * b31, a12 = f12 * b22 + f14 * b42 + f13 * b32;
+  double a22 = f24 * b42;
+
+  // F*C*Ft = C + (b + bt + a)
+  c00 += b00 + b00 + a00;
+  c10 += b10 + b01 + a01;
+  c20 += b20 + b02 + a02;
+  c30 += b30;
+  c40 += b40;
+  c11 += b11 + b11 + a11;
+  c21 += b21 + b12 + a12;
+  c31 += b31;
+  c41 += b41;
+  c22 += b22 + b22 + a22;
+  c32 += b32;
+  c42 += b42;
+
+  checkCovariance();
 
   return true;
 }
 
+//______________________________________________
+template 
+GPUd() void TrackParametrizationWithError::checkCorrelations()
+{
+  // This function forces the abs of correlation coefficients to be <1.
+  constexpr float MaxCorr = 0.99;
+  for (int i = kNParams; i--;) {
+    for (int j = i; j--;) {
+      auto sig2 = mC[DiagMap[i]] * mC[DiagMap[j]];
+      auto& cov = mC[CovarMap[i][j]];
+      if (cov * cov >= MaxCorr * sig2) { // constrain correlation
+        cov = gpu::CAMath::Sqrt(sig2) * (cov > 0. ? MaxCorr : -MaxCorr);
+      }
+    }
+  }
+}
+
 //______________________________________________
 template 
 GPUd() void TrackParametrizationWithError::checkCovariance()
 {
-  // This function forces the diagonal elements of the covariance matrix to be positive.
+  // This function forces the diagonal elements of the covariance matrix to be positive and abs of correlation coefficients to be <1.
   // In case the diagonal element is bigger than the maximal allowed value, it is set to
   // the limit and the off-diagonal elements that correspond to it are set to zero.
 
@@ -711,17 +1114,49 @@ GPUd() auto TrackParametrizationWithError::getPredictedChi2(const value
     return constants::math::VeryBig;
   }
 
+  value_t d = this->getY() - p[0];
+  value_t z = this->getZ() - p[1];
+  auto chi2 = (d * (szz * d - sdz * z) + z * (sdd * z - d * sdz)) / det;
+  if (chi2 < 0.) {
+#ifndef GPUCA_ALIGPUCODE
+    LOGP(warning, "Negative chi2={}, Cluster: {} {} {} Dy:{} Dz:{} | sdd:{} sdz:{} szz:{} det:{}", chi2, cov[0], cov[1], cov[2], d, z, sdd, sdz, szz, det);
+    LOGP(warning, "Track: {}", asString());
+#endif
+  }
+  return chi2;
+}
+
+//______________________________________________
+template 
+GPUd() auto TrackParametrizationWithError::getPredictedChi2Quiet(const value_t* p, const value_t* cov) const -> value_t
+{
+  // Estimate the chi2 of the space point "p" with the cov. matrix "cov"
+  auto sdd = static_cast(getSigmaY2()) + static_cast(cov[0]);
+  auto sdz = static_cast(getSigmaZY()) + static_cast(cov[1]);
+  auto szz = static_cast(getSigmaZ2()) + static_cast(cov[2]);
+  auto det = sdd * szz - sdz * sdz;
+
+  if (gpu::CAMath::Abs(det) < constants::math::Almost0) {
+    return constants::math::VeryBig;
+  }
+
   value_t d = this->getY() - p[0];
   value_t z = this->getZ() - p[1];
 
   return (d * (szz * d - sdz * z) + z * (sdd * z - d * sdz)) / det;
 }
 
-#if !defined(GPUCA_GPUCODE) && !defined(GPUCA_STANDALONE) // Disable function relying on ROOT SMatrix on GPU
+//______________________________________________
+template 
+GPUd() auto TrackParametrizationWithError::getPredictedChi2(const TrackParametrizationWithError& rhs) const -> value_t
+{
+  MatrixDSym5 cov; // perform matrix operations in double!
+  return getPredictedChi2(rhs, cov);
+}
 
 //______________________________________________
 template 
-void TrackParametrizationWithError::buildCombinedCovMatrix(const TrackParametrizationWithError& rhs, MatrixDSym5& cov) const
+GPUd() void TrackParametrizationWithError::buildCombinedCovMatrix(const TrackParametrizationWithError& rhs, MatrixDSym5& cov) const
 {
   // fill combined cov.matrix (NOT inverted)
   cov(kY, kY) = static_cast(getSigmaY2()) + static_cast(rhs.getSigmaY2());
@@ -743,24 +1178,16 @@ void TrackParametrizationWithError::buildCombinedCovMatrix(const TrackP
 
 //______________________________________________
 template 
-auto TrackParametrizationWithError::getPredictedChi2(const TrackParametrizationWithError& rhs) const -> value_t
-{
-  MatrixDSym5 cov; // perform matrix operations in double!
-  return getPredictedChi2(rhs, cov);
-}
-
-//______________________________________________
-template 
-auto TrackParametrizationWithError::getPredictedChi2(const TrackParametrizationWithError& rhs, MatrixDSym5& covToSet) const -> value_t
+GPUd() auto TrackParametrizationWithError::getPredictedChi2(const TrackParametrizationWithError& rhs, MatrixDSym5& covToSet) const -> value_t
 {
   // get chi2 wrt other track, which must be defined at the same parameters X,alpha
   // Supplied non-initialized covToSet matrix is filled by inverse combined matrix for further use
 
-  if (gpu::CAMath::Abs(this->getAlpha() - rhs.getAlpha()) > FLT_EPSILON) {
+  if (gpu::CAMath::Abs(this->getAlpha() - rhs.getAlpha()) > o2::constants::math::Epsilon) {
     LOG(error) << "The reference Alpha of the tracks differ: " << this->getAlpha() << " : " << rhs.getAlpha();
     return 2.f * HugeF;
   }
-  if (gpu::CAMath::Abs(this->getX() - rhs.getX()) > FLT_EPSILON) {
+  if (gpu::CAMath::Abs(this->getX() - rhs.getX()) > o2::constants::math::Epsilon) {
     LOG(error) << "The reference X of the tracks differ: " << this->getX() << " : " << rhs.getX();
     return 2.f * HugeF;
   }
@@ -784,16 +1211,16 @@ auto TrackParametrizationWithError::getPredictedChi2(const TrackParamet
 
 //______________________________________________
 template 
-bool TrackParametrizationWithError::update(const TrackParametrizationWithError& rhs, const MatrixDSym5& covInv)
+GPUd() bool TrackParametrizationWithError::update(const TrackParametrizationWithError& rhs, const MatrixDSym5& covInv)
 {
   // update track with other track, the inverted combined cov matrix should be supplied
 
   // consider skipping this check, since it is usually already done upstream
-  if (gpu::CAMath::Abs(this->getAlpha() - rhs.getAlpha()) > FLT_EPSILON) {
+  if (gpu::CAMath::Abs(this->getAlpha() - rhs.getAlpha()) > o2::constants::math::Epsilon) {
     LOG(error) << "The reference Alpha of the tracks differ: " << this->getAlpha() << " : " << rhs.getAlpha();
     return false;
   }
-  if (gpu::CAMath::Abs(this->getX() - rhs.getX()) > FLT_EPSILON) {
+  if (gpu::CAMath::Abs(this->getX() - rhs.getX()) > o2::constants::math::Epsilon) {
     LOG(error) << "The reference X of the tracks differ: " << this->getX() << " : " << rhs.getX();
     return false;
   }
@@ -830,7 +1257,7 @@ bool TrackParametrizationWithError::update(const TrackParametrizationWi
   }
 
   // updated covariance: Cov0 = Cov0 - K*Cov0
-  matK *= ROOT::Math::SMatrix>(matC0);
+  matK *= o2::math_utils::SMatrix>(matC0);
   mC[kSigY2] -= matK(kY, kY);
   mC[kSigZY] -= matK(kZ, kY);
   mC[kSigZ2] -= matK(kZ, kZ);
@@ -852,7 +1279,7 @@ bool TrackParametrizationWithError::update(const TrackParametrizationWi
 
 //______________________________________________
 template 
-bool TrackParametrizationWithError::update(const TrackParametrizationWithError& rhs)
+GPUd() bool TrackParametrizationWithError::update(const TrackParametrizationWithError& rhs)
 {
   // update track with other track
   MatrixDSym5 covI; // perform matrix operations in double!
@@ -864,8 +1291,6 @@ bool TrackParametrizationWithError::update(const TrackParametrizationWi
   return update(rhs, covI);
 }
 
-#endif
-
 //______________________________________________
 template 
 GPUd() bool TrackParametrizationWithError::update(const value_t* p, const value_t* cov)
@@ -950,7 +1375,7 @@ GPUd() value_T TrackParametrizationWithError::update(const o2::dataform
 
 //______________________________________________
 template 
-GPUd() bool TrackParametrizationWithError::correctForMaterial(value_t x2x0, value_t xrho, bool anglecorr, value_t dedx)
+GPUd() bool TrackParametrizationWithError::correctForMaterial(value_t x2x0, value_t xrho, bool anglecorr)
 {
   //------------------------------------------------------------------
   // This function corrects the track parameters for the crossed material.
@@ -962,84 +1387,257 @@ GPUd() bool TrackParametrizationWithError::correctForMaterial(value_t x
   // "anglecorr" - switch for the angular correction
   //------------------------------------------------------------------
   constexpr value_t kMSConst2 = 0.0136f * 0.0136f;
-  constexpr value_t kMaxELossFrac = 0.3f; // max allowed fractional eloss
-  constexpr value_t kMinP = 0.01f;        // kill below this momentum
+  constexpr value_t kMinP = 0.01f; // kill below this momentum
 
-  value_t& fC22 = mC[kSigSnp2];
-  value_t& fC33 = mC[kSigTgl2];
-  value_t& fC43 = mC[kSigQ2PtTgl];
-  value_t& fC44 = mC[kSigQ2Pt2];
-  //
   value_t csp2 = (1.f - this->getSnp()) * (1.f + this->getSnp()); // cos(phi)^2
   value_t cst2I = (1.f + this->getTgl() * this->getTgl());        // 1/cos(lambda)^2
-  // Apply angle correction, if requested
-  if (anglecorr) {
+  if (anglecorr) {                                                // Apply angle correction, if requested
     value_t angle = gpu::CAMath::Sqrt(cst2I / csp2);
     x2x0 *= angle;
     xrho *= angle;
   }
-  value_t p = this->getP();
-  value_t p2 = p * p;
-  value_t e2 = p2 + this->getPID().getMass2();
-  value_t beta2 = p2 / e2;
+  auto m = this->getPID().getMass();
+  int charge2 = this->getAbsCharge() * this->getAbsCharge();
+  value_t p = this->getP(), p0 = p, p02 = p * p, e2 = p02 + this->getPID().getMass2(), massInv = 1. / m, bg = p * massInv, dETot = 0.;
+  value_t e = gpu::CAMath::Sqrt(e2), e0 = e;
+  if (m > 0 && xrho != 0.f) {
+    value_t ekin = e - m, dedx = this->getdEdxBBOpt(bg);
+#ifdef _BB_NONCONST_CORR_
+    value_t dedxDer = 0., dedx1 = dedx;
+#endif
+    if (charge2 != 1) {
+      dedx *= charge2;
+    }
+    value_t dE = dedx * xrho;
+    int na = 1 + int(gpu::CAMath::Abs(dE) / ekin * ELoss2EKinThreshInv);
+    if (na > MaxELossIter) {
+      na = MaxELossIter;
+    }
+    if (na > 1) {
+      dE /= na;
+      xrho /= na;
+#ifdef _BB_NONCONST_CORR_
+      dedxDer = this->getBetheBlochSolidDerivativeApprox(dedx1, bg); // require correction for non-constantness of dedx vs betagamma
+      if (charge2 != 1) {
+        dedxDer *= charge2;
+      }
+#endif
+    }
+    while (na--) {
+#ifdef _BB_NONCONST_CORR_
+      if (dedxDer != 0.) { // correction for non-constantness of dedx vs beta*gamma (in linear approximation): for a single step dE -> dE * [(exp(dedxDer) - 1)/dedxDer]
+        if (xrho < 0) {
+          dedxDer = -dedxDer; // E.loss ( -> positive derivative)
+        }
+        auto corrC = (gpu::CAMath::Exp(dedxDer) - 1.) / dedxDer;
+        dE *= corrC;
+      }
+#endif
+      e += dE;
+      if (e > m) { // stopped
+        p = gpu::CAMath::Sqrt(e * e - this->getPID().getMass2());
+      } else {
+        return false;
+      }
+      if (na) {
+        bg = p * massInv;
+        dedx = this->getdEdxBBOpt(bg);
+#ifdef _BB_NONCONST_CORR_
+        dedxDer = this->getBetheBlochSolidDerivativeApprox(dedx, bg);
+#endif
+        if (charge2 != 1) {
+          dedx *= charge2;
+#ifdef _BB_NONCONST_CORR_
+          dedxDer *= charge2;
+#endif
+        }
+        dE = dedx * xrho;
+      }
+    }
+
+    if (p < kMinP) {
+      return false;
+    }
+    dETot = e - e0;
+  } // end of e.loss correction
 
   // Calculating the multiple scattering corrections******************
+  value_t& fC22 = mC[kSigSnp2];
+  value_t& fC33 = mC[kSigTgl2];
+  value_t& fC43 = mC[kSigQ2PtTgl];
+  value_t& fC44 = mC[kSigQ2Pt2];
+  //
   value_t cC22(0.f), cC33(0.f), cC43(0.f), cC44(0.f);
   if (x2x0 != 0.f) {
-    value_t theta2 = kMSConst2 / (beta2 * p2) * gpu::CAMath::Abs(x2x0);
-    if (this->getAbsCharge() != 1) {
-      theta2 *= this->getAbsCharge() * this->getAbsCharge();
+    value_t beta2 = p02 / e2, theta2 = kMSConst2 / (beta2 * p02) * gpu::CAMath::Abs(x2x0);
+    value_t fp34 = this->getTgl();
+    if (charge2 != 1) {
+      theta2 *= charge2;
+      fp34 *= this->getCharge2Pt();
     }
     if (theta2 > constants::math::PI * constants::math::PI) {
       return false;
     }
-    value_t fp34 = this->getTgl() * this->getCharge2Pt();
     value_t t2c2I = theta2 * cst2I;
     cC22 = t2c2I * csp2;
     cC33 = t2c2I * cst2I;
     cC43 = t2c2I * fp34;
     cC44 = theta2 * fp34 * fp34;
-    // optimes this
+    // optimize this
     //    cC22 = theta2*((1.-getSnp())*(1.+getSnp()))*(1. + this->getTgl()*getTgl());
     //    cC33 = theta2*(1. + this->getTgl()*getTgl())*(1. + this->getTgl()*getTgl());
     //    cC43 = theta2*getTgl()*this->getQ2Pt()*(1. + this->getTgl()*getTgl());
     //    cC44 = theta2*getTgl()*this->getQ2Pt()*getTgl()*this->getQ2Pt();
   }
 
-  // Calculating the energy loss corrections************************
-  value_t cP4 = 1.f;
-  if ((xrho != 0.f) && (beta2 < 1.f)) {
-    if (dedx < kCalcdEdxAuto + constants::math::Almost1) { // request to calculate dedx on the fly
-      dedx = BetheBlochSolid(p / this->getPID().getMass());
-      if (this->getAbsCharge() != 1) {
-        dedx *= this->getAbsCharge() * this->getAbsCharge();
+  // the energy loss correction contribution to cov.matrix: approximate energy loss fluctuation (M.Ivanov)
+  constexpr value_t knst = 0.0007f; // To be tuned.
+  value_t sigmadE = knst * gpu::CAMath::Sqrt(gpu::CAMath::Abs(dETot)) * e0 / p02 * this->getCharge2Pt();
+  cC44 += sigmadE * sigmadE;
+
+  // Applying the corrections*****************************
+  fC22 += cC22;
+  fC33 += cC33;
+  fC43 += cC43;
+  fC44 += cC44;
+  this->setQ2Pt(this->getQ2Pt() * p0 / p);
+
+  checkCovariance();
+
+  return true;
+}
+
+//______________________________________________
+template 
+GPUd() bool TrackParametrizationWithError::correctForMaterial(TrackParametrization& linRef, value_t x2x0, value_t xrho, bool anglecorr)
+{
+  //------------------------------------------------------------------
+  // This function corrects the reference and current track parameters for the crossed material
+  // "x2x0"   - X/X0, the thickness in units of the radiation length.
+  // "xrho" - is the product length*density (g/cm^2).
+  //     It should be passed as negative when propagating tracks
+  //     from the intreaction point to the outside of the central barrel.
+  // "dedx" - mean enery loss (GeV/(g/cm^2), if <=kCalcdEdxAuto : calculate on the fly
+  // "anglecorr" - switch for the angular correction
+  //------------------------------------------------------------------
+  constexpr value_t kMSConst2 = 0.0136f * 0.0136f;
+  constexpr value_t kMinP = 0.01f; // kill below this momentum
+
+  value_t csp2 = (1.f - linRef.getSnp()) * (1.f + linRef.getSnp()); // cos(phi)^2
+  value_t cst2I = (1.f + linRef.getTgl() * linRef.getTgl());        // 1/cos(lambda)^2
+  if (anglecorr) {                                                  // Apply angle correction, if requested
+    value_t angle = gpu::CAMath::Sqrt(cst2I / csp2);
+    x2x0 *= angle;
+    xrho *= angle;
+  }
+  auto pid = linRef.getPID();
+  auto m = pid.getMass();
+  int charge2 = linRef.getAbsCharge() * linRef.getAbsCharge();
+  value_t p = linRef.getP(), p0 = p, p02 = p * p, e2 = p02 + pid.getMass2(), massInv = 1. / m, bg = p * massInv, dETot = 0.;
+  value_t e = gpu::CAMath::Sqrt(e2), e0 = e;
+  if (m > 0 && xrho != 0.f) {
+    value_t ekin = e - m, dedx = this->getdEdxBBOpt(bg);
+#ifdef _BB_NONCONST_CORR_
+    value_t dedxDer = 0., dedx1 = dedx;
+#endif
+    if (charge2 != 1) {
+      dedx *= charge2;
+    }
+    value_t dE = dedx * xrho;
+    int na = 1 + int(gpu::CAMath::Abs(dE) / ekin * ELoss2EKinThreshInv);
+    if (na > MaxELossIter) {
+      na = MaxELossIter;
+    }
+    if (na > 1) {
+      dE /= na;
+      xrho /= na;
+#ifdef _BB_NONCONST_CORR_
+      dedxDer = this->getBetheBlochSolidDerivativeApprox(dedx1, bg); // require correction for non-constantness of dedx vs betagamma
+      if (charge2 != 1) {
+        dedxDer *= charge2;
+      }
+#endif
+    }
+    while (na--) {
+#ifdef _BB_NONCONST_CORR_
+      if (dedxDer != 0.) { // correction for non-constantness of dedx vs beta*gamma (in linear approximation): for a single step dE -> dE * [(exp(dedxDer) - 1)/dedxDer]
+        if (xrho < 0) {
+          dedxDer = -dedxDer; // E.loss ( -> positive derivative)
+        }
+        auto corrC = (gpu::CAMath::Exp(dedxDer) - 1.) / dedxDer;
+        dE *= corrC;
+      }
+#endif
+      e += dE;
+      if (e > m) { // stopped
+        p = gpu::CAMath::Sqrt(e * e - pid.getMass2());
+      } else {
+        return false;
+      }
+      if (na) {
+        bg = p * massInv;
+        dedx = this->getdEdxBBOpt(bg);
+#ifdef _BB_NONCONST_CORR_
+        dedxDer = this->getBetheBlochSolidDerivativeApprox(dedx, bg);
+#endif
+        if (charge2 != 1) {
+          dedx *= charge2;
+#ifdef _BB_NONCONST_CORR_
+          dedxDer *= charge2;
+#endif
+        }
+        dE = dedx * xrho;
       }
     }
 
-    value_t dE = dedx * xrho;
-    value_t e = gpu::CAMath::Sqrt(e2);
-    if (gpu::CAMath::Abs(dE) > kMaxELossFrac * e) {
-      return false; // 30% energy loss is too much!
+    if (p < kMinP) {
+      return false;
     }
-    value_t eupd = e + dE;
-    value_t pupd2 = eupd * eupd - this->getPID().getMass2();
-    if (pupd2 < kMinP * kMinP) {
+    dETot = e - e0;
+  } // end of e.loss correction
+
+  // Calculating the multiple scattering corrections******************
+  value_t& fC22 = mC[kSigSnp2];
+  value_t& fC33 = mC[kSigTgl2];
+  value_t& fC43 = mC[kSigQ2PtTgl];
+  value_t& fC44 = mC[kSigQ2Pt2];
+  //
+  value_t cC22(0.f), cC33(0.f), cC43(0.f), cC44(0.f);
+  if (x2x0 != 0.f) {
+    value_t beta2 = p02 / e2, theta2 = kMSConst2 / (beta2 * p02) * gpu::CAMath::Abs(x2x0);
+    value_t fp34 = linRef.getTgl();
+    if (charge2 != 1) {
+      theta2 *= charge2;
+      fp34 *= linRef.getCharge2Pt();
+    }
+    if (theta2 > constants::math::PI * constants::math::PI) {
       return false;
     }
-    cP4 = p / gpu::CAMath::Sqrt(pupd2);
-    //
-    // Approximate energy loss fluctuation (M.Ivanov)
-    constexpr value_t knst = 0.07f; // To be tuned.
-    value_t sigmadE = knst * gpu::CAMath::Sqrt(gpu::CAMath::Abs(dE)) * e / p2 * this->getCharge2Pt();
-    cC44 += sigmadE * sigmadE;
+    value_t t2c2I = theta2 * cst2I;
+    cC22 = t2c2I * csp2;
+    cC33 = t2c2I * cst2I;
+    cC43 = t2c2I * fp34;
+    cC44 = theta2 * fp34 * fp34;
+    // optimize this
+    //    cC22 = theta2*((1.-getSnp())*(1.+getSnp()))*(1. + this->getTgl()*getTgl());
+    //    cC33 = theta2*(1. + this->getTgl()*getTgl())*(1. + this->getTgl()*getTgl());
+    //    cC43 = theta2*getTgl()*this->getQ2Pt()*(1. + this->getTgl()*getTgl());
+    //    cC44 = theta2*getTgl()*this->getQ2Pt()*getTgl()*this->getQ2Pt();
   }
 
+  // the energy loss correction contribution to cov.matrix: approximate energy loss fluctuation (M.Ivanov)
+  constexpr value_t knst = 0.0007f; // To be tuned.
+  value_t sigmadE = knst * gpu::CAMath::Sqrt(gpu::CAMath::Abs(dETot)) * e0 / p02 * linRef.getCharge2Pt();
+  cC44 += sigmadE * sigmadE;
+
   // Applying the corrections*****************************
   fC22 += cC22;
   fC33 += cC33;
   fC43 += cC43;
   fC44 += cC44;
-  this->setQ2Pt(this->getQ2Pt() * cP4);
+  auto pscale = p0 / p;
+  linRef.setQ2Pt(linRef.getQ2Pt() * pscale);
+  this->setQ2Pt(this->getQ2Pt() * pscale);
 
   checkCovariance();
 
@@ -1048,7 +1646,7 @@ GPUd() bool TrackParametrizationWithError::correctForMaterial(value_t x
 
 //______________________________________________________________
 template 
-GPUd() bool TrackParametrizationWithError::getCovXYZPxPyPzGlo(gpu::gpustd::array& cv) const
+GPUd() bool TrackParametrizationWithError::getCovXYZPxPyPzGlo(std::array& cv) const
 {
   //---------------------------------------------------------------------
   // This function returns the global covariance matrix of the track params
@@ -1115,15 +1713,22 @@ template 
 std::string TrackParametrizationWithError::asString() const
 {
   return TrackParametrization::asString() +
-         fmt::format(
-           "\n{:7s} {:+.3e}\n"
-           "{:7s} {:+.3e} {:+.3e}\n"
-           "{:7s} {:+.3e} {:+.3e} {:+.3e}\n"
-           "{:7s} {:+.3e} {:+.3e} {:+.3e} {:+.3e}\n"
-           "{:7s} {:+.3e} {:+.3e} {:+.3e} {:+.3e} {:+.3e}",
-           "CovMat:", mC[kSigY2], "", mC[kSigZY], mC[kSigZ2], "", mC[kSigSnpY], mC[kSigSnpZ], mC[kSigSnp2], "", mC[kSigTglY],
-           mC[kSigTglZ], mC[kSigTglSnp], mC[kSigTgl2], "", mC[kSigQ2PtY], mC[kSigQ2PtZ], mC[kSigQ2PtSnp], mC[kSigQ2PtTgl],
-           mC[kSigQ2Pt2]);
+         fmt::format(" Cov: [{:+.3e}] [{:+.3e} {:+.3e}] [{:+.3e} {:+.3e} {:+.3e}] [{:+.3e} {:+.3e} {:+.3e} {:+.3e}] [{:+.3e} {:+.3e} {:+.3e} {:+.3e} {:+.3e}]",
+                     mC[kSigY2], mC[kSigZY], mC[kSigZ2], mC[kSigSnpY], mC[kSigSnpZ], mC[kSigSnp2], mC[kSigTglY],
+                     mC[kSigTglZ], mC[kSigTglSnp], mC[kSigTgl2], mC[kSigQ2PtY], mC[kSigQ2PtZ], mC[kSigQ2PtSnp], mC[kSigQ2PtTgl],
+                     mC[kSigQ2Pt2]);
+}
+
+template 
+std::string TrackParametrizationWithError::asStringHexadecimal()
+{
+  return TrackParametrization::asStringHexadecimal() +
+         fmt::format(" Cov: [{:x}] [{:x} {:x}] [{:x} {:x} {:x}] [{:x} {:x} {:x} {:x}] [{:x} {:x} {:x} {:x} {:x}]",
+                     reinterpret_cast(mC[kSigY2]), reinterpret_cast(mC[kSigZY]), reinterpret_cast(mC[kSigZ2]),
+                     reinterpret_cast(mC[kSigSnpY]), reinterpret_cast(mC[kSigSnpZ]), reinterpret_cast(mC[kSigSnp2]),
+                     reinterpret_cast(mC[kSigTglY]), reinterpret_cast(mC[kSigTglZ]), reinterpret_cast(mC[kSigTglSnp]),
+                     reinterpret_cast(mC[kSigTgl2]), reinterpret_cast(mC[kSigQ2PtY]), reinterpret_cast(mC[kSigQ2PtZ]),
+                     reinterpret_cast(mC[kSigQ2PtSnp]), reinterpret_cast(mC[kSigQ2PtTgl]), reinterpret_cast(mC[kSigQ2Pt2]));
 }
 #endif
 
@@ -1134,13 +1739,41 @@ GPUd() void TrackParametrizationWithError::print() const
   // print parameters
 #ifndef GPUCA_ALIGPUCODE
   printf("%s\n", asString().c_str());
+#elif !defined(GPUCA_GPUCODE_DEVICE) || (!defined(__OPENCL__) && defined(GPUCA_GPU_DEBUG_PRINT))
+  TrackParametrization::printParam();
+  printf(
+    " Cov: [%+.3e] [%+.3e %+.3e] [%+.3e %+.3e %+.3e] [%+.3e %+.3e %+.3e %+.3e] [%+.3e %+.3e %+.3e %+.3e %+.3e]\n",
+    mC[kSigY2], mC[kSigZY], mC[kSigZ2], mC[kSigSnpY], mC[kSigSnpZ], mC[kSigSnp2], mC[kSigTglY],
+    mC[kSigTglZ], mC[kSigTglSnp], mC[kSigTgl2], mC[kSigQ2PtY], mC[kSigQ2PtZ], mC[kSigQ2PtSnp], mC[kSigQ2PtTgl],
+    mC[kSigQ2Pt2]);
+#endif
+}
+
+//______________________________________________________________
+template 
+GPUd() void TrackParametrizationWithError::printHexadecimal()
+{
+  // print parameters
+#ifndef GPUCA_ALIGPUCODE
+  printf("%s\n", asStringHexadecimal().c_str());
+#elif !defined(GPUCA_GPUCODE_DEVICE) || (!defined(__OPENCL__) && defined(GPUCA_GPU_DEBUG_PRINT))
+  TrackParametrization::printParamHexadecimal();
+  printf(
+    " Cov: [%x] [%x %x] [%x %x %x] [%x %x %x %x] [%x %x %x %x %x]\n",
+    gpu::CAMath::Float2UIntReint(mC[kSigY2]),
+    gpu::CAMath::Float2UIntReint(mC[kSigZY]), gpu::CAMath::Float2UIntReint(mC[kSigZ2]),
+    gpu::CAMath::Float2UIntReint(mC[kSigSnpY]), gpu::CAMath::Float2UIntReint(mC[kSigSnpZ]), gpu::CAMath::Float2UIntReint(mC[kSigSnp2]),
+    gpu::CAMath::Float2UIntReint(mC[kSigTglY]), gpu::CAMath::Float2UIntReint(mC[kSigTglZ]), gpu::CAMath::Float2UIntReint(mC[kSigTglSnp]), gpu::CAMath::Float2UIntReint(mC[kSigTgl2]),
+    gpu::CAMath::Float2UIntReint(mC[kSigQ2PtY]), gpu::CAMath::Float2UIntReint(mC[kSigQ2PtZ]), gpu::CAMath::Float2UIntReint(mC[kSigQ2PtSnp]), gpu::CAMath::Float2UIntReint(mC[kSigQ2PtTgl]), gpu::CAMath::Float2UIntReint(mC[kSigQ2Pt2]));
 #endif
 }
 
 namespace o2::track
 {
+#if !defined(GPUCA_GPUCODE) || defined(GPUCA_GPUCODE_DEVICE) // FIXME: DR: WORKAROUND to avoid CUDA bug creating host symbols for device code.
 template class TrackParametrizationWithError;
-#ifndef GPUCA_GPUCODE_DEVICE
+#endif
+#ifndef GPUCA_GPUCODE
 template class TrackParametrizationWithError;
 #endif
 } // namespace o2::track
diff --git a/DataFormats/Reconstruction/src/V0.cxx b/DataFormats/Reconstruction/src/V0.cxx
index ad2c84f499bf8..986d52fed35f6 100644
--- a/DataFormats/Reconstruction/src/V0.cxx
+++ b/DataFormats/Reconstruction/src/V0.cxx
@@ -14,20 +14,18 @@
 using namespace o2::dataformats;
 
 V0::V0(const std::array& xyz, const std::array& pxyz, const std::array& covxyz,
-       const o2::track::TrackParCov& trPos, const o2::track::TrackParCov& trNeg,
-       GIndex trPosID, GIndex trNegID, o2::track::PID pid)
-  : mProngIDs{trPosID, trNegID}, mProngs{trPos, trNeg}
+       const o2::track::TrackParCov& trPos, const o2::track::TrackParCov& trNeg, o2::track::PID pid) : mProngs{trPos, trNeg}
 {
-  std::array covV{}, covN{};
-  trPos.getCovXYZPxPyPzGlo(covV);
+  std::array covV{0.}, covP, covN;
+  trPos.getCovXYZPxPyPzGlo(covP);
   trNeg.getCovXYZPxPyPzGlo(covN);
-  for (int i = 0; i < 21; i++) {
-    covV[i] += covN[i];
-  }
+  constexpr int MomInd[6] = {9, 13, 14, 18, 19, 20}; // cov matrix elements for momentum component
   for (int i = 0; i < 6; i++) {
     covV[i] = covxyz[i];
+    covV[MomInd[i]] = covP[MomInd[i]] + covN[MomInd[i]];
   }
   this->set(xyz, pxyz, covV, trPos.getCharge() + trNeg.getCharge(), true, pid);
+  this->checkCorrelations();
 }
 
 float V0::calcMass2(float massPos2, float massNeg2) const
@@ -37,3 +35,36 @@ float V0::calcMass2(float massPos2, float massNeg2) const
   auto energy = std::sqrt(massPos2 + p2pos) + std::sqrt(massNeg2 + p2neg);
   return energy * energy - p2;
 }
+
+float V0::calcAPAlpha() const
+{
+  // calculate Armenteros-Podolanski alpha
+  std::array pP, pN, pV0;
+  float alp = 0.f, pV0tot2 = 0.f;
+  getProng(0).getPxPyPzGlo(pP);
+  getProng(1).getPxPyPzGlo(pN);
+  for (int i = 0; i < 3; i++) {
+    pV0[i] = pP[i] + pN[i];
+    alp += pV0[i] * (pP[i] - pN[i]);
+    pV0tot2 += pV0[i] * pV0[i];
+  }
+  alp /= pV0tot2;
+  return alp;
+}
+
+float V0::calcAPQt() const
+{
+  // calculate Armenteros-Podolanski qt
+  std::array pP, pN, pV0;
+  float pPtot2 = 0.f, pV0tot2 = 0.f, cross = 0.f;
+  getProng(0).getPxPyPzGlo(pP);
+  getProng(1).getPxPyPzGlo(pN);
+  for (int i = 0; i < 3; i++) {
+    pV0[i] = pP[i] + pN[i];
+    pPtot2 += pP[i] * pP[i];
+    pV0tot2 += pV0[i] * pV0[i];
+    cross += pV0[i] * pP[i]; // -> pP * pV0 * cos
+  }
+  float qt = pPtot2 - (cross * cross) / pV0tot2;
+  return qt > 0 ? std::sqrt(qt) : 0;
+}
diff --git a/DataFormats/Reconstruction/src/Vertex.cxx b/DataFormats/Reconstruction/src/Vertex.cxx
index dfbc5eac281e8..85145683ddd97 100644
--- a/DataFormats/Reconstruction/src/Vertex.cxx
+++ b/DataFormats/Reconstruction/src/Vertex.cxx
@@ -10,9 +10,9 @@
 // or submit itself to any jurisdiction.
 
 #include "ReconstructionDataFormats/Vertex.h"
-#include 
 #ifndef GPUCA_NO_FMT
-#include 
+#include 
+#include 
 #endif
 
 namespace o2
@@ -56,5 +56,8 @@ bool VertexBase::operator==(const VertexBase& other) const
 
 #endif
 
+template class o2::dataformats::Vertex>;
+template class o2::dataformats::Vertex>;
+
 } // namespace dataformats
 } // namespace o2
diff --git a/DataFormats/Reconstruction/test/testLTOFIntegration.cxx b/DataFormats/Reconstruction/test/testLTOFIntegration.cxx
index bb65c60d08d18..f737b1df53666 100644
--- a/DataFormats/Reconstruction/test/testLTOFIntegration.cxx
+++ b/DataFormats/Reconstruction/test/testLTOFIntegration.cxx
@@ -33,8 +33,8 @@ BOOST_AUTO_TEST_CASE(TrackLTIntegral)
   const int nStep = 100;
   const float dx2x0 = 0.01f;
   for (int i = 0; i < nStep; i++) {
-    lt.addStep(1., trc.getP2Inv());
-    lt1.addStep(1., trc1.getP2Inv());
+    lt.addStep(1., trc.getQ2P2());
+    lt1.addStep(1., trc1.getQ2P2());
     lt1.addX2X0(dx2x0);
   }
   trc.printParam();
diff --git a/DataFormats/common/CMakeLists.txt b/DataFormats/common/CMakeLists.txt
index 9d6fb40e7696d..6b0081a920c8d 100644
--- a/DataFormats/common/CMakeLists.txt
+++ b/DataFormats/common/CMakeLists.txt
@@ -16,8 +16,9 @@ o2_add_library(CommonDataFormat
                        src/FlatHisto2D.cxx
                        src/AbstractRefAccessor.cxx
                        src/TFIDInfo.cxx
+                       src/TimeStamp.cxx
                PUBLIC_LINK_LIBRARIES O2::CommonConstants O2::GPUCommon
-                                     ROOT::Core FairRoot::Base O2::MathUtils Microsoft.GSL::GSL)
+                                     ROOT::Core O2::MathUtils Microsoft.GSL::GSL)
 
 o2_target_root_dictionary(CommonDataFormat
                           HEADERS include/CommonDataFormat/TimeStamp.h
@@ -30,6 +31,7 @@ o2_target_root_dictionary(CommonDataFormat
                                   include/CommonDataFormat/FlatHisto1D.h
                                   include/CommonDataFormat/FlatHisto2D.h
                                   include/CommonDataFormat/TFIDInfo.h
+                                  include/CommonDataFormat/Triplet.h
                                   include/CommonDataFormat/Pair.h)
 
 o2_add_test(TimeStamp
diff --git a/DataFormats/common/include/CommonDataFormat/AbstractRef.h b/DataFormats/common/include/CommonDataFormat/AbstractRef.h
index cdc65f6c412d6..72c195cfb7bc8 100644
--- a/DataFormats/common/include/CommonDataFormat/AbstractRef.h
+++ b/DataFormats/common/include/CommonDataFormat/AbstractRef.h
@@ -18,11 +18,12 @@
 
 #include "GPUCommonDef.h"
 #include "GPUCommonRtypes.h"
-#include "GPUCommonTypeTraits.h"
+#ifndef GPUCA_GPUCODE_DEVICE
+#include 
+#endif
 
-namespace o2
-{
-namespace dataformats
+
+namespace o2::dataformats
 {
 
 template 
@@ -32,8 +33,19 @@ class AbstractRef
   static constexpr auto MVAR()
   {
     static_assert(NBIT <= 64, "> 64 bits not supported");
-    typename std::conditional<(NBIT > 32), uint64_t, typename std::conditional<(NBIT > 16), uint32_t, typename std::conditional<(NBIT > 8), uint16_t, uint8_t>::type>::type>::type tp = 0;
-    return tp;
+    if constexpr (NBIT > 32) {
+      uint64_t tp = 0;
+      return tp;
+    } else if constexpr (NBIT > 16) {
+      uint32_t tp = 0;
+      return tp;
+    } else if constexpr (NBIT > 8) {
+      uint16_t tp = 0;
+      return tp;
+    } else {
+      uint8_t tp = 0;
+      return tp;
+    }
   }
 
  public:
@@ -79,6 +91,7 @@ class AbstractRef
 
   GPUdi() bool operator==(const AbstractRef& o) const { return getRawWOFlags() == o.getRawWOFlags(); }
   GPUdi() bool operator!=(const AbstractRef& o) const { return !operator==(o); }
+  GPUdi() void clear() { setRaw((Base_t(SrcMask & ((0x1 << NBSrc) - 1)) << NBIdx) | Base_t(IdxMask & ((0x1 << NBIdx) - 1))); }
 
  protected:
   Base_t mRef = IdxMask | (SrcMask << NBIdx); // packed reference, dummy by default
@@ -86,7 +99,6 @@ class AbstractRef
   ClassDefNV(AbstractRef, 1);
 };
 
-} // namespace dataformats
-} // namespace o2
+} // namespace o2::dataformats
 
 #endif
diff --git a/DataFormats/common/include/CommonDataFormat/BunchFilling.h b/DataFormats/common/include/CommonDataFormat/BunchFilling.h
index 182a665532668..f11ce2498d04b 100644
--- a/DataFormats/common/include/CommonDataFormat/BunchFilling.h
+++ b/DataFormats/common/include/CommonDataFormat/BunchFilling.h
@@ -107,7 +107,6 @@ class BunchFilling
 
   ClassDefNV(BunchFilling, 2);
 };
-} // namespace o2
 
 namespace framework
 {
@@ -118,5 +117,6 @@ struct is_messageable : std::true_type {
 };
 
 } // namespace framework
+} // namespace o2
 
 #endif
diff --git a/DataFormats/common/include/CommonDataFormat/FlatHisto1D.h b/DataFormats/common/include/CommonDataFormat/FlatHisto1D.h
index ac91eafc4d147..f8d49042dd8c5 100644
--- a/DataFormats/common/include/CommonDataFormat/FlatHisto1D.h
+++ b/DataFormats/common/include/CommonDataFormat/FlatHisto1D.h
@@ -34,7 +34,7 @@ namespace dataformats
   Fast 1D histo class which can be messages as
   FlatHisto1D histo(nbins, xmin, xmax);
   histo.fill(...);
-  pc.outputs().snapshot(Output{"Origin", "Desc", 0, Lifetime::Timeframe}, histo.getBase());
+  pc.outputs().snapshot(Output{"Origin", "Desc", 0}, histo.getBase());
 
   and received (read only!) as
   const auto hdata = pc.inputs().get>("histodata");
diff --git a/DataFormats/common/include/CommonDataFormat/FlatHisto2D.h b/DataFormats/common/include/CommonDataFormat/FlatHisto2D.h
index 9102d53edeaaa..47793618c0649 100644
--- a/DataFormats/common/include/CommonDataFormat/FlatHisto2D.h
+++ b/DataFormats/common/include/CommonDataFormat/FlatHisto2D.h
@@ -35,7 +35,7 @@ namespace dataformats
   Fast 2D histo class which can be messages as
   FlatHisto2D histo(nbinsX, xmin, xmax, nbinsY, ymin, ymax);
   histo.fill(...);
-  pc.outputs().snapshot(Output{"Origin", "Desc", 0, Lifetime::Timeframe}, histo.getBase());
+  pc.outputs().snapshot(Output{"Origin", "Desc", 0}, histo.getBase());
 
   and received (read only!) as
   const auto hdata = pc.inputs().get>("histodata");
diff --git a/DataFormats/common/include/CommonDataFormat/IRFrame.h b/DataFormats/common/include/CommonDataFormat/IRFrame.h
index 26b1e33f17e50..d404a9538306a 100644
--- a/DataFormats/common/include/CommonDataFormat/IRFrame.h
+++ b/DataFormats/common/include/CommonDataFormat/IRFrame.h
@@ -28,9 +28,22 @@ namespace dataformats
 // We could just alias it to the bracket specialization, but this would create
 // problems with fwd.declaration
 struct IRFrame : public o2::math_utils::detail::Bracket {
+  static constexpr uint64_t LastIRFrame = uint64_t(0x1) << 63;
+
   using o2::math_utils::detail::Bracket::Bracket;
 
-  uint64_t info = 0;
+  uint64_t info = uint64_t(0);
+
+  void setLast(bool v = true)
+  {
+    if (v) {
+      info |= LastIRFrame;
+    } else {
+      v &= ~LastIRFrame;
+    }
+  }
+
+  bool isLast() const { return (info & LastIRFrame) != 0UL; }
 
   ClassDefNV(IRFrame, 2);
 };
diff --git a/DataFormats/common/include/CommonDataFormat/InteractionRecord.h b/DataFormats/common/include/CommonDataFormat/InteractionRecord.h
index 43e7185bc6205..e99f338a16343 100644
--- a/DataFormats/common/include/CommonDataFormat/InteractionRecord.h
+++ b/DataFormats/common/include/CommonDataFormat/InteractionRecord.h
@@ -29,22 +29,24 @@ struct InteractionRecord {
   static constexpr uint16_t DummyBC = 0xffff;
   static constexpr uint32_t DummyOrbit = 0xffffffff;
   static constexpr double DummyTime = DummyBC * o2::constants::lhc::LHCBunchSpacingNS + DummyOrbit * o2::constants::lhc::LHCOrbitNS;
-
+  static constexpr int64_t MaxGlobalBCs = (int64_t(DummyOrbit) * o2::constants::lhc::LHCMaxBunches) + (o2::constants::lhc::LHCMaxBunches - 1);
+  static constexpr InteractionRecord getIRMaxBC() { return {o2::constants::lhc::LHCMaxBunches - 1, DummyOrbit}; }
   uint16_t bc = DummyBC;       ///< bunch crossing ID of interaction
   uint32_t orbit = DummyOrbit; ///< LHC orbit
 
-  InteractionRecord() = default;
+  constexpr InteractionRecord() = default;
 
   InteractionRecord(double tNS)
   {
     setFromNS(tNS);
   }
 
-  InteractionRecord(uint16_t b, uint32_t orb) : bc(b), orbit(orb)
+  constexpr InteractionRecord(uint16_t b, uint32_t orb) : bc(b), orbit(orb)
   {
   }
 
   InteractionRecord(const InteractionRecord& src) = default;
+  InteractionRecord& operator=(const InteractionRecord& src) = default;
 
   void clear()
   {
@@ -69,9 +71,9 @@ struct InteractionRecord {
 
   static int ns2bc(double ns, unsigned int& orb)
   {
-    orb = ns > 0 ? ns / o2::constants::lhc::LHCOrbitNS : 0;
-    ns -= orb * o2::constants::lhc::LHCOrbitNS;
-    return std::round(ns / o2::constants::lhc::LHCBunchSpacingNS);
+    long nb = std::round(ns / o2::constants::lhc::LHCBunchSpacingNS);
+    orb = nb / o2::constants::lhc::LHCMaxBunches;
+    return nb % o2::constants::lhc::LHCMaxBunches;
   }
 
   double bc2ns() const
@@ -107,7 +109,7 @@ struct InteractionRecord {
 
   float differenceInBCMUS(const InteractionRecord& other) const
   {
-    // return difference in bunch-crossings in ms
+    // return difference in bunch-crossings in us
     return differenceInBC(other) * o2::constants::lhc::LHCBunchSpacingMUS;
   }
 
@@ -152,10 +154,14 @@ struct InteractionRecord {
 
   InteractionRecord operator--()
   {
-    // prefix decrement operator, no check for orbit wrap
+    // prefix decrement operator
     if (!bc--) {
       orbit--;
       bc = o2::constants::lhc::LHCMaxBunches - 1;
+      if (orbit == DummyOrbit) { // wrapped?
+        orbit = 0;
+        bc = 0;
+      }
     }
     return InteractionRecord(*this);
   }
@@ -167,35 +173,59 @@ struct InteractionRecord {
     if (!bc--) {
       orbit--;
       bc = o2::constants::lhc::LHCMaxBunches - 1;
+      if (orbit == DummyOrbit) { // wrapped?
+        orbit = 0;
+        bc = 0;
+      }
     }
     return tmp;
   }
 
   InteractionRecord operator++()
   {
-    // prefix increment operator,no check for orbit wrap
+    // prefix increment operator
     if ((++bc) == o2::constants::lhc::LHCMaxBunches) {
       orbit++;
       bc = 0;
+      if (orbit == 0) { // wrapped?
+        orbit = DummyOrbit;
+        bc = o2::constants::lhc::LHCMaxBunches - 1;
+      }
     }
     return InteractionRecord(*this);
   }
 
   InteractionRecord operator++(int)
   {
-    // postfix increment operator, no check for orbit wrap
+    // postfix increment operator
     InteractionRecord tmp(*this);
     if ((++bc) == o2::constants::lhc::LHCMaxBunches) {
       orbit++;
       bc = 0;
+      if (orbit == 0) { // wrapped?
+        orbit = DummyOrbit;
+        bc = o2::constants::lhc::LHCMaxBunches - 1;
+      }
     }
     return tmp;
   }
 
   InteractionRecord& operator+=(int64_t dbc)
   {
-    // bc self-addition operator, no check for orbit wrap
-    auto l = toLong() + dbc;
+    // bc self-addition operator, avoid wrapping
+    auto l = toLong();
+    if (dbc >= 0) {
+      if (MaxGlobalBCs - dbc < l) {
+        l = MaxGlobalBCs;
+        dbc = 0;
+      }
+    } else {
+      if (l < -dbc) {
+        l = 0;
+        dbc = 0;
+      }
+    }
+    l += dbc;
     bc = l % o2::constants::lhc::LHCMaxBunches;
     orbit = l / o2::constants::lhc::LHCMaxBunches;
     return *this;
@@ -203,57 +233,55 @@ struct InteractionRecord {
 
   InteractionRecord& operator-=(int64_t dbc)
   {
-    // bc self-subtraction operator, no check for orbit wrap
+    // bc self-subtraction operator
     return operator+=(-dbc);
   }
 
   InteractionRecord& operator+=(const InteractionRecord& add)
   {
-    // InteractionRecord self-addition operator, no check for orbit wrap
-    auto l = this->toLong() + add.toLong();
-    bc = l % o2::constants::lhc::LHCMaxBunches;
-    orbit = l / o2::constants::lhc::LHCMaxBunches;
-    return *this;
+    // InteractionRecord self-addition operator
+    return operator+=(add.toLong());
   }
 
   InteractionRecord& operator-=(const InteractionRecord& add)
   {
-    // InteractionRecord self-subtraction operator, no check for orbit wrap
-    auto l = this->toLong() - add.toLong();
-    bc = l % o2::constants::lhc::LHCMaxBunches;
-    orbit = l / o2::constants::lhc::LHCMaxBunches;
-    return *this;
+    // InteractionRecord self-subtraction operator
+    return operator-=(add.toLong());
   }
 
   InteractionRecord operator+(int64_t dbc) const
   {
-    // bc addition operator, no check for orbit wrap
-    auto l = toLong() + dbc;
-    return InteractionRecord(l % o2::constants::lhc::LHCMaxBunches, l / o2::constants::lhc::LHCMaxBunches);
+    // bc addition operator
+    InteractionRecord tmp(*this);
+    tmp += dbc;
+    return tmp;
   }
 
   InteractionRecord operator-(int64_t dbc) const
   {
-    // bc subtraction operator, no check for orbit wrap
-    auto l = toLong() - dbc;
-    return InteractionRecord(l % o2::constants::lhc::LHCMaxBunches, l / o2::constants::lhc::LHCMaxBunches);
+    // bc subtraction operator
+    InteractionRecord tmp(*this);
+    tmp -= dbc;
+    return tmp;
   }
 
   InteractionRecord operator+(const InteractionRecord& add) const
   {
     // InteractionRecord addition operator, no check for orbit wrap
-    auto l = this->toLong() + add.toLong();
-    return InteractionRecord(l % o2::constants::lhc::LHCMaxBunches, l / o2::constants::lhc::LHCMaxBunches);
+    InteractionRecord tmp(*this);
+    tmp += add;
+    return tmp;
   }
 
   InteractionRecord operator-(const InteractionRecord& add) const
   {
     // InteractionRecord subtraction operator, no check for orbit wrap
-    auto l = this->toLong() - add.toLong();
-    return InteractionRecord(l % o2::constants::lhc::LHCMaxBunches, l / o2::constants::lhc::LHCMaxBunches);
+    InteractionRecord tmp(*this);
+    tmp -= add;
+    return tmp;
   }
 
-#ifndef GPUCA_ALIGPUCODE
+#if !defined(GPUCA_GPUCODE) && !defined(GPUCA_STANDALONE)
   void print() const;
   std::string asString() const;
   friend std::ostream& operator<<(std::ostream& stream, InteractionRecord const& ir);
@@ -331,7 +359,7 @@ struct InteractionTimeRecord : public InteractionRecord {
     return !((*this) > other);
   }
 
-#ifndef GPUCA_ALIGPUCODE
+#if !defined(GPUCA_GPUCODE) && !defined(GPUCA_STANDALONE)
   void print() const;
   std::string asString() const;
   friend std::ostream& operator<<(std::ostream& stream, InteractionTimeRecord const& ir);
diff --git a/DataFormats/common/include/CommonDataFormat/RangeReference.h b/DataFormats/common/include/CommonDataFormat/RangeReference.h
index 0308d3b8af937..3d0c58298de03 100644
--- a/DataFormats/common/include/CommonDataFormat/RangeReference.h
+++ b/DataFormats/common/include/CommonDataFormat/RangeReference.h
@@ -29,23 +29,23 @@ template 
 class RangeReference
 {
  public:
-  GPUd() RangeReference(FirstEntry ent, NElem n) { set(ent, n); }
-  GPUdDefault() RangeReference(const RangeReference& src) = default;
-  GPUdDefault() RangeReference() = default;
-  GPUdDefault() ~RangeReference() = default;
-  GPUd() void set(FirstEntry ent, NElem n)
+  GPUhd() RangeReference(FirstEntry ent, NElem n) { set(ent, n); }
+  GPUhdDefault() RangeReference(const RangeReference& src) = default;
+  GPUhdDefault() RangeReference() = default;
+  GPUhdDefault() ~RangeReference() = default;
+  GPUhd() void set(FirstEntry ent, NElem n)
   {
     mFirstEntry = ent;
     mEntries = n;
   }
-  GPUd() void clear() { set(0, 0); }
-  GPUd() FirstEntry getFirstEntry() const { return mFirstEntry; }
-  GPUd() FirstEntry getEntriesBound() const { return mFirstEntry + mEntries; }
-  GPUd() NElem getEntries() const { return mEntries; }
-  GPUd() void setFirstEntry(FirstEntry ent) { mFirstEntry = ent; }
-  GPUd() void setEntries(NElem n) { mEntries = n; }
-  GPUd() void changeEntriesBy(NElem inc) { mEntries += inc; }
-  GPUd() bool operator==(const RangeReference& other) const
+  GPUhd() void clear() { set(0, 0); }
+  GPUhd() FirstEntry getFirstEntry() const { return mFirstEntry; }
+  GPUhd() FirstEntry getEntriesBound() const { return mFirstEntry + mEntries; }
+  GPUhd() NElem getEntries() const { return mEntries; }
+  GPUhd() void setFirstEntry(FirstEntry ent) { mFirstEntry = ent; }
+  GPUhd() void setEntries(NElem n) { mEntries = n; }
+  GPUhd() void changeEntriesBy(NElem inc) { mEntries += inc; }
+  GPUhd() bool operator==(const RangeReference& other) const
   {
     return mFirstEntry == other.mFirstEntry && mEntries == other.mEntries;
   }
@@ -68,21 +68,21 @@ class RangeRefComp
   static constexpr Base MaskN = ((0x1 << NBitsN) - 1);
   static constexpr Base MaskR = (~Base(0)) & (~MaskN);
   Base mData = 0; ///< packed 1st entry reference + N entries
-  GPUd() void sanityCheck()
+  GPUhd() void sanityCheck()
   {
     static_assert(NBitsN < NBitsTotal, "NBitsN too large");
   }
 
  public:
-  GPUd() RangeRefComp(int ent, int n) { set(ent, n); }
-  GPUdDefault() RangeRefComp() = default;
-  GPUdDefault() RangeRefComp(const RangeRefComp& src) = default;
+  GPUhd() RangeRefComp(int ent, int n) { set(ent, n); }
+  GPUhdDefault() RangeRefComp() = default;
+  GPUhdDefault() RangeRefComp(const RangeRefComp& src) = default;
   GPUhd() void set(int ent, int n)
   {
     mData = (Base(ent) << NBitsN) + (Base(n) & MaskN);
   }
-  GPUd() static constexpr Base getMaxFirstEntry() { return MaskR >> NBitsN; }
-  GPUd() static constexpr Base getMaxEntries() { return MaskN; }
+  GPUhd() static constexpr Base getMaxFirstEntry() { return MaskR >> NBitsN; }
+  GPUhd() static constexpr Base getMaxEntries() { return MaskN; }
   GPUhd() int getFirstEntry() const { return mData >> NBitsN; }
   GPUhd() int getEntries() const { return mData & ((0x1 << NBitsN) - 1); }
   GPUhd() int getEntriesBound() const { return getFirstEntry() + getEntries(); }
diff --git a/DataFormats/common/include/CommonDataFormat/TimeStamp.h b/DataFormats/common/include/CommonDataFormat/TimeStamp.h
index c4732523d6e93..56a71414c6b86 100644
--- a/DataFormats/common/include/CommonDataFormat/TimeStamp.h
+++ b/DataFormats/common/include/CommonDataFormat/TimeStamp.h
@@ -25,10 +25,10 @@ template 
 class TimeStamp
 {
  public:
-  GPUhdDefault() TimeStamp() CON_DEFAULT;
-  GPUhdDefault() ~TimeStamp() CON_DEFAULT;
+  GPUhdDefault() TimeStamp() = default;
+  GPUhdDefault() ~TimeStamp() = default;
   GPUdi() TimeStamp(T time) { mTimeStamp = time; }
-  GPUdi() T getTimeStamp() const { return mTimeStamp; }
+  GPUhdi() T getTimeStamp() const { return mTimeStamp; }
   GPUdi() void setTimeStamp(T t) { mTimeStamp = t; }
   GPUdi() bool operator==(const TimeStamp& t) const { return mTimeStamp == t.mTimeStamp; }
 
@@ -44,6 +44,7 @@ class TimeStampWithError : public TimeStamp
   GPUdDefault() TimeStampWithError() = default;
   GPUd() TimeStampWithError(T t, E te) : TimeStamp(t), mTimeStampError(te) {}
   GPUdi() E getTimeStampError() const { return mTimeStampError; }
+  GPUdi() E getTimeStampError2() const { return mTimeStampError * mTimeStampError; }
   GPUdi() void setTimeStampError(E te) { mTimeStampError = te; }
 
  private:
diff --git a/DataFormats/common/include/CommonDataFormat/Triplet.h b/DataFormats/common/include/CommonDataFormat/Triplet.h
new file mode 100644
index 0000000000000..ed0b43a964726
--- /dev/null
+++ b/DataFormats/common/include/CommonDataFormat/Triplet.h
@@ -0,0 +1,35 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef ALICEO2_COMMON_TRIPLET_H
+#define ALICEO2_COMMON_TRIPLET_H
+
+#include "GPUCommonRtypes.h"
+
+namespace o2
+{
+namespace dataformats
+{
+// A messageable tripplet
+template 
+struct Triplet {
+  Triplet() = default;
+  Triplet(F f, S s, T t) : first(f), second(s), third(t) {}
+  F first{};
+  S second{};
+  T third{};
+  ClassDefNV(Triplet, 1);
+};
+
+} // namespace dataformats
+} // namespace o2
+
+#endif /* ALICEO2_COMMON_TRIPPLET_H */
diff --git a/DataFormats/common/src/CommonDataFormatLinkDef.h b/DataFormats/common/src/CommonDataFormatLinkDef.h
index 1463f707b72fa..d66e89af637cc 100644
--- a/DataFormats/common/src/CommonDataFormatLinkDef.h
+++ b/DataFormats/common/src/CommonDataFormatLinkDef.h
@@ -26,10 +26,12 @@
 #pragma link C++ class o2::dataformats::TimeStamp < float> + ;
 #pragma link C++ class o2::dataformats::TimeStamp < double> + ;
 #pragma link C++ class o2::dataformats::TimeStamp < int> + ;
-#pragma link C++ class o2::dataformats::TimeStamp < Float16_t > + ;
+#pragma link C++ class o2::dataformats::TimeStamp < uint32_t> + ;
+#pragma link C++ class o2::dataformats::TimeStamp < Float16_t> + ;
 #pragma link C++ class o2::dataformats::TimeStampWithError < float, float> + ;
 #pragma link C++ class o2::dataformats::TimeStampWithError < double, double> + ;
 #pragma link C++ class o2::dataformats::TimeStampWithError < int, int> + ;
+#pragma link C++ class o2::dataformats::TimeStampWithError < uint32_t, uint16_t> + ;
 
 #pragma link C++ class o2::dataformats::EvIndex < int, int> + ;
 #pragma link C++ class o2::dataformats::RangeReference < int, int> + ;
@@ -68,6 +70,9 @@
 #pragma link C++ class o2::dataformats::Pair < float, float> + ;
 #pragma link C++ class std::vector < o2::dataformats::Pair < float, float>> + ;
 
+#pragma link C++ class o2::dataformats::Triplet < float, float, float> + ;
+#pragma link C++ class std::vector < o2::dataformats::Triplet < float, float, float>> + ;
+
 #include "CommonDataFormat/TFIDInfo.h"
 #pragma link C++ class o2::dataformats::TFIDInfo + ;
 #pragma link C++ class std::vector < o2::dataformats::TFIDInfo> + ;
diff --git a/DataFormats/common/src/FlatHisto1D.cxx b/DataFormats/common/src/FlatHisto1D.cxx
index 1313da72da8b7..8e04c2c58614e 100644
--- a/DataFormats/common/src/FlatHisto1D.cxx
+++ b/DataFormats/common/src/FlatHisto1D.cxx
@@ -40,9 +40,10 @@ FlatHisto1D& FlatHisto1D::operator=(const FlatHisto1D& rhs)
   if (this == &rhs) {
     return *this;
   }
-  if (!rhs.canFill()) {
+  if (!rhs.canFill() && rhs.getNBins()) { // was initialized
     throw std::runtime_error("trying to copy read-only histogram");
-  } else {
+  }
+  if (rhs.getNBins()) {
     mContainer = rhs.mContainer;
     init(gsl::span(mContainer.data(), mContainer.size()));
   }
diff --git a/DataFormats/common/src/FlatHisto2D.cxx b/DataFormats/common/src/FlatHisto2D.cxx
index b06241ff8d193..aa064eee10bca 100644
--- a/DataFormats/common/src/FlatHisto2D.cxx
+++ b/DataFormats/common/src/FlatHisto2D.cxx
@@ -43,9 +43,10 @@ FlatHisto2D& FlatHisto2D::operator=(const FlatHisto2D& rhs)
   if (this == &rhs) {
     return *this;
   }
-  if (!rhs.canFill()) {
+  if (!rhs.canFill() && rhs.getNBins()) { // was initialized
     throw std::runtime_error("trying to copy read-only histogram");
-  } else {
+  }
+  if (rhs.getNBins()) {
     mContainer = rhs.mContainer;
     init(gsl::span(mContainer.data(), mContainer.size()));
   }
diff --git a/DataFormats/common/src/TimeStamp.cxx b/DataFormats/common/src/TimeStamp.cxx
new file mode 100644
index 0000000000000..2681c9721ba42
--- /dev/null
+++ b/DataFormats/common/src/TimeStamp.cxx
@@ -0,0 +1,19 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "CommonDataFormat/TimeStamp.h"
+
+template class o2::dataformats::TimeStamp;
+template class o2::dataformats::TimeStamp;
+template class o2::dataformats::TimeStamp;
+template class o2::dataformats::TimeStampWithError;
+template class o2::dataformats::TimeStampWithError;
+template class o2::dataformats::TimeStampWithError;
diff --git a/DataFormats/common/test/testRangeRef.cxx b/DataFormats/common/test/testRangeRef.cxx
index bdf22bae860c6..f0fa50f5b078a 100644
--- a/DataFormats/common/test/testRangeRef.cxx
+++ b/DataFormats/common/test/testRangeRef.cxx
@@ -14,7 +14,7 @@
 #define BOOST_TEST_DYN_LINK
 #include 
 #include "CommonDataFormat/RangeReference.h"
-#include 
+#include 
 
 namespace o2
 {
diff --git a/DataFormats/simulation/CMakeLists.txt b/DataFormats/simulation/CMakeLists.txt
index 1dbe2a17415ad..33c91337c77e9 100644
--- a/DataFormats/simulation/CMakeLists.txt
+++ b/DataFormats/simulation/CMakeLists.txt
@@ -10,8 +10,7 @@
 # or submit itself to any jurisdiction.
 
 o2_add_library(SimulationDataFormat
-               SOURCES src/Stack.cxx
-                       src/MCTrack.cxx
+               SOURCES src/MCTrack.cxx
                        src/MCCompLabel.cxx
                        src/MCEventLabel.cxx
                        src/DigitizationContext.cxx
@@ -20,15 +19,20 @@ o2_add_library(SimulationDataFormat
                        src/CustomStreamers.cxx
                        src/MCUtils.cxx
                        src/O2DatabasePDG.cxx
+                       src/InteractionSampler.cxx
+                       src/ConstMCTruthContainer.cxx
                PUBLIC_LINK_LIBRARIES Microsoft.GSL::GSL
+                                     FairRoot::Base
                                      O2::DetectorsCommonDataFormats
-                                     O2::GPUCommon O2::DetectorsBase
-                                     O2::SimConfig)
+                                     O2::DataFormatsParameters
+                                     O2::DataFormatsCalibration
+                                     O2::DataFormatsCTP
+                                     O2::GPUCommon
+                                     ROOT::TreePlayer)
 
 o2_target_root_dictionary(
   SimulationDataFormat
-  HEADERS include/SimulationDataFormat/Stack.h
-          include/SimulationDataFormat/StackParam.h
+  HEADERS include/SimulationDataFormat/StackParam.h
           include/SimulationDataFormat/MCTrack.h
           include/SimulationDataFormat/BaseHits.h
           include/SimulationDataFormat/MCTruthContainer.h
@@ -44,12 +48,18 @@ o2_target_root_dictionary(
           include/SimulationDataFormat/IOMCTruthContainerView.h
           include/SimulationDataFormat/MCUtils.h
           include/SimulationDataFormat/O2DatabasePDG.h
+          include/SimulationDataFormat/InteractionSampler.h
   LINKDEF src/SimulationDataLinkDef.h)
 # note the explicit LINKDEF as the linkdef in src is
 #
 # * src/SimulationDataLinkDef.h
 # * and not src/SimulationDataFormatLinkDef.h
 
+o2_add_test(InteractionSampler
+            SOURCES test/testInteractionSampler.cxx
+            COMPONENT_NAME SimulationDataFormat
+            PUBLIC_LINK_LIBRARIES O2::SimulationDataFormat)
+
 o2_add_test(BasicHits
             SOURCES test/testBasicHits.cxx
             COMPONENT_NAME SimulationDataFormat
@@ -74,3 +84,13 @@ o2_add_test(MCTrack
             SOURCES test/MCTrack.cxx
             COMPONENT_NAME SimulationDataFormat
             PUBLIC_LINK_LIBRARIES O2::SimulationDataFormat)
+
+o2_add_test(MCGenStatus
+            SOURCES test/testMCGenStatus.cxx
+            COMPONENT_NAME SimulationDataFormat
+            PUBLIC_LINK_LIBRARIES O2::SimulationDataFormat)
+
+o2_add_test(MCGenId
+            SOURCES test/testMCGenId.cxx
+            COMPONENT_NAME SimulationDataFormat
+            PUBLIC_LINK_LIBRARIES O2::SimulationDataFormat)
diff --git a/DataFormats/simulation/include/SimulationDataFormat/BaseHits.h b/DataFormats/simulation/include/SimulationDataFormat/BaseHits.h
index b9ed356ec8b5a..d1e1ee357c1cf 100644
--- a/DataFormats/simulation/include/SimulationDataFormat/BaseHits.h
+++ b/DataFormats/simulation/include/SimulationDataFormat/BaseHits.h
@@ -49,13 +49,13 @@ class BasicXYZVHit : public BaseHit
   math_utils::Point3D mPos; // cartesian position of Hit
   E mTime;                     // time of flight
   V mHitValue;                 // hit value
-  short mDetectorID;           // the detector/sensor id
+  unsigned short mDetectorID;  // the detector/sensor id
 
  public:
   BasicXYZVHit() = default; // for ROOT IO
 
   // constructor
-  BasicXYZVHit(T x, T y, T z, E time, V val, int trackid, short did)
+  BasicXYZVHit(T x, T y, T z, E time, V val, int trackid, unsigned short did)
     : mPos(x, y, z), mTime(time), mHitValue(val), BaseHit(trackid), mDetectorID(did)
   {
   }
@@ -70,12 +70,12 @@ class BasicXYZVHit : public BaseHit
   // getting the time
   E GetTime() const { return mTime; }
   // get detector + track information
-  short GetDetectorID() const { return mDetectorID; }
+  unsigned short GetDetectorID() const { return mDetectorID; }
 
   // modifiers
   void SetTime(E time) { mTime = time; }
   void SetHitValue(V val) { mHitValue = val; }
-  void SetDetectorID(short detID) { mDetectorID = detID; }
+  void SetDetectorID(unsigned short detID) { mDetectorID = detID; }
   void SetX(T x) { mPos.SetX(x); }
   void SetY(T y) { mPos.SetY(y); }
   void SetZ(T z) { mPos.SetZ(z); }
@@ -87,7 +87,7 @@ class BasicXYZVHit : public BaseHit
   }
   void SetPos(math_utils::Point3D const& p) { mPos = p; }
 
-  ClassDefNV(BasicXYZVHit, 1);
+  ClassDefNV(BasicXYZVHit, 2);
 };
 
 // Class for a hit containing energy loss as hit value
diff --git a/DataFormats/simulation/include/SimulationDataFormat/ConstMCTruthContainer.h b/DataFormats/simulation/include/SimulationDataFormat/ConstMCTruthContainer.h
index 64a8b02c856ab..72c66c863f698 100644
--- a/DataFormats/simulation/include/SimulationDataFormat/ConstMCTruthContainer.h
+++ b/DataFormats/simulation/include/SimulationDataFormat/ConstMCTruthContainer.h
@@ -21,6 +21,8 @@
 #include 
 #endif
 
+class TTree;
+
 namespace o2
 {
 namespace dataformats
@@ -137,7 +139,7 @@ class ConstMCTruthContainerView
   {
     (void)0;
   }
-  ConstMCTruthContainerView(const ConstMCTruthContainerView&) = default;
+  // ConstMCTruthContainerView(const ConstMCTruthContainerView&) = default;
 
   // const data access
   // get individual const "view" container for a given data index
@@ -211,6 +213,14 @@ class ConstMCTruthContainerView
 using ConstMCLabelContainer = o2::dataformats::ConstMCTruthContainer;
 using ConstMCLabelContainerView = o2::dataformats::ConstMCTruthContainerView;
 
+class MCLabelIOHelper
+{
+ public:
+  /// Convenience function to loads MC labels for some entry from a TTree and TBranch.
+  /// Labels can be stored as either MCTruthContainer or IOMCTruthContainer. The caller takes ownership of the returned pointer.
+  static ConstMCTruthContainer* loadFromTTree(TTree* tree, std::string const& brname, int entry);
+};
+
 } // namespace dataformats
 } // namespace o2
 
diff --git a/DataFormats/simulation/include/SimulationDataFormat/DigitizationContext.h b/DataFormats/simulation/include/SimulationDataFormat/DigitizationContext.h
index 264ac1a48e15b..0dc3806e52cf2 100644
--- a/DataFormats/simulation/include/SimulationDataFormat/DigitizationContext.h
+++ b/DataFormats/simulation/include/SimulationDataFormat/DigitizationContext.h
@@ -20,6 +20,10 @@
 #include "DetectorsCommonDataFormats/DetID.h"
 #include "DataFormatsParameters/GRPObject.h"
 #include 
+#include 
+#include 
+#include 
+#include 
 
 namespace o2
 {
@@ -43,7 +47,10 @@ struct EventPart {
   ClassDefNV(EventPart, 1);
 };
 
-// class fully describing the Collision contexts
+// Class fully describing the Collision context or timeframe structure.
+// The context fixes things such as times (orbits and bunch crossings)
+// at which collision happen inside a timeframe and how they are composed
+// in terms of MC events.
 class DigitizationContext
 {
  public:
@@ -64,6 +71,9 @@ class DigitizationContext
   const std::vector& getEventRecords(bool withQED = false) const { return withQED ? mEventRecordsWithQED : mEventRecords; }
   const std::vector>& getEventParts(bool withQED = false) const { return withQED ? mEventPartsWithQED : mEventParts; }
 
+  // returns a collection of (first) collision indices that have this "source" included
+  std::unordered_map getCollisionIndicesForSource(int source) const;
+
   bool isQEDProvided() const { return !mEventRecordsWithQED.empty(); }
 
   void setBunchFilling(o2::BunchFilling const& bf) { mBCFilling = bf; }
@@ -72,15 +82,23 @@ class DigitizationContext
   void setMuPerBC(float m) { mMuBC = m; }
   float getMuPerBC() const { return mMuBC; }
 
+  /// returns the main (hadronic interaction rate) associated to this digitization context
+  float getCalculatedInteractionRate() const { return getMuPerBC() * getBunchFilling().getNBunches() * o2::constants::lhc::LHCRevFreq; }
+
   void printCollisionSummary(bool withQED = false, int truncateOutputTo = -1) const;
 
   // we need a method to fill the file names
   void setSimPrefixes(std::vector const& p);
   std::vector const& getSimPrefixes() const { return mSimPrefixes; }
+  // returns the source for a given simprefix ... otherwise -1 if not found
+  int findSimPrefix(std::string const& prefix) const;
+
+  /// add QED contributions to context, giving prefix; maximal event number and qed interaction rate
+  void fillQED(std::string_view QEDprefix, int max_events, double qedrate);
 
   /// add QED contributions to context; QEDprefix is prefix of QED production
   /// irecord is vector of QED interaction times (sampled externally)
-  void fillQED(std::string_view QEDprefix, std::vector const& irecord);
+  void fillQED(std::string_view QEDprefix, std::vector const& irecord, int max_events = -1, bool fromKinematics = true);
 
   /// Common functions the setup input TChains for reading, given the state (prefixes) encapsulated
   /// by this context. The input vector needs to be empty otherwise nothing will be done.
@@ -95,6 +113,11 @@ class DigitizationContext
   /// Check collision parts for vertex consistency.
   bool checkVertexCompatibility(bool verbose = false) const;
 
+  /// retrieves collision context for a single timeframe-id (which may be needed by simulation)
+  /// (Only copies collision context without QED information. This can be added to the result with the fillQED method
+  ///  in a second step. Takes as input a timeframe indices collection)
+  DigitizationContext extractSingleTimeframe(int timeframeid, std::vector> const& timeframeindices, std::vector const& sources_to_offset);
+
   /// function reading the hits from a chain (previously initialized with initSimChains
   /// The hits pointer will be initialized (what to we do about ownership??)
   template 
@@ -107,10 +130,46 @@ class DigitizationContext
   /// returns the GRP object associated to this context
   o2::parameters::GRPObject const& getGRP() const;
 
+  // apply collision number cuts and potential relabeling of eventID, (keeps collisions which fall into the orbitsEarly range for the next timeframe)
+  // needs a timeframe index structure (determined by calcTimeframeIndices), which is adjusted during the process to reflect the filtering
+  void applyMaxCollisionFilter(std::vector>& timeframeindices, long startOrbit, long orbitsPerTF, int maxColl, double orbitsEarly = 0.);
+
+  /// get timeframe structure --> index markers where timeframe starts/ends/is_influenced_by
+  std::vector> calcTimeframeIndices(long startOrbit, long orbitsPerTF, double orbitsEarly = 0.) const;
+
+  // Sample and fix interaction vertices (according to some distribution). Makes sure that same event ids
+  // have to have same vertex, as well as event ids associated to same collision.
+  void sampleInteractionVertices(o2::dataformats::MeanVertexObject const& v);
+
+  // Function allowing to inject interaction vertixes from the outside.
+  // Useful when this is given from data for instance. The vertex vector needs to be of same
+  // size as the interaction record.
+  // Returns 0 if success. 1 if there is a problem.
+  int setInteractionVertices(std::vector> const& vertices);
+
   // helper functions to save and load a context
   void saveToFile(std::string_view filename) const;
 
-  static DigitizationContext const* loadFromFile(std::string_view filename = "");
+  // Return the vector of interaction vertices associated with collisions
+  // The vector is empty if no vertices were provided or sampled. In this case, one
+  // may call "sampleInteractionVertices".
+  std::vector> const& getInteractionVertices() const { return mInteractionVertices; }
+
+  static DigitizationContext* loadFromFile(std::string_view filename = "");
+
+  void setCTPDigits(std::vector const* ctpdigits) const
+  {
+    mCTPTrigger = ctpdigits;
+    if (mCTPTrigger) {
+      mHasTrigger = true;
+    }
+  }
+
+  void setDigitizerInteractionRate(float intRate) { mDigitizerInteractionRate = intRate; }
+  float getDigitizerInteractionRate() const { return mDigitizerInteractionRate; }
+
+  std::vector const* getCTPDigits() const { return mCTPTrigger; }
+  bool hasTriggerInput() const { return mHasTrigger; }
 
  private:
   int mNofEntries = 0;
@@ -123,7 +182,10 @@ class DigitizationContext
   // for each collision we record the constituents (which shall not exceed mMaxPartNumber)
   std::vector> mEventParts;
 
-  // the collision records _with_ QED interleaved;
+  // for each collisionstd::vector> &timeframeindice we may record/fix the interaction vertex (to be used in event generation)
+  std::vector> mInteractionVertices;
+
+  // the collision records **with** QED interleaved;
   std::vector mEventRecordsWithQED;
   std::vector> mEventPartsWithQED;
 
@@ -133,7 +195,16 @@ class DigitizationContext
   std::string mQEDSimPrefix;                         // prefix for QED production/contribution
   mutable o2::parameters::GRPObject* mGRP = nullptr; //!
 
-  ClassDefNV(DigitizationContext, 4);
+  mutable std::vector const* mCTPTrigger = nullptr; // CTP trigger info associated to this digitization context
+  mutable bool mHasTrigger = false;                                    //
+
+  // The global ALICE interaction hadronic interaction rate as applied in digitization.
+  // It should be consistent with mMuPerBC but it might be easier to handle.
+  // The value will be filled/inserted by the digitization workflow so that digiters can access it.
+  // There is no guarantee that the value is available elsewhere.
+  float mDigitizerInteractionRate{-1};
+
+  ClassDefNV(DigitizationContext, 6);
 };
 
 /// function reading the hits from a chain (previously initialized with initSimChains
@@ -144,12 +215,19 @@ inline void DigitizationContext::retrieveHits(std::vector const& chains
                                               int entryID,
                                               std::vector* hits) const
 {
+  if (chains.size() <= sourceID) {
+    return;
+  }
   auto br = chains[sourceID]->GetBranch(brname);
   if (!br) {
     LOG(error) << "No branch found with name " << brname;
     return;
   }
   br->SetAddress(&hits);
+  auto maxEntries = br->GetEntries();
+  if (maxEntries) {
+    entryID %= maxEntries;
+  }
   br->GetEntry(entryID);
 }
 
diff --git a/DataFormats/simulation/include/SimulationDataFormat/InteractionSampler.h b/DataFormats/simulation/include/SimulationDataFormat/InteractionSampler.h
new file mode 100644
index 0000000000000..47dd4f5e4652d
--- /dev/null
+++ b/DataFormats/simulation/include/SimulationDataFormat/InteractionSampler.h
@@ -0,0 +1,161 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// @brief Simulated interaction record sampler
+
+#ifndef ALICEO2_INTERACTIONSAMPLER_H
+#define ALICEO2_INTERACTIONSAMPLER_H
+
+#include 
+#include 
+#include 
+#include 
+#include "CommonDataFormat/InteractionRecord.h"
+#include "CommonDataFormat/BunchFilling.h"
+#include "CommonConstants/LHCConstants.h"
+#include "MathUtils/RandomRing.h"
+#include 
+
+namespace o2
+{
+namespace steer
+{
+class InteractionSampler
+{
+ public:
+  static constexpr float Sec2NanoSec = 1.e9; // s->ns conversion
+  const o2::InteractionTimeRecord& generateCollisionTime();
+  void generateCollisionTimes(std::vector& dest);
+
+  void init();
+
+  void setInteractionRate(float rateHz)
+  {
+    mIntRate = rateHz;
+    mMuBC = -1.; // invalidate
+  }
+  float getInteractionRate() const { return mIntRate; }
+  void setFirstIR(const o2::InteractionRecord& ir)
+  {
+    mFirstIR.InteractionRecord::operator=(ir);
+    if (mFirstIR.orbit == 0 && mFirstIR.bc < 4) {
+      mFirstIR.bc = 4;
+    }
+  }
+  const o2::InteractionRecord& getFirstIR() const { return mFirstIR; }
+
+  void setMuPerBC(float mu)
+  {
+    mMuBC = mu;
+    mIntRate = -1.; // invalidate
+  }
+  float getMuPerBC() const { return mMuBC; }
+  void setBCTimeRMS(float tNS = 0.2) { mBCTimeRMS = tNS; }
+  float getBCTimeRMS() const { return mBCTimeRMS; }
+  const BunchFilling& getBunchFilling() const { return mBCFilling; }
+  BunchFilling& getBunchFilling() { return mBCFilling; }
+  void setBunchFilling(const BunchFilling& bc) { mBCFilling = bc; }
+  void setBunchFilling(const std::string& bcFillingFile);
+
+  void print() const;
+
+ protected:
+  virtual int simulateInteractingBC();
+  void nextCollidingBC(int n);
+
+  o2::math_utils::RandomRing<10000> mBCJumpGenerator;  // generator of random jumps in BC
+  o2::math_utils::RandomRing<1000> mNCollBCGenerator;  // generator of number of interactions in BC
+  o2::math_utils::RandomRing<1000> mCollTimeGenerator; // generator of number of interactions in BC
+
+  o2::InteractionTimeRecord mIR{{0, 0}, 0.};
+  o2::InteractionTimeRecord mFirstIR{{4, 0}, 0.};
+  int mIntBCCache = 0; ///< N interactions left for current BC
+
+  float mIntRate = -1.;   ///< total interaction rate in Hz
+  float mBCTimeRMS = 0.2; ///< BC time spread in NANOSECONDS
+  double mMuBC = -1.;     ///< interaction probability per BC
+
+  o2::BunchFilling mBCFilling;           ///< patter of active BCs
+  std::vector mTimeInBC;          ///< interaction times within single BC
+  std::vector mInteractingBCs; // vector of interacting BCs
+  int mCurrBCIdx = 0;                    ///< counter for current interacting bunch
+
+  static constexpr float DefIntRate = 50e3; ///< default interaction rate
+
+  ClassDef(InteractionSampler, 1);
+};
+
+//_________________________________________________
+inline void InteractionSampler::generateCollisionTimes(std::vector& dest)
+{
+  // fill vector with interaction records
+  dest.clear();
+  for (int i = dest.capacity(); i--;) {
+    dest.push_back(generateCollisionTime());
+  }
+}
+
+//_________________________________________________
+inline void InteractionSampler::nextCollidingBC(int n)
+{
+  /// get colliding BC as n-th after current one
+  if ((mCurrBCIdx += n) >= (int)mInteractingBCs.size()) {
+    mIR.orbit += mCurrBCIdx / mInteractingBCs.size();
+    mCurrBCIdx %= mInteractingBCs.size();
+  }
+  mIR.bc = mInteractingBCs[mCurrBCIdx];
+}
+
+// Special case of InteractionSampler without actual sampling.
+// Engineers interaction sequence by putting one in each N-th BC with multiplicity mult.
+class FixedSkipBC_InteractionSampler : public InteractionSampler
+{
+
+ public:
+  FixedSkipBC_InteractionSampler(int every_n, int mult) : mEveryN{every_n}, mMultiplicity{mult}, InteractionSampler() {}
+
+ protected:
+  int simulateInteractingBC() override;
+
+ private:
+  int mEveryN;       // the skip number ---> fills every N-th BC in the bunch filling scheme
+  int mMultiplicity; // how many events to put if bc is filled
+  ClassDef(FixedSkipBC_InteractionSampler, 1);
+};
+
+// A version of the interaction sampler which can sample according to non-uniform mu(bc) as
+// observed during data taking.
+class NonUniformMuInteractionSampler : public InteractionSampler
+{
+ public:
+  NonUniformMuInteractionSampler() : InteractionSampler() { mBCIntensityScales.resize(o2::constants::lhc::LHCMaxBunches, 1); }
+  bool setBCIntensityScales(const std::vector& scales_from_vector);
+  bool setBCIntensityScales(const TH1F& scales_from_histo); // initialize scales
+
+  // helper function to determine the scales from a histogram (count from event selection analysis)
+  std::vector determineBCIntensityScalesFromHistogram(const TH1F& scales_from_histo);
+
+  const std::vector& getBCIntensityScales() const { return mBCIntensityScales; }
+
+ protected:
+  int simulateInteractingBC() override;
+  int getBCJump() const;
+
+ private:
+  // non-uniformity
+  std::vector mBCIntensityScales;
+  ClassDef(NonUniformMuInteractionSampler, 1);
+};
+
+} // namespace steer
+} // namespace o2
+
+#endif
diff --git a/DataFormats/simulation/include/SimulationDataFormat/MCCompLabel.h b/DataFormats/simulation/include/SimulationDataFormat/MCCompLabel.h
index ec81285c9e802..74c47c87f22d5 100644
--- a/DataFormats/simulation/include/SimulationDataFormat/MCCompLabel.h
+++ b/DataFormats/simulation/include/SimulationDataFormat/MCCompLabel.h
@@ -12,7 +12,9 @@
 #ifndef ALICEO2_MCCOMPLABEL_H
 #define ALICEO2_MCCOMPLABEL_H
 
+#include 
 #include "GPUCommonRtypes.h"
+#include 
 
 namespace o2
 {
@@ -65,8 +67,10 @@ class MCCompLabel
   bool isSet() const { return mLabel != NotSet; }
   // check if label was not assigned
   bool isEmpty() const { return mLabel == NotSet; }
-  // check if label corresponds to real particle
-  bool isNoise() const { return mLabel == Noise; }
+  // check if label comes from QED contrib
+  bool isQED() const { return getSourceID() == 99; }
+  // check if label corresponds to real particle (for the moment QED is not included)
+  bool isNoise() const { return mLabel == Noise || isQED(); }
   // check if label was assigned as for correctly identified particle
   bool isValid() const { return isSet() && !isNoise(); }
 
@@ -125,7 +129,7 @@ class MCCompLabel
   int getEventID() const { return (mLabel >> nbitsTrackID) & maskEvID; }
   int getSourceID() const { return (mLabel >> (nbitsTrackID + nbitsEvID)) & maskSrcID; }
   uint64_t getTrackEventSourceID() const { return static_cast(mLabel & maskFull); }
-  void get(int& trackID, int& evID, int& srcID, bool& fake)
+  void get(int& trackID, int& evID, int& srcID, bool& fake) const
   {
     /// parse label
     trackID = getTrackID();
@@ -135,6 +139,7 @@ class MCCompLabel
   }
 
   void print() const;
+  std::string asString() const;
 
   static constexpr int maxSourceID() { return maskSrcID; }
   static constexpr int maxEventID() { return maskEvID; }
diff --git a/DataFormats/simulation/include/SimulationDataFormat/MCEventHeader.h b/DataFormats/simulation/include/SimulationDataFormat/MCEventHeader.h
index d9c5b80141df0..efaf46806a89a 100644
--- a/DataFormats/simulation/include/SimulationDataFormat/MCEventHeader.h
+++ b/DataFormats/simulation/include/SimulationDataFormat/MCEventHeader.h
@@ -27,6 +27,73 @@ namespace dataformats
 
 class GeneratorHeader;
 
+/** Common keys for information in MC event header */
+struct MCInfoKeys {
+  /** @{
+@name HepMC3 heavy-ion fields */
+  static constexpr const char* impactParameter = "Bimpact";
+  static constexpr const char* nPart = "Npart";
+  static constexpr const char* nPartProjectile = "Npart_proj";
+  static constexpr const char* nPartTarget = "Npart_targ";
+  static constexpr const char* nColl = "Ncoll";
+  static constexpr const char* nCollHard = "Ncoll_hard";
+  static constexpr const char* nCollNNWounded = "NColl_NNw";
+  static constexpr const char* nCollNWoundedN = "NColl_NwN";
+  static constexpr const char* nCollNWoundedNwounded = "NColl_NwNW";
+  static constexpr const char* planeAngle = "eventPsi";
+  static constexpr const char* sigmaInelNN = "sigmaInelNN";
+  static constexpr const char* centrality = "centrality";
+  static constexpr const char* nSpecProjectileProton = "Nspec_proj_p";
+  static constexpr const char* nSpecProjectileNeutron = "Nspec_proj_n";
+  static constexpr const char* nSpecTargetProton = "Nspec_targ_p";
+  static constexpr const char* nSpecTargetNeutron = "Nspec_targ_n";
+  /** @} */
+  /** @{
+@name HepMC3 PDF information
+
+In principle a header can have many of these.  In that case,
+each set should be prefixed with "_" where "" is a
+serial number.
+  */
+  static constexpr const char* pdfParton1Id = "pdf_parton_1_id";
+  static constexpr const char* pdfParton2Id = "pdf_parton_2_id";
+  static constexpr const char* pdfX1 = "pdf_x1";
+  static constexpr const char* pdfX2 = "pdf_x2";
+  static constexpr const char* pdfScale = "pdf_scale";
+  static constexpr const char* pdfXF1 = "pdf_par_x1";
+  static constexpr const char* pdfXF2 = "pdf_par_x2";
+  static constexpr const char* pdfCode1 = "pdf_lhc_1_id";
+  static constexpr const char* pdfCode2 = "pdf_lhc_2_id";
+  /** @} */
+  /** @{
+@name HepMC3 cross-section information
+
+In principle we can have one cross section per weight. In that
+case, each should be post-fixed by "_" where "" is a
+serial number.  These should then matcht possible names of
+weights.
+  */
+  static constexpr const char* acceptedEvents = "accepted_events";
+  static constexpr const char* attemptedEvents = "attempted_events";
+  static constexpr const char* xSection = "cross_section";
+  static constexpr const char* xSectionError = "cross_section_error";
+  /** @} */
+  /** @{
+@name Common fields */
+  static constexpr const char* generator = "generator";
+  static constexpr const char* generatorVersion = "version";
+  static constexpr const char* processName = "processName";
+  static constexpr const char* processCode = "processCode";
+  static constexpr const char* weight = "weight";
+  /** @} */
+  /** @{
+@name Pythia8-specific fields */
+  static constexpr const char* processID = "signal_process_id";
+  static constexpr const char* eventScale = "event_scale";
+  static constexpr const char* mpi = "mpi";
+  /** @} */
+};
+
 /*****************************************************************/
 /*****************************************************************/
 
@@ -75,17 +142,28 @@ class MCEventHeader : public FairMCEventHeader
     return ref;
   };
 
+  void print() const;
+
   /// prints a summary of info keys/types attached to this header
   void printInfo() const
   {
     mEventInfo.print();
   }
 
+  /// inits info fields from another Event header
+  void copyInfoFrom(MCEventHeader const& other)
+  {
+    mEventInfo.copyFrom(other.mEventInfo);
+  }
+
   /** methods **/
   virtual void Reset();
 
   MCEventStats& getMCEventStats() { return mEventStats; }
 
+  void setDetId2HitBitLUT(std::vector const& v) { mDetId2HitBitIndex = v; }
+  std::vector const& getDetId2HitBitLUT() const { return mDetId2HitBitIndex; }
+
   /// create a standalone ROOT file/tree with only the MCHeader branch
   static void extractFileFromKinematics(std::string_view kinefilename, std::string_view targetfilename);
 
@@ -93,12 +171,13 @@ class MCEventHeader : public FairMCEventHeader
   std::string mEmbeddingFileName;
   Int_t mEmbeddingEventIndex = 0;
 
-  // store a view global properties that this event
+  // store a few global properties that this event
   // had in the current simulation (which can be used quick filtering/searching)
   MCEventStats mEventStats{};
+  std::vector mDetId2HitBitIndex;
   o2::utils::RootSerializableKeyValueStore mEventInfo;
 
-  ClassDefOverride(MCEventHeader, 3);
+  ClassDefOverride(MCEventHeader, 4);
 
 }; /** class MCEventHeader **/
 
diff --git a/DataFormats/simulation/include/SimulationDataFormat/MCEventLabel.h b/DataFormats/simulation/include/SimulationDataFormat/MCEventLabel.h
index 449d67a60be15..bd5e6d840298c 100644
--- a/DataFormats/simulation/include/SimulationDataFormat/MCEventLabel.h
+++ b/DataFormats/simulation/include/SimulationDataFormat/MCEventLabel.h
@@ -15,6 +15,7 @@
 #include "GPUCommonRtypes.h"
 #include 
 #include 
+#include 
 
 namespace o2
 {
@@ -108,6 +109,7 @@ class MCEventLabel
   }
 
   void print() const;
+  std::string asString() const;
 
   static constexpr uint32_t MaxSourceID() { return MaskSrcID - 1; }
   static constexpr uint32_t MaxEventID() { return MaskEvID - 1; }
diff --git a/DataFormats/simulation/include/SimulationDataFormat/MCGenProperties.h b/DataFormats/simulation/include/SimulationDataFormat/MCGenProperties.h
new file mode 100644
index 0000000000000..1a4f86cc98122
--- /dev/null
+++ b/DataFormats/simulation/include/SimulationDataFormat/MCGenProperties.h
@@ -0,0 +1,132 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef ALICEO2_SIMDATA_MCGENPROPERTIES_H_
+#define ALICEO2_SIMDATA_MCGENPROPERTIES_H_
+
+namespace o2
+{
+namespace mcgenstatus
+{
+
+// Value to check MCGenStatusEncoding::isEncoded against to decide whether or not the stored value is encoded or basically only the HepMC status code
+// as it used to be
+constexpr unsigned int isEncodedValue{5};
+
+// internal structure to allow convenient manipulation of properties as bits on an int to (dis)entangle HepMC and specific generator status codes
+union MCGenStatusEncoding {
+  MCGenStatusEncoding() : fullEncoding(0) {}
+  MCGenStatusEncoding(int enc) : fullEncoding(enc) {}
+  // To be backward-compatible, only set transport to 1 if hepmc status is 1
+  MCGenStatusEncoding(int hepmcIn, int genIn) : isEncoded(isEncodedValue), hepmc(hepmcIn), gen(genIn), reserved(0) {}
+  int fullEncoding;
+  struct {
+    int hepmc : 9;              // HepMC status code
+    int gen : 10;               // specific generator status code
+    int reserved : 10;          // reserved bits for future usage
+    unsigned int isEncoded : 3; // special bits to check whether or not the fullEncoding is a combination of HepMC and gen status codes
+  };
+};
+
+inline bool isEncoded(MCGenStatusEncoding statusCode)
+{
+  return (statusCode.isEncoded == isEncodedValue);
+}
+
+inline bool isEncoded(int statusCode)
+{
+  return isEncoded(MCGenStatusEncoding(statusCode));
+}
+
+inline int getHepMCStatusCode(MCGenStatusEncoding enc)
+{
+  if (!isEncoded(enc)) {
+    // in this case simply set hepmc code to what was given
+    return enc.fullEncoding;
+  }
+  return enc.hepmc;
+}
+
+inline int getGenStatusCode(MCGenStatusEncoding enc)
+{
+  if (!isEncoded(enc)) {
+    // in this case simply set hepmc code to what was given
+    return enc.fullEncoding;
+  }
+  return enc.gen;
+}
+
+inline int getHepMCStatusCode(int encoded)
+{
+  return getHepMCStatusCode(MCGenStatusEncoding(encoded));
+}
+
+inline int getGenStatusCode(int encoded)
+{
+  return getGenStatusCode(MCGenStatusEncoding(encoded));
+}
+
+} // namespace mcgenstatus
+
+namespace mcgenid
+{
+
+// Define some common properties that can be set for Generators
+class GeneratorProperty
+{
+ public:
+  typedef const char* Property;
+  static constexpr Property GENERATORID{"generator_id"};
+  static constexpr Property GENERATORDESCRIPTION{"generator_description"};
+  static constexpr Property SUBGENERATORID{"subgenerator_id"};
+  static constexpr Property SUBGENERATORDESCRIPTIONMAP{"subgenerator_description_map"};
+};
+
+// internal structure to allow encoding of generator IDs and map different numbers to a single short
+union MCGenIdEncoding {
+  MCGenIdEncoding() : fullEncoding(0) {}
+  MCGenIdEncoding(int enc) : fullEncoding(enc) {}
+  MCGenIdEncoding(int generatorId, int sourceId, int subGeneratorId) : generatorId(generatorId), sourceId(sourceId), subGeneratorId(subGeneratorId) {}
+  short fullEncoding;
+  struct {
+    unsigned short generatorId : 7;    // an additional identifier for a generator which can be set by the user
+    unsigned short sourceId : 4;       // ID used in embedding scenarios
+    unsigned short subGeneratorId : 5; // sub generator ID in case a generator implements some additional logic
+  };
+};
+
+inline short getEncodedGenId(int generatorId, int sourceId, int subGeneratorId = -1)
+{
+
+  return MCGenIdEncoding(generatorId, sourceId, subGeneratorId + 1).fullEncoding;
+}
+
+inline int getGeneratorId(short encoded)
+{
+
+  return static_cast(MCGenIdEncoding(encoded).generatorId);
+}
+
+inline int getSourceId(short encoded)
+{
+  return static_cast(MCGenIdEncoding(encoded).sourceId);
+}
+
+inline int getSubGeneratorId(short encoded)
+{
+  return static_cast(MCGenIdEncoding(encoded).subGeneratorId) - 1;
+}
+
+} // namespace mcgenid
+
+} // namespace o2
+
+#endif
diff --git a/DataFormats/simulation/include/SimulationDataFormat/MCTrack.h b/DataFormats/simulation/include/SimulationDataFormat/MCTrack.h
index b8de00597117d..b4cf26b11f82e 100644
--- a/DataFormats/simulation/include/SimulationDataFormat/MCTrack.h
+++ b/DataFormats/simulation/include/SimulationDataFormat/MCTrack.h
@@ -17,6 +17,7 @@
 #define ALICEO2_DATA_MCTRACK_H_
 
 #include "SimulationDataFormat/ParticleStatus.h"
+#include "SimulationDataFormat/MCGenProperties.h"
 #include "DetectorsCommonDataFormats/DetID.h"
 #include "Rtypes.h"
 #include "SimulationDataFormat/O2DatabasePDG.h"
@@ -26,6 +27,7 @@
 #include "TParticle.h"
 #include "TParticlePDG.h"
 #include "TVector3.h"
+#include 
 
 namespace o2
 {
@@ -44,6 +46,8 @@ template 
 class MCTrackT
 {
  public:
+  static constexpr int NHITBITS = 22; // do not modify this
+
   ///  Default constructor
   MCTrackT();
 
@@ -68,7 +72,7 @@ class MCTrackT
   Int_t GetPdgCode() const { return mPdgCode; }
   Int_t getMotherTrackId() const { return mMotherTrackId; }
   Int_t getSecondMotherTrackId() const { return mSecondMotherTrackId; }
-  bool isPrimary() const { return getProcess() == TMCProcess::kPPrimary; }
+  bool isPrimary() const { return (getProcess() == TMCProcess::kPPrimary) || (getMotherTrackId() < 0 && getSecondMotherTrackId() < 0); }
   bool isSecondary() const { return !isPrimary(); }
   Int_t getFirstDaughterTrackId() const { return mFirstDaughterTrackId; }
   Int_t getLastDaughterTrackId() const { return mLastDaughterTrackId; }
@@ -80,9 +84,17 @@ class MCTrackT
   Double_t GetStartVertexCoordinatesZ() const { return mStartVertexCoordinatesZ; }
   Double_t GetStartVertexCoordinatesT() const { return mStartVertexCoordinatesT; }
 
+  /// production radius squared
+  Double_t R2() const { return Vx() * Vx() + Vy() * Vy(); }
+  /// production radius
+  Double_t R() const { return std::sqrt(R2()); }
+
   /// return mass from PDG Database if known (print message in case cannot look up)
   Double_t GetMass() const;
 
+  /// return particle weight
+  _T getWeight() const { return mWeight; }
+
   Double_t GetEnergy() const;
 
   // Alternative accessors with TParticle like shorter names
@@ -113,7 +125,7 @@ class MCTrackT
   {
     double mx(mStartVertexMomentumX);
     double my(mStartVertexMomentumY);
-    return (TMath::Pi() + TMath::ATan2(-mx, -my));
+    return (TMath::Pi() + TMath::ATan2(-my, -mx));
   }
 
   Double_t GetEta() const
@@ -127,6 +139,12 @@ class MCTrackT
     }
   }
 
+  Double_t GetTgl() const
+  {
+    auto pT = GetPt();
+    return pT > 1e-6 ? mStartVertexMomentumZ / pT : (GetStartVertexMomentumZ() > 0 ? 999. : -999.);
+  }
+
   Double_t GetTheta() const
   {
     double mz(mStartVertexMomentumZ);
@@ -135,11 +153,11 @@ class MCTrackT
 
   Double_t GetRapidity() const;
 
-  void GetMomentum(TVector3& momentum);
+  void GetMomentum(TVector3& momentum) const;
 
-  void Get4Momentum(TLorentzVector& momentum);
+  void Get4Momentum(TLorentzVector& momentum) const;
 
-  void GetStartVertex(TVector3& vertex);
+  void GetStartVertex(TVector3& vertex) const;
 
   /// Accessors to the hit mask
   Int_t getHitMask() const { return ((PropEncoding)mProp).hitmask; }
@@ -149,24 +167,40 @@ class MCTrackT
   void SetSecondMotherTrackId(Int_t id) { mSecondMotherTrackId = id; }
   void SetFirstDaughterTrackId(Int_t id) { mFirstDaughterTrackId = id; }
   void SetLastDaughterTrackId(Int_t id) { mLastDaughterTrackId = id; }
+
   // set bit indicating that this track
-  // left a hit in detector with id iDet
-  void setHit(Int_t iDet)
+  // left a hit in detector that corresponds to bit iDetBit. This bit is found in
+  // a lookup table.
+  void setHit(Int_t iDetBit)
   {
-    assert(0 <= iDet && iDet < o2::detectors::DetID::nDetectors);
+    assert(0 <= iDetBit && iDetBit < o2::detectors::DetID::nDetectors);
     auto prop = ((PropEncoding)mProp);
-    prop.hitmask |= 1 << iDet;
+    prop.hitmask |= 1 << iDetBit;
     mProp = prop.i;
   }
 
-  // did detector iDet see this track?
-  bool leftTrace(Int_t iDet) const { return (((PropEncoding)mProp).hitmask & (1 << iDet)) > 0; }
+  bool leftTraceGivenBitField(int bit) const
+  {
+    return (((PropEncoding)mProp).hitmask & (1 << bit)) > 0;
+  }
+
+  /// Returns of detector with DetID iDet has seen this track.
+  /// Needs lookup table detIDToBit mapping detectorIDs to bitfields (which is persistified in MCEventHeaders).
+  bool leftTrace(Int_t iDet, std::vector const& detIDtoBit) const
+  {
+    auto bit = detIDtoBit[iDet];
+    if (bit != -1) {
+      return leftTraceGivenBitField(bit);
+    }
+    return false;
+  }
+
   // determine how many detectors "saw" this track
   int getNumDet() const
   {
     int count = 0;
-    for (auto i = o2::detectors::DetID::First; i < o2::detectors::DetID::nDetectors; ++i) {
-      if (leftTrace(i)) {
+    for (auto i = 0; i < NHITBITS; ++i) {
+      if (leftTraceGivenBitField(i)) {
         count++;
       }
     }
@@ -196,7 +230,7 @@ class MCTrackT
   int getProcess() const { return ((PropEncoding)mProp).process; }
 
   /// get generator status code
-  int getStatusCode() const { return mStatusCode; }
+  o2::mcgenstatus::MCGenStatusEncoding getStatusCode() const { return ((o2::mcgenstatus::MCGenStatusEncoding)mStatusCode); }
 
   void setToBeDone(bool f)
   {
@@ -226,6 +260,9 @@ class MCTrackT
   /// Coordinates of start vertex [cm, ns]
   _T mStartVertexCoordinatesX, mStartVertexCoordinatesY, mStartVertexCoordinatesZ, mStartVertexCoordinatesT;
 
+  /// particle weight
+  _T mWeight;
+
   ///  PDG particle code
   Int_t mPdgCode;
 
@@ -248,9 +285,8 @@ class MCTrackT
     struct {
       int storage : 1;  // encoding whether to store this track to the output
       unsigned int process : 6; // encoding process that created this track (enough to store TMCProcess from ROOT)
-      int hitmask : 21; // encoding hits per detector
-      int reserved1 : 1; // bit reserved for possible future purposes
-      int reserved2 : 1; // bit reserved for possible future purposes
+      int hitmask : NHITBITS;   // encoding hits per detector
+      int reserved1 : 1;        // bit reserved for possible future purposes
       int inhibited : 1; // whether tracking of this was inhibited
       int toBeDone : 1; // whether this (still) needs tracking --> we might more complete information to cover full ParticleStatus space
     };
@@ -261,31 +297,33 @@ class MCTrackT
   // such as part of mProp (process) or mPDG
   Int_t mStatusCode = 0;
 
-  ClassDefNV(MCTrackT, 5);
+  ClassDefNV(MCTrackT, 8);
 };
 
 template 
 inline Double_t MCTrackT::GetEnergy() const
 {
   const auto mass = GetMass();
-  return std::sqrt(mass * mass + mStartVertexMomentumX * mStartVertexMomentumX +
-                   mStartVertexMomentumY * mStartVertexMomentumY + mStartVertexMomentumZ * mStartVertexMomentumZ);
+  Double_t px = mStartVertexMomentumX;
+  Double_t py = mStartVertexMomentumY;
+  Double_t pz = mStartVertexMomentumZ;
+  return std::sqrt(mass * mass + px * px + py * py + pz * pz);
 }
 
 template 
-inline void MCTrackT::GetMomentum(TVector3& momentum)
+inline void MCTrackT::GetMomentum(TVector3& momentum) const
 {
   momentum.SetXYZ(mStartVertexMomentumX, mStartVertexMomentumY, mStartVertexMomentumZ);
 }
 
 template 
-inline void MCTrackT::Get4Momentum(TLorentzVector& momentum)
+inline void MCTrackT::Get4Momentum(TLorentzVector& momentum) const
 {
   momentum.SetXYZT(mStartVertexMomentumX, mStartVertexMomentumY, mStartVertexMomentumZ, GetEnergy());
 }
 
 template 
-inline void MCTrackT::GetStartVertex(TVector3& vertex)
+inline void MCTrackT::GetStartVertex(TVector3& vertex) const
 {
   vertex.SetXYZ(mStartVertexCoordinatesX, mStartVertexCoordinatesY, mStartVertexCoordinatesZ);
 }
@@ -304,7 +342,8 @@ inline MCTrackT::MCTrackT()
     mStartVertexCoordinatesY(0.),
     mStartVertexCoordinatesZ(0.),
     mStartVertexCoordinatesT(0.),
-    mProp(0)
+    mProp(0),
+    mWeight(0)
 {
 }
 
@@ -324,7 +363,8 @@ inline MCTrackT::MCTrackT(Int_t pdgCode, Int_t motherId, Int_t secondMotherId
     mStartVertexCoordinatesY(y),
     mStartVertexCoordinatesZ(z),
     mStartVertexCoordinatesT(t),
-    mProp(mask)
+    mProp(mask),
+    mWeight(0)
 {
 }
 
@@ -342,6 +382,7 @@ inline MCTrackT::MCTrackT(const TParticle& part)
     mStartVertexCoordinatesY(part.Vy()),
     mStartVertexCoordinatesZ(part.Vz()),
     mStartVertexCoordinatesT(part.T() * 1e09),
+    mWeight(part.GetWeight()),
     mProp(0),
     mStatusCode(0)
 {
@@ -371,15 +412,13 @@ inline void MCTrackT::Print(Int_t trackId) const
 template 
 inline Double_t MCTrackT::GetMass() const
 {
-  TDatabasePDG* pdgdb = O2DatabasePDG::Instance();
-  if (pdgdb) {
-    auto particle = pdgdb->GetParticle(mPdgCode);
-    if (particle) {
-      return particle->Mass();
-    }
+  bool success{};
+  auto mass = O2DatabasePDG::Mass(mPdgCode, success);
+  if (!success) {
+    // coming here is a mistake which should not happen
+    MCTrackHelper::printMassError(mPdgCode);
   }
-  MCTrackHelper::printMassError(mPdgCode);
-  return 0; // coming here is a mistake which should not happen
+  return mass;
 }
 
 template 
diff --git a/DataFormats/simulation/include/SimulationDataFormat/MCTruthContainer.h b/DataFormats/simulation/include/SimulationDataFormat/MCTruthContainer.h
index 92156bd35db78..cab6088da4f0b 100644
--- a/DataFormats/simulation/include/SimulationDataFormat/MCTruthContainer.h
+++ b/DataFormats/simulation/include/SimulationDataFormat/MCTruthContainer.h
@@ -83,10 +83,10 @@ class MCTruthContainer
 {
  private:
   // for the moment we require the truth element to be messageable in order to simply flatten the object
-  // if it turnes out that other types are required this needs to be extended and method flatten nees to
+  // if it turnes out that other types are required this needs to be extended and method flatten needs to
   // be conditionally added
   // TODO: activate this check
-  //static_assert(o2::framework::is_messageable::value, "truth element type must be messageable");
+  // static_assert(o2::framework::is_messageable::value, "truth element type must be messageable");
 
   std::vector mHeaderArray; // the header structure array serves as an index into the actual storage
   std::vector mTruthArray;          // the buffer containing the actual truth information
@@ -140,7 +140,7 @@ class MCTruthContainer
   TruthElement const& getElement(uint32_t elementindex) const { return mTruthArray[elementindex]; }
   // return the number of original data indexed here
   size_t getIndexedSize() const { return mHeaderArray.size(); }
-  // return the number of elements managed in this container
+  // return the number of elements (labels) pointed to in this container
   size_t getNElements() const { return mTruthArray.size(); }
   // return unterlaying vector of elements
   const std::vector& getTruthArray() const
@@ -186,7 +186,8 @@ class MCTruthContainer
 
   // add element for a particular dataindex
   // at the moment only strictly consecutive modes are supported
-  void addElement(uint32_t dataindex, TruthElement const& element)
+  // noElement indicates that actually no label/element will be added for this dataindex
+  void addElement(uint32_t dataindex, TruthElement const& element, bool noElement = false)
   {
     if (dataindex < mHeaderArray.size()) {
       // look if we have something for this dataindex already
@@ -206,7 +207,15 @@ class MCTruthContainer
       // add a new one
       mHeaderArray.emplace_back(mTruthArray.size());
     }
-    mTruthArray.emplace_back(element);
+    if (!noElement) {
+      mTruthArray.emplace_back(element);
+    }
+  }
+
+  /// adds a data index that has no label
+  void addNoLabelIndex(uint32_t dataindex)
+  {
+    addElement(dataindex, TruthElement(), true);
   }
 
   // convenience interface to add multiple labels at once
@@ -218,6 +227,7 @@ class MCTruthContainer
                     std::is_assignable::value ||
                     std::is_base_of::value,
                   "Need to add compatible labels");
+    addNoLabelIndex(dataindex);
     for (auto& e : elements) {
       addElement(dataindex, e);
     }
diff --git a/DataFormats/simulation/include/SimulationDataFormat/MCUtils.h b/DataFormats/simulation/include/SimulationDataFormat/MCUtils.h
index 38a862089ecf5..c54b424eb0238 100644
--- a/DataFormats/simulation/include/SimulationDataFormat/MCUtils.h
+++ b/DataFormats/simulation/include/SimulationDataFormat/MCUtils.h
@@ -16,13 +16,17 @@
 #ifndef O2_MCUTILS_H
 #define O2_MCUTILS_H
 
+#include 
 #include 
+#include 
 #include "TPDGCode.h"
+#include "TParticle.h"
 
 namespace o2
 {
 namespace mcutils
 {
+
 /// A couple of functions to query on MC tracks ( that needs navigation within the global container
 /// of available tracks. It is a class so as to make it available for interactive ROOT more easily.
 class MCTrackNavigator
@@ -69,6 +73,16 @@ class MCTrackNavigator
   ClassDefNV(MCTrackNavigator, 1);
 };
 
+class MCGenHelper
+{
+ public:
+  // Helper function for users which takes over encoding of status code as well as (re-)setting correct transport bit.
+  // Has to be in a class as a static methid. Just in a namespace it doesn't work to use this function in ROOT macros.
+  static void encodeParticleStatusAndTracking(TParticle& particle, bool wanttracking = true);
+  static void encodeParticleStatusAndTracking(TParticle& particle, int hepmcStatus, int genStatus, bool wanttracking = true);
+  ClassDefNV(MCGenHelper, 1)
+};
+
 /// Determine if a particle (identified by pdg) is stable
 inline bool isStable(int pdg)
 {
@@ -116,4 +130,4 @@ inline bool isStable(int pdg)
 } // namespace mcutils
 } // namespace o2
 
-#endif //O2_MCUTILS_H
+#endif // O2_MCUTILS_H
diff --git a/DataFormats/simulation/include/SimulationDataFormat/O2DatabasePDG.h b/DataFormats/simulation/include/SimulationDataFormat/O2DatabasePDG.h
index 8cc4ec2836de8..23dc30119aa7a 100644
--- a/DataFormats/simulation/include/SimulationDataFormat/O2DatabasePDG.h
+++ b/DataFormats/simulation/include/SimulationDataFormat/O2DatabasePDG.h
@@ -15,7 +15,10 @@
 
 #ifndef O2_O2DATABASEPDG_H
 #define O2_O2DATABASEPDG_H
+
+#include 
 #include "TDatabasePDG.h"
+#include "TParticlePDG.h"
 
 namespace o2
 {
@@ -42,10 +45,41 @@ class O2DatabasePDG
 
   // adds ALICE particles to a given TDatabasePDG instance
   static void addALICEParticles(TDatabasePDG* db = TDatabasePDG::Instance());
+  static void addParticlesFromExternalFile(TDatabasePDG* db);
 
- private:
-  // private constructor
+  // get particle's (if any) mass
+  static Double_t MassImpl(TParticlePDG* particle, bool& success)
+  {
+    success = false;
+    if (!particle) {
+      return -1.;
+    }
+    success = true;
+    return particle->Mass();
+  }
+
+  // determine particle to get mass for based on PDG
+  static Double_t Mass(int pdg, bool& success, TDatabasePDG* db = O2DatabasePDG::Instance())
+  {
+    if (pdg < IONBASELOW || pdg > IONBASEHIGH) {
+      // not an ion, return immediately
+      return MassImpl(db->GetParticle(pdg), success);
+    }
+    if (auto particle = db->GetParticle(pdg)) {
+      // see if this ion can be found
+      return MassImpl(particle, success);
+    }
+    // if we are here, try one last time to look for ground state of potential isomere state
+    pdg = pdg / 10 * 10;
+    return MassImpl(db->GetParticle(pdg), success);
+  }
+
+  // remove default constructor
   O2DatabasePDG() = delete;
+
+ private:
+  static constexpr int IONBASELOW{1000000000};
+  static constexpr int IONBASEHIGH{1099999999};
 };
 
 // by keeping this inline, we can use it in other parts of the code, for instance Framework or Analysis,
@@ -71,18 +105,23 @@ inline void O2DatabasePDG::addALICEParticles(TDatabasePDG* db)
   const Double_t khShGev = khSlash * kErg2Gev;
   const Double_t kYear2Sec = 3600 * 24 * 365.25;
 
+  // Heavy-flavour particles
+
+  // χc1(3872) aka X(3872), taken from PDG 2022 (https://pdg.lbl.gov/2022/listings/rpp2022-list-chi-c1-3872.pdf)
+  db->AddParticle("Chi_c1(3872)", "Chi_c1(3872)", 3.87165, kFALSE, 0, 0, "CCBarMeson", 9920443);
+
   //
   // Bottom mesons
   // mass and life-time from PDG
   //
   db->AddParticle("Upsilon(3S)", "Upsilon(3S)", 10.3552, kTRUE,
-                  0, 1, "Bottonium", 200553);
+                  0, 0, "Bottonium", 200553);
 
   // QCD diffractive states
   db->AddParticle("rho_diff0", "rho_diff0", 0, kTRUE,
                   0, 0, "QCD diffr. state", 9900110);
   db->AddParticle("pi_diffr+", "pi_diffr+", 0, kTRUE,
-                  0, 1, "QCD diffr. state", 9900210);
+                  0, 3, "QCD diffr. state", 9900210);
   db->AddParticle("omega_di", "omega_di", 0, kTRUE,
                   0, 0, "QCD diffr. state", 9900220);
   db->AddParticle("phi_diff", "phi_diff", 0, kTRUE,
@@ -92,7 +131,7 @@ inline void O2DatabasePDG::addALICEParticles(TDatabasePDG* db)
   db->AddParticle("n_diffr0", "n_diffr0", 0, kTRUE,
                   0, 0, "QCD diffr. state", 9902110);
   db->AddParticle("p_diffr+", "p_diffr+", 0, kTRUE,
-                  0, 1, "QCD diffr. state", 9902210);
+                  0, 3, "QCD diffr. state", 9902210);
 
   // From Herwig
   db->AddParticle("PSID    ", " ", 3.7699, kFALSE, 0.0, 0, "meson", 30443);
@@ -192,81 +231,67 @@ inline void O2DatabasePDG::addALICEParticles(TDatabasePDG* db)
                   0, 0, "Special", kspe + 50);
   db->AddParticle("FeedbackPhoton", "FeedbackPhoton", 0, kFALSE,
                   0, 0, "Special", kspe + 51);
-  db->AddParticle("Lambda1520", "Lambda1520", 1.5195, kFALSE,
-                  0.0156, 0, "Resonance", 3124);
-  db->AddAntiParticle("Lambda1520bar", -3124);
 
   //Hyper nuclei and exotica
   ionCode = 1010010030;
   if (!db->GetParticle(ionCode)) {
-    db->AddParticle("HyperTriton", "HyperTriton", 2.99131, kFALSE,
+    db->AddParticle("HyperTriton", "HyperTriton", 2.991134, kFALSE,
                     2.5e-15, 3, "Ion", ionCode);
   }
 
   ionCode = -1010010030;
   if (!db->GetParticle(ionCode)) {
-    db->AddParticle("AntiHyperTriton", "AntiHyperTriton", 2.99131, kFALSE,
+    db->AddParticle("AntiHyperTriton", "AntiHyperTriton", 2.991134, kFALSE,
                     2.5e-15, 3, "Ion", ionCode);
   }
 
   //hyper hydrogen 4 ground state
   ionCode = 1010010040;
   if (!db->GetParticle(ionCode)) {
-    db->AddParticle("Hyperhydrog4", "Hyperhydrog4", 3.9226, kFALSE,
+    db->AddParticle("Hyperhydrog4", "Hyperhydrog4", 3.922434, kFALSE,
                     2.5e-15, 3, "Ion", ionCode);
   }
   //anti hyper hydrogen 4 ground state
   ionCode = -1010010040;
   if (!db->GetParticle(ionCode)) {
-    db->AddParticle("AntiHyperhydrog4", "AntiHyperhydrog4", 3.9226, kFALSE,
-                    2.5e-15, 3, "Ion", ionCode);
-  }
-  //hyper hydrogen 4 excited state
-  ionCode = 1010010041;
-  if (!db->GetParticle(ionCode)) {
-    db->AddParticle("Hyperhydrog4*", "Hyperhydrog4*", 3.9237, kFALSE,
-                    2.5e-15, 3, "Ion", ionCode);
-  }
-  //anti hyper hydrogen 4 excited state
-  ionCode = -1010010041;
-  if (!db->GetParticle(ionCode)) {
-    db->AddParticle("AntiHyperhydrog4*", "AntiHyperhydrog4*", 3.9237, kFALSE,
+    db->AddParticle("AntiHyperhydrog4", "AntiHyperhydrog4", 3.922434, kFALSE,
                     2.5e-15, 3, "Ion", ionCode);
   }
   //hyper helium 4 ground state
   ionCode = 1010020040;
   if (!db->GetParticle(ionCode)) {
-    db->AddParticle("Hyperhelium4", "Hyperhelium4", 3.9217, kFALSE,
+    db->AddParticle("Hyperhelium4", "Hyperhelium4", 3.921728, kFALSE,
                     2.5e-15, 6, "Ion", ionCode);
   }
   //anti hyper helium 4 ground state
   ionCode = -1010020040;
   if (!db->GetParticle(ionCode)) {
-    db->AddParticle("AntiHyperhelium4", "AntiHyperhelium4", 3.9217, kFALSE,
+    db->AddParticle("AntiHyperhelium4", "AntiHyperhelium4", 3.921728, kFALSE,
                     2.5e-15, 6, "Ion", ionCode);
   }
-  //hyper helium 4 excited state
-  ionCode = 1010020041;
+
+  // Lithium 4 ground state
+  ionCode = 1000030040;
   if (!db->GetParticle(ionCode)) {
-    db->AddParticle("Hyperhelium4*", "Hyperhelium4*", 3.9231, kFALSE,
-                    2.5e-15, 6, "Ion", ionCode);
+    db->AddParticle("Lithium4", "Lithium4", 3.7513, kFALSE,
+                    0.003, 9, "Ion", ionCode);
   }
-  //anti hyper helium 4 excited state
-  ionCode = -1010020041;
+  // anti Lithium 4 ground state
+  ionCode = -1000030040;
   if (!db->GetParticle(ionCode)) {
-    db->AddParticle("AntiHyperhelium4*", "AntiHyperhelium4*", 3.9231, kFALSE,
-                    2.5e-15, 6, "Ion", ionCode);
+    db->AddParticle("AntiLithium4", "AntiLithium4", 3.7513, kFALSE,
+                    0.003, 9, "Ion", ionCode);
   }
 
   ionCode = 1010020050;
   if (!db->GetParticle(ionCode)) {
-    db->AddParticle("Hyperhelium5", "Hyperhelium5", 4.841, kFALSE,
+    db->AddParticle("Hyperhelium5", "Hyperhelium5", 4.839961, kFALSE,
                     2.5e-15, 6, "Ion", ionCode);
   }
 
   ionCode = -1010020050;
   if (!db->GetParticle(ionCode)) {
-    db->AddParticle("AntiHyperhelium5", "AntiHyperhelium5", 4.841, kFALSE,
+    db->AddParticle("AntiHyperhelium5", "AntiHyperhelium5", 4.839961, kFALSE,
                     2.5e-15, 6, "Ion", ionCode);
   }
 
@@ -282,6 +307,33 @@ inline void O2DatabasePDG::addALICEParticles(TDatabasePDG* db)
                     2.5e-15, 6, "Ion", ionCode);
   }
 
+  // 4-Xi-He
+  ionCode = 1120020040;
+  if (!db->GetParticle(ionCode)) {
+    db->AddParticle("4XiHe", "4XiHe", 4.128, kFALSE, 4.04e-15, 3, "Ion", ionCode);
+    db->AddAntiParticle("Anti4XiHe", -ionCode);
+  }
+
+  // 4-Xi-H
+  ionCode = 1120010040;
+  if (!db->GetParticle(ionCode)) {
+    db->AddParticle("4XiH", "4XiH", 4.128, kFALSE, 4.04e-15, 3, "Ion", ionCode);
+    db->AddAntiParticle("Anti4XiH", -ionCode);
+  }
+
+  // hyper helium 4 sigma
+  ionCode = 1110020040;
+  if (!db->GetParticle(ionCode)) {
+    db->AddParticle("Hyperhelium4sigma", "Hyperhelium4sigma", 3.995, kFALSE,
+                    2.5e-15, 6, "Ion", ionCode);
+  }
+  // anti-hyper helium 4 sigma
+  ionCode = -1110020040;
+  if (!db->GetParticle(ionCode)) {
+    db->AddParticle("AntiHyperhelium4sigma", "AntiHyperhelium4sigma", 3.995, kFALSE,
+                    2.5e-15, 6, "Ion", ionCode);
+  }
+
   ionCode = 1010000020;
   if (!db->GetParticle(ionCode)) {
     db->AddParticle("LambdaNeutron", "LambdaNeutron", 2.054, kFALSE,
@@ -418,6 +470,60 @@ inline void O2DatabasePDG::addALICEParticles(TDatabasePDG* db)
                     0.185, 0, "Resonance", ionCode);
   }
 
+  // Lambda(1405)0
+  ionCode = 102132;
+  if (!db->GetParticle(ionCode)) {
+    db->AddParticle("Lambda_1405_0", "Lambda_1405_0", 1.405, kFALSE, 0.05, 0, "Resonance", ionCode);
+  }
+  if (!db->GetParticle(-ionCode)) {
+    db->AddParticle("AntiLambda_1405_0", "AntiLambda_1405_0", 1.405, kFALSE, 0.05, 0, "Resonance", -ionCode);
+  }
+
+  // Lambda(1520)0
+  ionCode = 102134;
+  if (!db->GetParticle(ionCode)) {
+    db->AddParticle("Lambda_1520_0", "Lambda_1520_0", 1.5195, kFALSE, 0.0156, 0, "Resonance", ionCode);
+  }
+  if (!db->GetParticle(-ionCode)) {
+    db->AddParticle("AntiLambda_1520_0", "AntiLambda_1520_0", 1.5195, kFALSE, 0.0156, 0, "Resonance", -ionCode);
+  }
+
+  // f1 study
+  ionCode = 20223;
+  if (!db->GetParticle(ionCode)) {
+    db->AddParticle("f1_1285", "f1_1285", 1.28210, kFALSE, 0.02420, 0, "Resonance", ionCode);
+  }
+  ionCode = 20333;
+  if (!db->GetParticle(ionCode)) {
+    db->AddParticle("f1_1420", "f1_1420", 1.42640, kFALSE, 0.05490, 0, "Resonance", ionCode);
+  }
+
+  // glueball hunting
+  ionCode = 225;
+  if (!db->GetParticle(ionCode)) {
+    db->AddParticle("f2_1270", "f2_1270", 1.2754, kFALSE, 0.1858, 0, "Resonance", ionCode);
+  }
+  ionCode = 115;
+  if (!db->GetParticle(ionCode)) {
+    db->AddParticle("a2_1320", "a2_1320", 1.3182, kFALSE, 0.1078, 0, "Resonance", ionCode);
+  }
+  ionCode = 10221;
+  if (!db->GetParticle(ionCode)) {
+    db->AddParticle("f0_1370", "f0_1370", 1.37, kFALSE, 0.200, 0, "Resonance", ionCode);
+  }
+  ionCode = 9030221;
+  if (!db->GetParticle(ionCode)) {
+    db->AddParticle("f0_1500", "f0_1500", 1.500, kFALSE, 0.112, 0, "Resonance", ionCode);
+  }
+  ionCode = 10331;
+  if (!db->GetParticle(ionCode)) {
+    db->AddParticle("f0_1710", "f0_1710", 1.710, kFALSE, 0.139, 0, "Resonance", ionCode);
+  }
+  ionCode = 335;
+  if (!db->GetParticle(ionCode)) {
+    db->AddParticle("f2_1525", "f2_1525", 1.525, kFALSE, 0.073, 0, "Resonance", ionCode);
+  }
+
   // Xi-/+ (1820)
   ionCode = 123314;
   if (!db->GetParticle(ionCode)) {
@@ -543,10 +649,47 @@ inline void O2DatabasePDG::addALICEParticles(TDatabasePDG* db)
     db->AddParticle("Anti-Theta_c_3100", "Anti-Theta_c_3100", 3.099, kFALSE,
                     83.e-6, 0, "Resonance", ionCode); // same width as D*+ (83 keV)
   }
-  if (!db->GetParticle(-ionCode)) {
-    db->AddParticle("Theta_c_3100", "Theta_c_3100", 3.099, kFALSE,
-                    83.e-6, 0, "Resonance", -ionCode); // same width as D*+ (83 keV)
+  db->AddAntiParticle("Theta_c_3100", -ionCode);
+
+  // Charm resonances not present in PYTHIA (consistent with https://github.com/AliceO2Group/O2DPG/blob/master/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays_DResoTrigger.cfg)
+  // Mesons
+  ionCode = 30433;
+  if (!db->GetParticle(ionCode)) {
+    db->AddParticle("D*_s1_Plus_2700", "D*_s1_Plus_2700", 2.714, false, 0.122, 3, "Resonance", ionCode);
+  }
+  db->AddAntiParticle("D*_s1_Minus_2700", -ionCode);
+  ionCode = 40433;
+  if (!db->GetParticle(ionCode)) {
+    db->AddParticle("D*_s1_Plus_2860", "D*_s1_Plus_2860", 2.859, false, 0.160, 3, "Resonance", ionCode);
+  }
+  db->AddAntiParticle("D*_s1_Minus_2860", -ionCode);
+  ionCode = 437;
+  if (!db->GetParticle(ionCode)) {
+    db->AddParticle("D*_s3_Plus_2860", "D*_s3_Plus_2860", 2.860, false, 0.053, 3, "Resonance", ionCode);
+  }
+  db->AddAntiParticle("D*_s3_Minus_2860", -ionCode);
+
+  // Baryons
+  ionCode = 4325;
+  if (!db->GetParticle(ionCode)) {
+    db->AddParticle("Xi_c_Plus_3055", "Xi_c_Plus_3055", 3.0559, false, 0.0078, 3, "Resonance", ionCode);
+  }
+  db->AddAntiParticle("Xi_c_Minus_3055", -ionCode);
+  ionCode = 4326;
+  if (!db->GetParticle(ionCode)) {
+    db->AddParticle("Xi_c_Plus_3080", "Xi_c_Plus_3080", 3.0772, false, 0.0036, 3, "Resonance", ionCode);
+  }
+  db->AddAntiParticle("Xi_c_Minus_3080", -ionCode);
+  ionCode = 4315;
+  if (!db->GetParticle(ionCode)) {
+    db->AddParticle("Xi_c_0_3055", "Xi_c_0_3055", 3.0590, false, 0.0064, 0, "Resonance", ionCode);
   }
+  db->AddAntiParticle("Anti-Xi_c_0_3055", -ionCode);
+  ionCode = 4316;
+  if (!db->GetParticle(ionCode)) {
+    db->AddParticle("Xi_c_0_3080", "Xi_c_0_3080", 3.0799, false, 0.0056, 0, "Resonance", ionCode);
+  }
+  db->AddAntiParticle("Anti-Xi_c_0_3080", -ionCode);
 
   // d*(2380) - dibaryon resonance
 
@@ -566,6 +709,26 @@ inline void O2DatabasePDG::addALICEParticles(TDatabasePDG* db)
   if (!db->GetParticle(-ionCode)) {
     db->AddParticle("AntiSexaquark", "AntiSexaquark", 2.0, kTRUE, 0.0, 0, "Special", -ionCode);
   }
+
+  // lastly, add particle from the the extra text file
+  addParticlesFromExternalFile(db);
+}
+
+inline void O2DatabasePDG::addParticlesFromExternalFile(TDatabasePDG* db)
+{
+  static bool initialized = false;
+  if (!initialized) {
+    // allow user to specify custom file
+    if (const char* custom = std::getenv("O2_SIM_CUSTOM_PDG")) {
+      // TODO: make sure this is a file
+      db->ReadPDGTable(custom);
+    } else if (const char* o2Root = std::getenv("O2_ROOT")) {
+      // take the maintained file from O2
+      auto inputExtraPDGs = std::string(o2Root) + "/share/Detectors/gconfig/data/extra_ions_pdg_table.dat";
+      db->ReadPDGTable(inputExtraPDGs.c_str());
+    }
+    initialized = true;
+  }
 }
 
 } // namespace o2
diff --git a/DataFormats/simulation/include/SimulationDataFormat/ParticleStatus.h b/DataFormats/simulation/include/SimulationDataFormat/ParticleStatus.h
index 552436aebfdbf..f61705529f04d 100644
--- a/DataFormats/simulation/include/SimulationDataFormat/ParticleStatus.h
+++ b/DataFormats/simulation/include/SimulationDataFormat/ParticleStatus.h
@@ -20,6 +20,8 @@ enum ParticleStatus { kKeep = BIT(14),
                       kToBeDone = BIT(16),
                       kPrimary = BIT(17),
                       kTransport = BIT(18),
-                      kInhibited = BIT(19) };
+                      kInhibited = BIT(19),
+                      kHasHits = BIT(20),
+                      kHasTrackRefs = BIT(21) };
 
 #endif
diff --git a/DataFormats/simulation/include/SimulationDataFormat/PrimaryChunk.h b/DataFormats/simulation/include/SimulationDataFormat/PrimaryChunk.h
index 86f2023c3680b..f50162a0f6f7f 100644
--- a/DataFormats/simulation/include/SimulationDataFormat/PrimaryChunk.h
+++ b/DataFormats/simulation/include/SimulationDataFormat/PrimaryChunk.h
@@ -29,7 +29,7 @@ struct SubEventInfo {
   int32_t runID = 0;      // the runID of this run
   uint16_t part = 0;      // which part of the eventID
   uint16_t nparts = 0;    // out of how many parts
-  uint32_t seed = 0;      // seed for RNG
+  uint64_t seed = 0;      // seed for RNG
   uint32_t index = 0;
   int32_t npersistenttracks = -1; // the number of persistent tracks for this SubEvent (might be set to cache it)
   int32_t nprimarytracks = -1;    // the number of primary tracks for this SubEvent
@@ -37,7 +37,7 @@ struct SubEventInfo {
 
   o2::dataformats::MCEventHeader mMCEventHeader; // associated FairMC header for vertex information
 
-  ClassDefNV(SubEventInfo, 1);
+  ClassDefNV(SubEventInfo, 2);
 };
 
 inline bool operator<(SubEventInfo const& a, SubEventInfo const& b)
diff --git a/DataFormats/simulation/include/SimulationDataFormat/ProcessingEventInfo.h b/DataFormats/simulation/include/SimulationDataFormat/ProcessingEventInfo.h
deleted file mode 100644
index 150a8272c7714..0000000000000
--- a/DataFormats/simulation/include/SimulationDataFormat/ProcessingEventInfo.h
+++ /dev/null
@@ -1,36 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-/// \file ProcessingEventInfo.h
-/// \brief Encapsulated meta information about current event being processed by FairRoot (analysis) tasks
-/// \author Sandro Wenzel
-
-#ifndef ALICEO2_DATA_EVENTINFO_H_
-#define ALICEO2_DATA_EVENTINFO_H_
-
-namespace o2
-{
-
-// A class encapsulating meta information about events being process
-// and the data being sent by run classes such as FairRunAna.
-// Can be send to processing tasks for usage so that they do no longer
-// need to access the FairRootManager directly.
-struct ProcessingEventInfo {
-  double eventTime;   //! time of the current event
-  int eventNumber;    //! the current entry
-  int sourceNumber;   //! the current source number
-  bool numberSources; //! number of sources
-  // can be extended further
-};
-
-} // namespace o2
-
-#endif
diff --git a/DataFormats/simulation/include/SimulationDataFormat/TrackReference.h b/DataFormats/simulation/include/SimulationDataFormat/TrackReference.h
index f3d41a17208f0..34d1c57aa9f0b 100644
--- a/DataFormats/simulation/include/SimulationDataFormat/TrackReference.h
+++ b/DataFormats/simulation/include/SimulationDataFormat/TrackReference.h
@@ -171,7 +171,7 @@ class TrackReference
   float mTrackLength = 0; ///< track length from its origin in cm
   float mTof = 0;         ///< time of flight in cm
   Int_t mUserId = 0;      ///< optional Id defined by user
-  Int_t mDetectorId = 0;  ///< Detector Id
+  Int_t mDetectorId = -1; ///< sensitive Detector Id (-1 if unknown or in passive material)
   SimTrackStatus mStatus; ///< encoding the track status
 
   friend std::ostream& operator<<(std::ostream&, const TrackReference&);
diff --git a/DataFormats/simulation/src/ConstMCTruthContainer.cxx b/DataFormats/simulation/src/ConstMCTruthContainer.cxx
new file mode 100644
index 0000000000000..257227a8eda31
--- /dev/null
+++ b/DataFormats/simulation/src/ConstMCTruthContainer.cxx
@@ -0,0 +1,59 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+using namespace o2::dataformats;
+
+ConstMCTruthContainer* MCLabelIOHelper::loadFromTTree(TTree* tree, std::string const& brname, int entry)
+{
+  o2::dataformats::MCTruthContainer* labels = nullptr;
+  o2::dataformats::IOMCTruthContainerView* labelROOTbuffer = nullptr;
+  o2::dataformats::ConstMCTruthContainer* constlabels = new ConstMCTruthContainer();
+
+  auto branch = tree->GetBranch(brname.c_str());
+  if (!branch) {
+    return nullptr;
+  }
+  auto labelClass = branch->GetClassName();
+  bool oldlabelformat = false;
+
+  // we check how the labels are persisted and react accordingly
+  if (TString(labelClass).Contains("IOMCTruthContainer")) {
+    branch->SetAddress(&labelROOTbuffer);
+    branch->GetEntry(entry);
+    if (labelROOTbuffer) {
+      labelROOTbuffer->copyandflatten(*constlabels);
+      delete labelROOTbuffer;
+    }
+  } else {
+    // case when we directly streamed MCTruthContainer
+    // TODO: support more cases such as ConstMCTruthContainer etc
+    std::string serializedtype("o2::dataformats::MCTruthContainer");
+    if (!TString(labelClass).EqualTo(serializedtype.c_str())) {
+      std::cerr << "Error: expected serialized type " << serializedtype << " but found " << labelClass;
+      return nullptr;
+    }
+    branch->SetAddress(&labels);
+    branch->GetEntry(entry);
+    if (labels) {
+      labels->flatten_to(*constlabels);
+      delete labels;
+    }
+  }
+  return constlabels;
+}
diff --git a/DataFormats/simulation/src/DigitizationContext.cxx b/DataFormats/simulation/src/DigitizationContext.cxx
index ad390686e4c0f..79e36aa9fa48b 100644
--- a/DataFormats/simulation/src/DigitizationContext.cxx
+++ b/DataFormats/simulation/src/DigitizationContext.cxx
@@ -11,12 +11,15 @@
 
 #include "SimulationDataFormat/DigitizationContext.h"
 #include "SimulationDataFormat/MCEventHeader.h"
+#include "SimulationDataFormat/InteractionSampler.h"
 #include "DetectorsCommonDataFormats/DetectorNameConf.h"
 #include 
 #include 
 #include 
 #include  // for iota
 #include 
+#include 
+#include 
 
 using namespace o2::steer;
 
@@ -28,6 +31,7 @@ void DigitizationContext::printCollisionSummary(bool withQED, int truncateOutput
   for (int p = 0; p < mSimPrefixes.size(); ++p) {
     std::cout << "Part " << p << " : " << mSimPrefixes[p] << "\n";
   }
+  std::cout << "QED information included " << isQEDProvided() << "\n";
   if (withQED) {
     std::cout << "Number of Collisions " << mEventRecords.size() << "\n";
     std::cout << "Number of QED events " << mEventRecordsWithQED.size() - mEventRecords.size() << "\n";
@@ -45,6 +49,27 @@ void DigitizationContext::printCollisionSummary(bool withQED, int truncateOutput
     }
   } else {
     std::cout << "Number of Collisions " << mEventRecords.size() << "\n";
+    if (mEventPartsWithQED.size() > 0) {
+      auto num_qed_events = mEventPartsWithQED.size() - mEventRecords.size();
+      if (num_qed_events > 0) {
+        std::cout << "Number of QED events (but not shown) " << num_qed_events << "\n";
+        // find first and last QED collision so that we can give the range in orbits where these
+        // things are included
+        auto firstQEDcoll_iter = std::find_if(mEventPartsWithQED.begin(), mEventPartsWithQED.end(),
+                                              [](const std::vector& vec) {
+                                                return std::find_if(vec.begin(), vec.end(), [](EventPart const& p) { return p.sourceID == 99; }) != vec.end();
+                                              });
+
+        auto lastColl_iter = std::find_if(mEventPartsWithQED.rbegin(), mEventPartsWithQED.rend(),
+                                          [](const std::vector& vec) {
+                                            return std::find_if(vec.begin(), vec.end(), [](EventPart const& p) { return p.sourceID == 99; }) != vec.end();
+                                          });
+
+        auto firstindex = std::distance(mEventPartsWithQED.begin(), firstQEDcoll_iter);
+        auto lastindex = std::distance(mEventPartsWithQED.begin(), lastColl_iter.base()) - 1;
+        std::cout << "QED from: " << mEventRecordsWithQED[firstindex] << " ---> " << mEventRecordsWithQED[lastindex] << "\n";
+      }
+    }
     for (int i = 0; i < mEventRecords.size(); ++i) {
       if (truncateOutputTo >= 0 && i > truncateOutputTo) {
         std::cout << "--- Output truncated to " << truncateOutputTo << " ---\n";
@@ -54,6 +79,9 @@ void DigitizationContext::printCollisionSummary(bool withQED, int truncateOutput
       for (auto& e : mEventParts[i]) {
         std::cout << " (" << e.sourceID << " , " << e.entryID << ")";
       }
+      if (i < mInteractionVertices.size()) {
+        std::cout << " sampled vertex : " << mInteractionVertices[i];
+      }
       std::cout << "\n";
     }
   }
@@ -71,6 +99,14 @@ bool DigitizationContext::initSimChains(o2::detectors::DetID detid, std::vector<
     return false;
   }
 
+  // check that all files are present, otherwise quit
+  for (int source = 0; source < mSimPrefixes.size(); ++source) {
+    if (!std::filesystem::exists(o2::base::DetectorNameConf::getHitsFileName(detid, mSimPrefixes[source].data()))) {
+      LOG(info) << "Not hit file present for " << detid.getName() << " (exiting SimChain setup)";
+      return false;
+    }
+  }
+
   simchains.emplace_back(new TChain("o2sim"));
   // add the main (background) file
   simchains.back()->AddFile(o2::base::DetectorNameConf::getHitsFileName(detid, mSimPrefixes[0].data()).c_str());
@@ -115,6 +151,19 @@ bool DigitizationContext::initSimKinematicsChains(std::vector& simkinem
     // add signal files
     simkinematicschains.back()->AddFile(o2::base::NameConf::getMCKinematicsFileName(mSimPrefixes[source].data()).c_str());
   }
+
+  // we add QED, if used in the digitization context
+  if (mEventRecordsWithQED.size() > 0) {
+    if (mSimPrefixes.size() >= QEDSOURCEID) {
+      LOG(fatal) << "Too many signal chains; crashes with QED source ID";
+    }
+
+    // it might be better to use an unordered_map for the simchains but this requires interface changes
+    simkinematicschains.resize(QEDSOURCEID + 1, nullptr);
+    simkinematicschains[QEDSOURCEID] = new TChain("o2sim");
+    simkinematicschains[QEDSOURCEID]->AddFile(o2::base::DetectorNameConf::getMCKinematicsFileName(mQEDSimPrefix).c_str());
+  }
+
   return true;
 }
 
@@ -190,13 +239,55 @@ o2::parameters::GRPObject const& DigitizationContext::getGRP() const
 
 void DigitizationContext::saveToFile(std::string_view filename) const
 {
+  // checks if the path content of filename exists ... otherwise it is created before creating the ROOT file
+  auto ensure_path_exists = [](std::string_view filename) {
+    try {
+      // Extract the directory path from the filename
+      std::filesystem::path file_path(filename);
+      std::filesystem::path dir_path = file_path.parent_path();
+
+      // Check if the directory path is empty (which means filename was just a name without path)
+      if (dir_path.empty()) {
+        // nothing to do
+        return true;
+      }
+
+      // Create directories if they do not exist
+      if (!std::filesystem::exists(dir_path)) {
+        if (std::filesystem::create_directories(dir_path)) {
+          // std::cout << "Directories created successfully: " << dir_path.string() << std::endl;
+          return true;
+        } else {
+          std::cerr << "Failed to create directories: " << dir_path.string() << std::endl;
+          return false;
+        }
+      }
+      return true;
+    } catch (const std::filesystem::filesystem_error& ex) {
+      std::cerr << "Filesystem error: " << ex.what() << std::endl;
+      return false;
+    } catch (const std::exception& ex) {
+      std::cerr << "General error: " << ex.what() << std::endl;
+      return false;
+    }
+  };
+
+  if (!ensure_path_exists(filename)) {
+    LOG(error) << "Filename contains path component which could not be created";
+    return;
+  }
+
   TFile file(filename.data(), "RECREATE");
-  auto cl = TClass::GetClass(typeid(*this));
-  file.WriteObjectAny(this, cl, "DigitizationContext");
-  file.Close();
+  if (file.IsOpen()) {
+    auto cl = TClass::GetClass(typeid(*this));
+    file.WriteObjectAny(this, cl, "DigitizationContext");
+    file.Close();
+  } else {
+    LOG(error) << "Could not write to file " << filename.data();
+  }
 }
 
-DigitizationContext const* DigitizationContext::loadFromFile(std::string_view filename)
+DigitizationContext* DigitizationContext::loadFromFile(std::string_view filename)
 {
   std::string tmpFile;
   if (filename == "") {
@@ -209,22 +300,59 @@ DigitizationContext const* DigitizationContext::loadFromFile(std::string_view fi
   return incontext;
 }
 
-void DigitizationContext::fillQED(std::string_view QEDprefix, std::vector const& irecord)
+void DigitizationContext::fillQED(std::string_view QEDprefix, int max_events, double qedrate)
+{
+  if (mEventRecords.size() <= 1) {
+    // nothing to do
+    return;
+  }
+
+  o2::steer::InteractionSampler qedInteractionSampler;
+  qedInteractionSampler.setBunchFilling(mBCFilling);
+
+  // get first and last "hadronic" interaction records and let
+  // QED events range from the first bunch crossing to the last bunch crossing
+  // in this range
+  auto first = mEventRecords.front();
+  auto last = mEventRecords.back();
+  first.bc = 0;
+  last.bc = o2::constants::lhc::LHCMaxBunches;
+  LOG(info) << "QED RATE " << qedrate;
+  qedInteractionSampler.setInteractionRate(qedrate);
+  qedInteractionSampler.setFirstIR(first);
+  qedInteractionSampler.init();
+  qedInteractionSampler.print();
+  std::vector qedinteractionrecords;
+  o2::InteractionTimeRecord t;
+  LOG(info) << "GENERATING COL TIMES";
+  t = qedInteractionSampler.generateCollisionTime();
+  while ((t = qedInteractionSampler.generateCollisionTime()) < last) {
+    qedinteractionrecords.push_back(t);
+  }
+  LOG(info) << "DONE GENERATING COL TIMES";
+  fillQED(QEDprefix, qedinteractionrecords, max_events, false);
+}
+
+void DigitizationContext::fillQED(std::string_view QEDprefix, std::vector const& irecord, int max_events, bool fromKinematics)
 {
   mQEDSimPrefix = QEDprefix;
 
   std::vector> qedEventParts;
 
-  // we need to fill the QED parts (using a simple round robin scheme)
-  auto qedKinematicsName = o2::base::NameConf::getMCKinematicsFileName(mQEDSimPrefix);
-  // find out how many events are stored
-  TFile f(qedKinematicsName.c_str(), "OPEN");
-  auto t = (TTree*)f.Get("o2sim");
-  if (!t) {
-    LOG(error) << "No QED kinematics found";
-    throw std::runtime_error("No QED kinematics found");
+  int numberQEDevents = max_events; // if this is -1 there will be no limitation
+  if (fromKinematics) {
+    // we need to fill the QED parts (using a simple round robin scheme)
+    auto qedKinematicsName = o2::base::NameConf::getMCKinematicsFileName(mQEDSimPrefix);
+    // find out how many events are stored
+    TFile f(qedKinematicsName.c_str(), "OPEN");
+    auto t = (TTree*)f.Get("o2sim");
+    if (!t) {
+      LOG(error) << "No QED kinematics found";
+      throw std::runtime_error("No QED kinematics found");
+    }
+    numberQEDevents = t->GetEntries();
   }
-  auto numberQEDevents = t->GetEntries();
+
   int eventID = 0;
   for (auto& tmp : irecord) {
     std::vector qedpart;
@@ -257,3 +385,366 @@ void DigitizationContext::fillQED(std::string_view QEDprefix, std::vector> getTimeFrameBoundaries(std::vector const& irecords, long startOrbit, long orbitsPerTF)
+{
+  std::vector> result;
+
+  // the goal is to determine timeframe boundaries inside the interaction record vectors
+  // determine if we can do anything
+  if (irecords.size() == 0) {
+    // nothing to do
+    return result;
+  }
+
+  if (irecords.back().orbit < startOrbit) {
+    LOG(error) << "start orbit larger than last collision entry";
+    return result;
+  }
+
+  // skip to the first index falling within our constrained
+  int left = 0;
+  while (left < irecords.size() && irecords[left].orbit < startOrbit) {
+    left++;
+  }
+
+  // now we can start (2 pointer approach)
+  auto right = left;
+  int timeframe_count = 1;
+  while (right < irecords.size()) {
+    if (irecords[right].orbit >= startOrbit + timeframe_count * orbitsPerTF) {
+      // we finished one timeframe
+      result.emplace_back(std::pair(left, right - 1));
+      timeframe_count++;
+      left = right;
+    }
+    right++;
+  }
+  // finished last timeframe
+  result.emplace_back(std::pair(left, right - 1));
+  return result;
+}
+
+// a common helper for timeframe structure - includes indices for orbits-early (orbits from last timeframe still affecting current one)
+std::vector> getTimeFrameBoundaries(std::vector const& irecords,
+                                                              long startOrbit,
+                                                              long orbitsPerTF,
+                                                              float orbitsEarly)
+{
+  // we could actually use the other method first ... then do another pass to fix the early-index ... or impact index
+  auto true_indices = getTimeFrameBoundaries(irecords, startOrbit, orbitsPerTF);
+
+  std::vector> indices_with_early{};
+  for (int ti = 0; ti < true_indices.size(); ++ti) {
+    // for each timeframe we copy the true indices
+    auto& tf_range = true_indices[ti];
+
+    // init new index without fixing the early index yet
+    indices_with_early.push_back(std::make_tuple(tf_range.first, tf_range.second, -1));
+
+    // from the second timeframe on we can determine the index in the previous timeframe
+    // which matches our criterion
+    if (orbitsEarly > 0. && ti > 0) {
+      auto& prev_tf_range = true_indices[ti - 1];
+      // in this range search the smallest index which precedes
+      // timeframe ti by not more than "orbitsEarly" orbits
+      // (could probably use binary search, in case optimization becomes necessary)
+      int earlyOrbitIndex = -1; // init to start of this timeframe ... there may not be early orbits
+
+      // this is the orbit of the ti-th timeframe start
+      auto orbit_timeframe_start = startOrbit + ti * orbitsPerTF;
+
+      auto orbit_timeframe_early_fractional = 1. * orbit_timeframe_start - orbitsEarly;
+      auto orbit_timeframe_early_integral = static_cast(std::floor(orbit_timeframe_early_fractional));
+
+      auto bc_early = (uint32_t)((orbit_timeframe_early_fractional - orbit_timeframe_early_integral) * o2::constants::lhc::LHCMaxBunches);
+
+      // this is the interaction record of the ti-th timeframe start
+      o2::InteractionRecord timeframe_start_record(0, orbit_timeframe_start);
+      // this is the interaction record in some previous timeframe after which interactions could still
+      // influence the ti-th timeframe according to orbitsEarly
+      o2::InteractionRecord timeframe_early_record(bc_early, orbit_timeframe_early_integral);
+
+      auto differenceInBCNS_max = timeframe_start_record.differenceInBCNS(timeframe_early_record);
+
+      for (int j = prev_tf_range.second; j >= prev_tf_range.first; --j) {
+        // determine difference in timing in NS; compare that with the limit given by orbitsEarly
+        auto timediff_NS = timeframe_start_record.differenceInBCNS(irecords[j]);
+        if (timediff_NS < differenceInBCNS_max) {
+          earlyOrbitIndex = j;
+        } else {
+          break;
+        }
+      }
+      std::get<2>(indices_with_early.back()) = earlyOrbitIndex;
+    }
+  }
+  return indices_with_early;
+}
+
+} // namespace
+
+void DigitizationContext::applyMaxCollisionFilter(std::vector>& timeframeindices, long startOrbit, long orbitsPerTF, int maxColl, double orbitsEarly)
+{
+  // the idea is to go through each timeframe and throw away collisions beyond a certain count
+  // then the indices should be condensed
+
+  std::vector> newparts;
+  std::vector newrecords;
+
+  std::unordered_map currMaxId;                           // the max id encountered for a source
+  std::unordered_map> reIndexMap; // for each source, a map of old to new index for the event parts
+
+  if (maxColl == -1) {
+    maxColl = mEventRecords.size();
+  }
+
+  // the actual first actual timeframe
+  int first_timeframe = orbitsEarly > 0. ? 1 : 0;
+
+  // mapping of old to new indices
+  std::unordered_map indices_old_to_new;
+
+  // now we can go through the structure timeframe by timeframe
+  for (int tf_id = first_timeframe; tf_id < timeframeindices.size(); ++tf_id) {
+    auto& tf_indices = timeframeindices[tf_id];
+
+    auto firstindex = std::get<0>(tf_indices); // .first;
+    auto lastindex = std::get<1>(tf_indices);  // .second;
+    auto previndex = std::get<2>(tf_indices);
+
+    LOG(info) << "timeframe indices " << previndex << " : " << firstindex << " : " << lastindex;
+
+    int collCount = 0; // counting collisions within timeframe
+    // copy to new structure
+    for (int index = previndex >= 0 ? previndex : firstindex; index <= lastindex; ++index) {
+      if (collCount >= maxColl) {
+        continue;
+      }
+
+      // look if this index was already done?
+      // avoid duplicate entries in transformed records
+      if (indices_old_to_new.find(index) != indices_old_to_new.end()) {
+        continue;
+      }
+
+      // we put these events under a certain condition
+      bool keep = index < firstindex || collCount < maxColl;
+
+      if (!keep) {
+        continue;
+      }
+
+      if (index >= firstindex) {
+        collCount++;
+      }
+
+      // we must also make sure that we don't duplicate the records
+      // moreover some records are merely put as precoll of tf2 ---> so they shouldn't be part of tf1 in the final
+      // extraction, ouch !
+      // maybe we should combine the filter and individual tf extraction in one step !!
+      indices_old_to_new[index] = newrecords.size();
+      newrecords.push_back(mEventRecords[index]);
+      newparts.push_back(mEventParts[index]);
+
+      // reindex the event parts to achieve compactification (and initial linear increase)
+      for (auto& part : newparts.back()) {
+        auto source = part.sourceID;
+        auto entry = part.entryID;
+        // have we remapped this entry before?
+        if (reIndexMap.find(source) != reIndexMap.end() && reIndexMap[source].find(entry) != reIndexMap[source].end()) {
+          part.entryID = reIndexMap[source][entry];
+        } else {
+          // assign the next free index
+          if (currMaxId.find(source) == currMaxId.end()) {
+            currMaxId[source] = 0;
+          }
+          part.entryID = currMaxId[source];
+          // cache this assignment
+          reIndexMap[source][entry] = currMaxId[source];
+          currMaxId[source] += 1;
+        }
+      }
+    } // ends one timeframe
+
+    // correct the timeframe indices
+    if (indices_old_to_new.find(firstindex) != indices_old_to_new.end()) {
+      std::get<0>(tf_indices) = indices_old_to_new[firstindex]; // start
+    }
+    if (indices_old_to_new.find(lastindex) != indices_old_to_new.end()) {
+      std::get<1>(tf_indices) = indices_old_to_new[lastindex]; // end;
+    } else {
+      std::get<1>(tf_indices) = newrecords.size() - 1; // end; -1 since index inclusif
+    }
+    if (indices_old_to_new.find(previndex) != indices_old_to_new.end()) {
+      std::get<2>(tf_indices) = indices_old_to_new[previndex]; // previous or "early" index
+    }
+  }
+  // reassignment
+  mEventRecords = newrecords;
+  mEventParts = newparts;
+}
+
+std::vector> DigitizationContext::calcTimeframeIndices(long startOrbit, long orbitsPerTF, double orbitsEarly) const
+{
+  auto timeframeindices = getTimeFrameBoundaries(mEventRecords, startOrbit, orbitsPerTF, orbitsEarly);
+  return timeframeindices;
+}
+
+std::unordered_map DigitizationContext::getCollisionIndicesForSource(int source) const
+{
+  // go through all collisions and pick those that have the give source
+  // then keep only the first collision index
+  std::unordered_map result;
+  const auto& parts = getEventParts(false);
+  for (int collindex = 0; collindex < parts.size(); ++collindex) {
+    for (auto& eventpart : parts[collindex]) {
+      if (eventpart.sourceID == source) {
+        result[eventpart.entryID] = collindex;
+      }
+    }
+  }
+  return result;
+}
+
+int DigitizationContext::findSimPrefix(std::string const& prefix) const
+{
+  auto iter = std::find(mSimPrefixes.begin(), mSimPrefixes.end(), prefix);
+  if (iter != mSimPrefixes.end()) {
+    return std::distance(mSimPrefixes.begin(), iter);
+  }
+  return -1;
+}
+
+namespace
+{
+struct pair_hash {
+  template 
+  std::size_t operator()(const std::pair& pair) const
+  {
+    return std::hash()(pair.first) ^ std::hash()(pair.second);
+  }
+};
+} // namespace
+
+int DigitizationContext::setInteractionVertices(std::vector> const& external_vertices)
+{
+  if (external_vertices.size() != mEventRecords.size()) {
+    LOG(error) << "Size mismatch with event record";
+    return 1;
+  }
+  mInteractionVertices.clear();
+  std::copy(external_vertices.begin(), external_vertices.end(), std::back_inserter(mInteractionVertices));
+  return 0;
+}
+
+void DigitizationContext::sampleInteractionVertices(o2::dataformats::MeanVertexObject const& meanv)
+{
+  // mapping of source x event --> index into mInteractionVertices
+  std::unordered_map, int, pair_hash> vertex_cache;
+
+  mInteractionVertices.clear();
+  int collcount = 0;
+
+  std::unordered_set vset; // used to detect vertex incompatibilities
+  for (auto& coll : mEventParts) {
+    collcount++;
+    vset.clear();
+
+    // first detect if any of these entries already has an associated vertex
+    for (auto& part : coll) {
+      auto source = part.sourceID;
+      auto event = part.entryID;
+      auto cached_iter = vertex_cache.find(std::pair(source, event));
+
+      if (cached_iter != vertex_cache.end()) {
+        vset.emplace(cached_iter->second);
+      }
+    }
+
+    // make sure that there is no conflict
+    if (vset.size() > 1) {
+      LOG(fatal) << "Impossible conflict during interaction vertex sampling";
+    }
+
+    int cacheindex = -1;
+    if (vset.size() == 1) {
+      // all of the parts need to be assigned the same existing vertex
+      cacheindex = *(vset.begin());
+      mInteractionVertices.push_back(mInteractionVertices[cacheindex]);
+    } else {
+      // we need to sample a new point
+      mInteractionVertices.emplace_back(meanv.sample());
+      cacheindex = mInteractionVertices.size() - 1;
+    }
+    // update the cache
+    for (auto& part : coll) {
+      auto source = part.sourceID;
+      auto event = part.entryID;
+      vertex_cache[std::pair(source, event)] = cacheindex;
+    }
+  }
+}
+
+DigitizationContext DigitizationContext::extractSingleTimeframe(int timeframeid, std::vector> const& timeframeindices, std::vector const& sources_to_offset)
+{
+  DigitizationContext r; // make a return object
+  if (timeframeindices.size() == 0) {
+    LOG(error) << "Timeframe index structure empty; Returning empty object.";
+    return r;
+  }
+  r.mSimPrefixes = mSimPrefixes;
+  r.mMuBC = mMuBC;
+  r.mBCFilling = mBCFilling;
+  r.mDigitizerInteractionRate = mDigitizerInteractionRate;
+  try {
+    auto tf_ranges = timeframeindices.at(timeframeid);
+
+    auto startindex = std::get<0>(tf_ranges);
+    auto endindex = std::get<1>(tf_ranges) + 1;
+    auto earlyindex = std::get<2>(tf_ranges);
+
+    if (earlyindex >= 0) {
+      startindex = earlyindex;
+    }
+    std::copy(mEventRecords.begin() + startindex, mEventRecords.begin() + endindex, std::back_inserter(r.mEventRecords));
+    std::copy(mEventParts.begin() + startindex, mEventParts.begin() + endindex, std::back_inserter(r.mEventParts));
+    if (mInteractionVertices.size() >= endindex) {
+      std::copy(mInteractionVertices.begin() + startindex, mInteractionVertices.begin() + endindex, std::back_inserter(r.mInteractionVertices));
+    }
+
+    // let's assume we want to fix the ids for source = source_id
+    // Then we find the first index that has this source_id and take the corresponding number
+    // as offset. Thereafter we subtract this offset from all known event parts.
+    auto perform_offsetting = [&r](int source_id) {
+      auto indices_for_source = r.getCollisionIndicesForSource(source_id);
+      int minvalue = std::numeric_limits::max();
+      for (auto& p : indices_for_source) {
+        if (p.first < minvalue) {
+          minvalue = p.first;
+        }
+      }
+      // now fix them
+      for (auto& p : indices_for_source) {
+        auto index_into_mEventParts = p.second;
+        for (auto& part : r.mEventParts[index_into_mEventParts]) {
+          if (part.sourceID == source_id) {
+            part.entryID -= minvalue;
+          }
+        }
+      }
+    };
+    for (auto source_id : sources_to_offset) {
+      perform_offsetting(source_id);
+    }
+
+  } catch (std::exception) {
+    LOG(warn) << "No such timeframe id in collision context. Returing empty object";
+  }
+  // fix number of collisions
+  r.setNCollisions(r.mEventRecords.size());
+  return r;
+}
diff --git a/DataFormats/simulation/src/InteractionSampler.cxx b/DataFormats/simulation/src/InteractionSampler.cxx
new file mode 100644
index 0000000000000..f3ece5c51f90b
--- /dev/null
+++ b/DataFormats/simulation/src/InteractionSampler.cxx
@@ -0,0 +1,262 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "SimulationDataFormat/InteractionSampler.h"
+#include 
+
+using namespace o2::steer;
+
+//_________________________________________________
+void InteractionSampler::init()
+{
+  // (re-)initialize sample and check parameters consistency
+
+  int nBCSet = mBCFilling.getNBunches();
+  if (!nBCSet) {
+    LOG(warning) << "No bunch filling provided, impose default one";
+    mBCFilling.setDefault();
+    nBCSet = mBCFilling.getNBunches();
+  }
+
+  if (mMuBC < 0. && mIntRate < 0.) {
+    LOG(warning) << "No IR or muBC is provided, setting default IR";
+    mIntRate = DefIntRate;
+  }
+
+  if (mMuBC > 0.) {
+    mIntRate = mMuBC * nBCSet * o2::constants::lhc::LHCRevFreq;
+    LOG(info) << "Deducing IR=" << mIntRate << "Hz from " << nBCSet << " BCs at mu=" << mMuBC;
+  } else {
+    mMuBC = mIntRate / (nBCSet * o2::constants::lhc::LHCRevFreq);
+    LOG(info) << "Deducing mu=" << mMuBC << " per BC from IR=" << mIntRate << " with " << nBCSet << " BCs";
+  }
+
+  mInteractingBCs.clear();
+  mInteractingBCs.reserve(nBCSet);
+  for (int i = 0; i < o2::constants::lhc::LHCMaxBunches; i++) {
+    if (mBCFilling.testBC(i)) {
+      mInteractingBCs.push_back(i);
+    }
+  }
+
+  auto mu = mMuBC;
+  // prob. of not having interaction in N consecutive BCs is P(N) = mu*exp(-(N-1)*mu), hence its cumulative distribution
+  // is T(N) = integral_1^N {P(N)} = 1. - exp(-(N-1)*mu)
+  // We generate random N using its inverse, N = 1 - log(1 - Rndm)/mu
+  mBCJumpGenerator.initialize([mu]() { return (1. - std::log(1. - gRandom->Rndm()) / mu); });
+
+  // Poisson distribution of number of collisions in the bunch excluding 0
+  mNCollBCGenerator.initialize([mu]() {
+    int n = 0;
+    while ((n = gRandom->Poisson(mu)) == 0) {
+      ;
+    }
+    return n;
+  });
+
+  auto trms = mBCTimeRMS;
+  mCollTimeGenerator.initialize([trms]() {
+    float t; // make sure it does not go outside half bunch
+    while (std::abs(t = gRandom->Gaus(0, trms)) > o2::constants::lhc::LHCBunchSpacingNS / 2.1) {
+      ;
+    }
+    return t;
+  });
+
+  mIntBCCache = 0;
+  mCurrBCIdx = 0;
+  mIR = mFirstIR;
+  while (mCurrBCIdx < mInteractingBCs.size() && mInteractingBCs[mCurrBCIdx] < mIR.bc) {
+    mCurrBCIdx++;
+  }
+  // set the "current BC" right in front of the 1st BC to generate. There will be a jump by at least 1 during generation
+  mCurrBCIdx--;
+}
+
+//_________________________________________________
+void InteractionSampler::print() const
+{
+  if (mIntRate < 0) {
+    LOG(error) << "not yet initialized";
+    return;
+  }
+  LOG(info) << "InteractionSampler with " << mInteractingBCs.size() << " colliding BCs, mu(BC)= "
+            << getMuPerBC() << " -> total IR= " << getInteractionRate();
+  LOG(info) << "Current " << mIR << '(' << mIntBCCache << " coll left)";
+}
+
+//_________________________________________________
+const o2::InteractionTimeRecord& InteractionSampler::generateCollisionTime()
+{
+  // generate single interaction record
+  if (mIntRate < 0) {
+    init();
+  }
+
+  if (mIntBCCache < 1) {                   // do we still have interaction in current BC?
+    mIntBCCache = simulateInteractingBC(); // decide which BC interacts and N collisions
+  }
+  mIR.timeInBCNS = mTimeInBC.back();
+  mTimeInBC.pop_back();
+  mIntBCCache--;
+
+  return mIR;
+}
+
+//_________________________________________________
+int InteractionSampler::simulateInteractingBC()
+{
+  // Returns number of collisions assigned to selected BC
+  nextCollidingBC(mBCJumpGenerator.getNextValue());
+
+  // once BC is decided, enforce at least one interaction
+  int ncoll = mNCollBCGenerator.getNextValue();
+
+  // assign random time withing a bunch
+  for (int i = ncoll; i--;) {
+    mTimeInBC.push_back(mCollTimeGenerator.getNextValue());
+  }
+  if (ncoll > 1) { // sort in DECREASING time order (we are reading vector from the end)
+    std::sort(mTimeInBC.begin(), mTimeInBC.end(), [](const float a, const float b) { return a > b; });
+  }
+  return ncoll;
+}
+
+//_________________________________________________
+int FixedSkipBC_InteractionSampler::simulateInteractingBC()
+{
+  // Returns number of collisions assigned to selected BC
+
+  nextCollidingBC(mEveryN);  // we jump regular intervals
+  int ncoll = mMultiplicity; // well defined pileup
+
+  // assign random time withing a bunch
+  for (int i = ncoll; i--;) {
+    mTimeInBC.push_back(mCollTimeGenerator.getNextValue());
+  }
+  if (ncoll > 1) { // sort in DECREASING time order (we are reading vector from the end)
+    std::sort(mTimeInBC.begin(), mTimeInBC.end(), [](const float a, const float b) { return a > b; });
+  }
+  return ncoll;
+}
+
+//_________________________________________________
+void InteractionSampler::setBunchFilling(const std::string& bcFillingFile)
+{
+  // load bunch filling from the file
+  auto* bc = o2::BunchFilling::loadFrom(bcFillingFile, "ccdb_object");
+  if (!bc) {
+    bc = o2::BunchFilling::loadFrom(bcFillingFile); // retry with default naming in case of failure
+  }
+  if (!bc) {
+    LOG(fatal) << "Failed to load bunch filling from " << bcFillingFile;
+  }
+  mBCFilling = *bc;
+  delete bc;
+}
+
+// ________________________________________________
+bool NonUniformMuInteractionSampler::setBCIntensityScales(const std::vector& scales_from_vector)
+{
+  // Sets the intensity scales per bunch crossing index
+  // The length of this vector needs to be compatible with the bunch filling chosen
+  mBCIntensityScales = scales_from_vector;
+
+  if (scales_from_vector.size() != mInteractingBCs.size()) {
+    LOG(error) << "Scaling factors and bunch filling scheme are not compatible. Not doing anything";
+    return false;
+  }
+
+  float sum = 0.;
+  for (auto v : mBCIntensityScales) {
+    sum += std::abs(v);
+  }
+  if (sum == 0) {
+    LOGP(warn, "total intensity is 0, assuming uniform");
+    for (auto& v : mBCIntensityScales) {
+      v = 1.f;
+    }
+  } else { // normalize
+    float norm = mBCIntensityScales.size() / sum;
+    for (auto& v : mBCIntensityScales) {
+      v = std::abs(v) * norm;
+    }
+  }
+  return false;
+}
+
+// ________________________________________________
+
+bool NonUniformMuInteractionSampler::setBCIntensityScales(const TH1F& hist)
+{
+  return setBCIntensityScales(determineBCIntensityScalesFromHistogram(hist));
+}
+
+std::vector NonUniformMuInteractionSampler::determineBCIntensityScalesFromHistogram(const TH1F& hist)
+{
+  if (mInteractingBCs.size() == 0) {
+    LOG(error) << " Initialize bunch crossing scheme before assigning scales";
+  }
+  std::vector scales;
+  // we go through the BCs and query the count from histogram
+  for (auto bc : mInteractingBCs) {
+    scales.push_back(hist.GetBinContent(bc + 1));
+  }
+  return scales;
+}
+
+int NonUniformMuInteractionSampler::getBCJump() const
+{
+  auto muFunc = [this](int bc_position) {
+    return mBCIntensityScales[bc_position % mInteractingBCs.size()] * mMuBC;
+  };
+
+  double U = gRandom->Rndm();    // uniform (0,1)
+  double T = -std::log(1.0 - U); // threshold
+  double sumMu = 0.0;
+  int offset = 0;
+  auto bcStart = mCurrBCIdx; // the current bc
+
+  while (sumMu < T) {
+    auto mu_here = muFunc(bcStart + offset); // mu at next BC
+    sumMu += mu_here;
+    if (sumMu >= T) {
+      break; // found BC with at least one collision
+    }
+    ++offset;
+  }
+  return offset;
+}
+
+int NonUniformMuInteractionSampler::simulateInteractingBC()
+{
+  nextCollidingBC(getBCJump());
+
+  auto muFunc = [this](int bc_position) {
+    return mBCIntensityScales[bc_position % mInteractingBCs.size()] * mMuBC;
+  };
+
+  // now sample number of collisions in chosenBC, conditioned >=1:
+  double mu_chosen = muFunc(mCurrBCIdx); // or does it need to be mCurrBCIdx
+  int ncoll = 0;
+  do {
+    ncoll = gRandom->Poisson(mu_chosen);
+  } while (ncoll == 0);
+
+  // assign random time withing a bunch
+  for (int i = ncoll; i--;) {
+    mTimeInBC.push_back(mCollTimeGenerator.getNextValue());
+  }
+  if (ncoll > 1) { // sort in DECREASING time order (we are reading vector from the end)
+    std::sort(mTimeInBC.begin(), mTimeInBC.end(), [](const float a, const float b) { return a > b; });
+  }
+  return ncoll;
+}
\ No newline at end of file
diff --git a/DataFormats/simulation/src/MCCompLabel.cxx b/DataFormats/simulation/src/MCCompLabel.cxx
index 8e4f2ad73f876..656fcd9770579 100644
--- a/DataFormats/simulation/src/MCCompLabel.cxx
+++ b/DataFormats/simulation/src/MCCompLabel.cxx
@@ -14,6 +14,7 @@
 #include 
 #include 
 #include 
+#include 
 
 namespace o2
 {
@@ -31,6 +32,16 @@ std::ostream& operator<<(std::ostream& os, MCCompLabel const& c)
   return os;
 }
 
+//_____________________________________________
+std::string MCCompLabel::asString() const
+{
+  // describe itself
+  if (isValid()) {
+    return fmt::format("[{}/{}{}/{:6d}]", getSourceID(), getEventID(), isFake() ? '-' : '+', getTrackID());
+  }
+  return isNoise() ? "[noise]" : "[unset]";
+}
+
 //_____________________________________________
 void MCCompLabel::print() const
 {
diff --git a/DataFormats/simulation/src/MCEventHeader.cxx b/DataFormats/simulation/src/MCEventHeader.cxx
index 99004fec46ffa..8a46d84805507 100644
--- a/DataFormats/simulation/src/MCEventHeader.cxx
+++ b/DataFormats/simulation/src/MCEventHeader.cxx
@@ -15,6 +15,7 @@
 #include "FairRootManager.h"
 #include 
 #include 
+#include 
 
 namespace o2
 {
@@ -61,6 +62,69 @@ void MCEventHeader::extractFileFromKinematics(std::string_view kinefilename, std
   }
 }
 
+void MCEventHeader::print() const
+{
+  // print some used parts from FairMCEventHeader
+  std::cout << "RunID " << fRunId << "\n";
+  std::cout << "EventID " << fEventId << "\n";
+  std::cout << "Vertex-X " << fX << "\n";
+  std::cout << "Vertex-Y " << fY << "\n";
+  std::cout << "Vertex-Z " << fZ << "\n";
+  std::cout << "Vertex-T " << fT << "\n";
+  std::cout << "Impact-B " << fB << "\n";
+  std::cout << "NPrim " << fNPrim << "\n";
+  // print meta-fields
+  printInfo();
+}
+
+/** alternative implementations below
+
+void MCEventHeader::extractFileFromKinematics(std::string_view kinefilename, std::string_view targetfilename) {
+  TFile kinefile(kinefilename.data(), "OPEN");
+  auto kinetree = static_cast(kinefile.Get("o2sim")); // belongs to "kinefile"
+  kinetree->SetBranchStatus("*", 0);
+  // activate header branch
+  kinetree->SetBranchStatus("MCEventHeader*", 1);
+  std::unique_ptr headeronlyfile{TFile::Open(targetfilename.data(), "RECREATE")};
+  auto headeronlytree = kinetree->CloneTree(0);
+
+  // copy the branches
+  headeronlytree->CopyEntries(kinetree, kinetree->GetEntries());
+  headeronlytree->SetEntries(kinetree->GetEntries());
+  // flush to disc
+  headeronlytree->Write();
+}
+
+void MCEventHeader::extractFileFromKinematics(std::string_view kinefilename, std::string_view targetfilename)
+{
+  TFile kinefile(kinefilename.data(), "OPEN");
+  auto kinetree = static_cast(kinefile.Get("o2sim")); // owned by "kinefile"
+  auto headerbranchorig = kinetree->GetBranch("MCEventHeader.");
+  if (!headerbranchorig) {
+    return;
+  }
+
+  std::unique_ptr headeronlyfile{TFile::Open(targetfilename.data(), "RECREATE")};
+  auto headeronlytree = new TTree("o2sim", "o2sim"); // owned by headeronlyfile
+  MCEventHeader* header = nullptr;
+  auto targetBranch = headeronlytree->Branch("MCEventHeader.", &header);
+  headerbranchorig->SetAddress(&header);
+
+  for (auto entry = 0; entry < kinetree->GetEntries(); ++entry) {
+    headerbranchorig->GetEntry(entry);
+    targetBranch->Fill();
+    if (header) {
+      delete header;
+      header = nullptr;
+    }
+  }
+  headeronlytree->SetEntries(kinetree->GetEntries());
+  // flush to disc
+  headeronlytree->Write();
+}
+
+*/
+
 /*****************************************************************/
 /*****************************************************************/
 
diff --git a/DataFormats/simulation/src/MCEventLabel.cxx b/DataFormats/simulation/src/MCEventLabel.cxx
index 31607eb6d3010..56ae521d4480e 100644
--- a/DataFormats/simulation/src/MCEventLabel.cxx
+++ b/DataFormats/simulation/src/MCEventLabel.cxx
@@ -14,6 +14,7 @@
 #include 
 #include 
 #include 
+#include 
 
 using namespace o2;
 
@@ -24,6 +25,13 @@ void MCEventLabel::print() const
   std::cout << (MCEventLabel) * this << std::endl;
 }
 
+//_____________________________________________
+std::string MCEventLabel::asString() const
+{
+  // stream itself
+  return isSet() ? fmt::format("[{}/{}/{}]", getSourceID(), getEventID(), getCorrWeight()) : "[unset]";
+}
+
 //_____________________________________________
 std::ostream& operator<<(std::ostream& os, const o2::MCEventLabel& c)
 {
diff --git a/DataFormats/simulation/src/MCTrack.cxx b/DataFormats/simulation/src/MCTrack.cxx
index 9d7c6b5e2a16c..bd5360c8f5d13 100644
--- a/DataFormats/simulation/src/MCTrack.cxx
+++ b/DataFormats/simulation/src/MCTrack.cxx
@@ -14,7 +14,7 @@
 /// \author M. Al-Turany, S. Wenzel - June 2014
 
 #include "SimulationDataFormat/MCTrack.h"
-#include "FairLogger.h"
+#include 
 
 namespace o2
 {
diff --git a/DataFormats/simulation/src/MCUtils.cxx b/DataFormats/simulation/src/MCUtils.cxx
index 91599a97b3889..14cf7ab7048a4 100644
--- a/DataFormats/simulation/src/MCUtils.cxx
+++ b/DataFormats/simulation/src/MCUtils.cxx
@@ -14,6 +14,7 @@
 //
 
 #include 
+#include 
 
 namespace o2::mcutils
 {
@@ -77,17 +78,17 @@ bool MCTrackNavigator::isPhysicalPrimary(o2::MCTrack const& p, std::vector
   // ist > 1 --> essentially means unstable
-  if ((ist > 1) && (pdg != 130) && p.isPrimary()) {
+  if ((hepmcStatusCode > 1) && (pdg != 130) && p.isPrimary()) {
     return false;
   }
-  if ((ist > 1) && p.isSecondary()) {
+  if ((hepmcStatusCode > 1) && p.isSecondary()) {
     return false;
   }
   // <-
@@ -207,4 +208,19 @@ bool MCTrackNavigator::isKeepPhysics(o2::MCTrack const& p, std::vector + ;
 #pragma link C++ class o2::MCTrackT < double> + ;
@@ -46,7 +48,8 @@
 #pragma link C++ class o2::BasicXYZQHit < double, double, int> + ;
 #pragma link C++ struct o2::dataformats::MCTruthHeaderElement + ;
 #pragma link C++ class o2::dataformats::MCTruthContainer < long> + ;
-#pragma link C++ class o2::dataformats::MCTruthContainer < o2::MCCompLabel > -;
+#pragma link C++ class o2::dataformats::MCTruthContainer < o2::MCCompLabel> - ;
+#pragma link C++ class o2::dataformats::ConstMCTruthContainer < o2::MCCompLabel> - ;
 #pragma link C++ class std::vector < o2::dataformats::MCTruthContainer < o2::MCCompLabel>> + ;
 #pragma link C++ class std::vector < o2::MCCompLabel> + ;
 #pragma link C++ class std::vector < o2::MCEventLabel> + ;
@@ -72,6 +75,7 @@
 #pragma link C++ class o2::dataformats::IOMCTruthContainerView + ;
 
 #pragma link C++ class o2::mcutils::MCTrackNavigator + ;
+#pragma link C++ class o2::mcutils::MCGenHelper + ;
 #pragma link C++ class o2::O2DatabasePDG + ;
 
 #endif
diff --git a/DataFormats/simulation/test/MCTrack.cxx b/DataFormats/simulation/test/MCTrack.cxx
index fc0b52bca1a59..e857e2b88da41 100644
--- a/DataFormats/simulation/test/MCTrack.cxx
+++ b/DataFormats/simulation/test/MCTrack.cxx
@@ -12,14 +12,9 @@
 #define BOOST_TEST_MODULE Test MCTrack class
 #define BOOST_TEST_MAIN
 #define BOOST_TEST_DYN_LINK
-#include "SimulationDataFormat/MCTrack.h"
 #include 
-#include 
-#include 
-#include 
+#include "SimulationDataFormat/MCTrack.h"
 #include "DetectorsCommonDataFormats/DetID.h"
-#include "SimulationDataFormat/Stack.h"
-#include "SimulationDataFormat/PrimaryChunk.h"
 #include "TFile.h"
 #include "TParticle.h"
 #include "TMCProcess.h"
@@ -29,10 +24,17 @@ using namespace o2;
 BOOST_AUTO_TEST_CASE(MCTrack_test)
 {
   MCTrack track;
+
+  // auxiliary lookup table needed to fetch and set hit properties
+  std::vector hitLUT(o2::detectors::DetID::nDetectors, -1);
+  // in this test we have a single fictional detector 1, which we map to
+  // the first bit
+  hitLUT[1] = 0;
+
   // check properties on default constructed object
   BOOST_CHECK(track.getStore() == false);
   for (auto i = o2::detectors::DetID::First; i < o2::detectors::DetID::nDetectors; ++i) {
-    BOOST_CHECK(track.leftTrace(i) == false);
+    BOOST_CHECK(track.leftTrace(i, hitLUT) == false);
   }
   BOOST_CHECK(track.getNumDet() == 0);
   BOOST_CHECK(track.hasHits() == false);
@@ -46,10 +48,10 @@ BOOST_AUTO_TEST_CASE(MCTrack_test)
   BOOST_CHECK(track.getStore() == true);
 
   // set hit for first detector
-  BOOST_CHECK(track.leftTrace(1) == false);
-  track.setHit(1);
+  BOOST_CHECK(track.leftTrace(1, hitLUT) == false);
+  track.setHit(hitLUT[1]);
   BOOST_CHECK(track.hasHits() == true);
-  BOOST_CHECK(track.leftTrace(1) == true);
+  BOOST_CHECK(track.leftTrace(1, hitLUT) == true);
   BOOST_CHECK(track.getNumDet() == 1);
 
   // check process encoding
@@ -73,29 +75,3 @@ BOOST_AUTO_TEST_CASE(MCTrack_test)
     BOOST_CHECK(intrack->hasHits() == true);
   }
 }
-
-// unit tests on stack
-BOOST_AUTO_TEST_CASE(Stack_test)
-{
-  o2::data::Stack st;
-  int a;
-  TMCProcess proc{kPPrimary};
-  // add a 2 primary particles
-  st.PushTrack(1, -1, 0, 0, 0., 0., 10., 5., 5., 5., 0.1, 0., 0., 0., proc, a, 1., 1);
-  st.PushTrack(1, -1, 0, 0, 0., 0., 10., 5., 5., 5., 0.1, 0., 0., 0., proc, a, 1., 1);
-  BOOST_CHECK(st.getPrimaries().size() == 2);
-
-  {
-    // serialize it
-    TFile f("StackOut.root", "RECREATE");
-    f.WriteObject(&st, "Stack");
-    f.Close();
-  }
-
-  {
-    o2::data::Stack* inst = nullptr;
-    TFile f("StackOut.root", "OPEN");
-    f.GetObject("Stack", inst);
-    BOOST_CHECK(inst->getPrimaries().size() == 2);
-  }
-}
diff --git a/DataFormats/simulation/test/testBasicHits.cxx b/DataFormats/simulation/test/testBasicHits.cxx
index e81c173fedae8..ccd16ae7a3671 100644
--- a/DataFormats/simulation/test/testBasicHits.cxx
+++ b/DataFormats/simulation/test/testBasicHits.cxx
@@ -66,7 +66,7 @@ BOOST_AUTO_TEST_CASE(BasicXYZHit_ROOTIO)
   }
 
   // same for double valued hits
-  using HitTypeD = BasicXYZEHit;
+  using HitTypeD = BasicXYZEHit;
   HitTypeD hitD(1., 2., 3., 0.01, -1.1, -1, 1);
 
   // try writing hit to a TBuffer
diff --git a/DataFormats/simulation/test/testInteractionSampler.cxx b/DataFormats/simulation/test/testInteractionSampler.cxx
new file mode 100644
index 0000000000000..b1b3691884ccf
--- /dev/null
+++ b/DataFormats/simulation/test/testInteractionSampler.cxx
@@ -0,0 +1,76 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#define BOOST_TEST_MODULE Test InteractionSampler class
+#define BOOST_TEST_MAIN
+#define BOOST_TEST_DYN_LINK
+
+#include 
+#include "SimulationDataFormat/InteractionSampler.h"
+#include "CCDB/BasicCCDBManager.h"
+#include "DataFormatsParameters/AggregatedRunInfo.h"
+#include "DataFormatsParameters/GRPLHCIFData.h"
+#include "TFile.h"
+#include "TGrid.h"
+#include 
+
+namespace o2
+{
+
+BOOST_AUTO_TEST_CASE(NonUniformSampler)
+{
+  auto run_number = 559827;
+  TGrid::Connect("alien");
+  if (gGrid) {
+    auto runInfo = o2::parameters::AggregatedRunInfo::buildAggregatedRunInfo(o2::ccdb::BasicCCDBManager::instance(), run_number);
+
+    o2::steer::NonUniformMuInteractionSampler sampler;
+    sampler.setBunchFilling(runInfo.grpLHC->getBunchFilling());
+
+    // the test distribution provided by Igor Altsybeev
+    auto distr_file = TFile::Open("alien:///alice/cern.ch/user/s/swenzel/AliceO2_TestData/NBcVTX_559827/hBcTVX_data_PbPb_24ar_559827.root");
+
+    //
+    if (distr_file && !distr_file->IsZombie()) {
+      auto hist = distr_file->Get("hBcTVX");
+      if (hist) {
+        sampler.init();
+        sampler.setBCIntensityScales(*hist);
+
+        // sample into a vector of a certain size
+        std::vector samples;
+
+        int N = 100000;
+        samples.resize(N);
+
+        sampler.generateCollisionTimes(samples);
+
+        // fill an output histogram
+        auto output_hist = (TH1F*)hist->Clone("h2"); // make a full copy
+        output_hist->Reset();
+
+        for (const auto& sample : samples) {
+          output_hist->Fill(sample.bc);
+        }
+
+        // Write out
+        auto fout = TFile::Open("NBCVTX_out.root", "RECREATE");
+        fout->WriteObject(output_hist, "NBcVTX");
+        fout->Close();
+
+        // compare mean values of original and newly sampled hist
+        BOOST_CHECK_CLOSE(hist->GetMean(), output_hist->GetMean(), 0.5);
+      }
+    }
+  }
+}
+
+} // namespace o2
diff --git a/DataFormats/simulation/test/testMCGenId.cxx b/DataFormats/simulation/test/testMCGenId.cxx
new file mode 100644
index 0000000000000..246268ac78d52
--- /dev/null
+++ b/DataFormats/simulation/test/testMCGenId.cxx
@@ -0,0 +1,50 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#define BOOST_TEST_MODULE Test MCGenId class
+#define BOOST_TEST_MAIN
+#define BOOST_TEST_DYN_LINK
+#include 
+#include "SimulationDataFormat/MCGenProperties.h"
+
+using namespace o2::mcgenid;
+
+BOOST_AUTO_TEST_CASE(MCGenId_test)
+{
+  // possible generator IDs range from 0 to 127 (included)
+  constexpr int highGenerator{128};
+  // possible sub-generator IDs range from -1 to 30 (included)
+  constexpr int highSubGenerator{31};
+  // possible soufce IDs range from 0 to 15 (included)
+  constexpr int highSource{16};
+
+  // test all combinations
+  for (int sourceId = 0; sourceId < highSource; sourceId++) {
+    for (int generatorId = 0; generatorId < highGenerator; generatorId++) {
+      for (int subGeneratorId = -1; subGeneratorId < highSubGenerator; subGeneratorId++) {
+        auto encoded = getEncodedGenId(generatorId, sourceId, subGeneratorId);
+        // decode them
+        auto sourceIdAfter = getSourceId(encoded);
+        auto generatorIdAfter = getGeneratorId(encoded);
+        auto subGeneratorIdAfter = getSubGeneratorId(encoded);
+
+        std::cout << "SourceID: " << sourceId << " ==> " << sourceIdAfter << "\n"
+                  << "generatorId: " << generatorId << " ==> " << generatorIdAfter << "\n"
+                  << "subGeneratorId: " << subGeneratorId << " ==> " << subGeneratorIdAfter << "\n";
+
+        // check if original and decoded numbers are the same
+        BOOST_CHECK(sourceIdAfter == sourceId);
+        BOOST_CHECK(generatorIdAfter == generatorId);
+        BOOST_CHECK(subGeneratorId == subGeneratorIdAfter);
+      }
+    }
+  }
+}
diff --git a/DataFormats/simulation/test/testMCGenStatus.cxx b/DataFormats/simulation/test/testMCGenStatus.cxx
new file mode 100644
index 0000000000000..a9b8b7277b3e7
--- /dev/null
+++ b/DataFormats/simulation/test/testMCGenStatus.cxx
@@ -0,0 +1,94 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#define BOOST_TEST_MODULE Test MCGenStatus class
+#define BOOST_TEST_MAIN
+#define BOOST_TEST_DYN_LINK
+#include 
+#include "SimulationDataFormat/MCTrack.h"
+#include "SimulationDataFormat/ParticleStatus.h"
+#include "SimulationDataFormat/MCGenProperties.h"
+#include "SimulationDataFormat/MCUtils.h"
+#include "TFile.h"
+#include "TParticle.h"
+
+using namespace o2;
+using namespace o2::mcgenstatus;
+
+BOOST_AUTO_TEST_CASE(MCGenStatus_test)
+{
+  // check basic properties
+  const int status{91};
+
+  // should not be flagged as encoded when the encoding is set
+  MCGenStatusEncoding enc1(isEncodedValue);
+  BOOST_CHECK(enc1.isEncoded != isEncodedValue);
+
+  // check if everuthing correctly assigned
+  MCGenStatusEncoding enc2(status, -status);
+  BOOST_CHECK(enc2.isEncoded == isEncodedValue);
+  BOOST_CHECK(enc2.hepmc == status);
+  BOOST_CHECK(enc2.gen == -status);
+
+  // check if helper functionality works as expected
+  BOOST_CHECK(getHepMCStatusCode(enc2) == status);
+  BOOST_CHECK(getGenStatusCode(enc2) == -status);
+  BOOST_CHECK(getHepMCStatusCode(enc2) == getHepMCStatusCode(enc2.fullEncoding));
+  BOOST_CHECK(getGenStatusCode(enc2) == getGenStatusCode(enc2.fullEncoding));
+
+  // check default constructed MCTrack object's status code
+  MCTrack track1;
+  BOOST_CHECK(track1.getStatusCode().fullEncoding == 0);
+
+  // check MCTrack object constructed from TParticle (primary)
+  TParticle part2(22, MCGenStatusEncoding(status, -status).fullEncoding, 1, 2, 3, 4, 0.1, 0.1, 0.1, 0.1, 0, 0, 0, 0);
+  part2.SetBit(ParticleStatus::kPrimary);
+  MCTrack track2(part2);
+  BOOST_CHECK(getHepMCStatusCode(track2.getStatusCode()) == status);
+  BOOST_CHECK(getGenStatusCode(track2.getStatusCode()) == -status);
+
+  // make sure status codes survive serialising and deserialising
+  {
+    // serialize it
+    TFile f("MCGenStatusOut.root", "RECREATE");
+    f.WriteObject(&track2, "MCTrack");
+    f.Close();
+  }
+
+  {
+    MCTrack* intrack = nullptr;
+    TFile f("MCGenStatusOut.root", "OPEN");
+    f.GetObject("MCTrack", intrack);
+    BOOST_CHECK(getHepMCStatusCode(intrack->getStatusCode()) == status);
+    BOOST_CHECK(getGenStatusCode(intrack->getStatusCode()) == -status);
+  }
+
+  // check MCTrack object constructed from TParticle (secondary)
+  MCTrack track3(TParticle(22, MCGenStatusEncoding(status, -status).fullEncoding, 1, 2, 3, 4, 0.1, 0.1, 0.1, 0.1, 0, 0, 0, 0));
+  // both must now be -1 since not encoced and hence only number is returned
+  BOOST_CHECK(getHepMCStatusCode(track3.getStatusCode()) == -1);
+  BOOST_CHECK(getGenStatusCode(track3.getStatusCode()) == -1);
+
+  // do tests to check utility functionality
+  int thisHepMCCode{4};
+  TParticle part3(22, thisHepMCCode, 1, 2, 3, 4, 0.1, 0.1, 0.1, 0.1, 0, 0, 0, 0);
+  std::vector trueFalse{true, false};
+  for (auto value : trueFalse) {
+    auto status3 = part3.GetStatusCode();
+    BOOST_CHECK(mcgenstatus::isEncoded(status3) != value);
+    mcutils::MCGenHelper::encodeParticleStatusAndTracking(part3, value);
+    status3 = part3.GetStatusCode();
+    BOOST_CHECK(mcgenstatus::isEncoded(status3));
+    BOOST_CHECK(getHepMCStatusCode(status3) == thisHepMCCode);
+    BOOST_CHECK(getGenStatusCode(status3) == 0);
+    BOOST_CHECK(part3.TestBit(ParticleStatus::kToBeDone) == value);
+  }
+}
diff --git a/DataFormats/simulation/test/testMCTruthContainer.cxx b/DataFormats/simulation/test/testMCTruthContainer.cxx
index 071421073e0ae..5691a2207842b 100644
--- a/DataFormats/simulation/test/testMCTruthContainer.cxx
+++ b/DataFormats/simulation/test/testMCTruthContainer.cxx
@@ -32,6 +32,8 @@ BOOST_AUTO_TEST_CASE(MCTruth)
   container.addElement(0, TruthElement(2));
   container.addElement(1, TruthElement(1));
   container.addElement(2, TruthElement(10));
+  container.addNoLabelIndex(3);
+  container.addElement(4, TruthElement(4));
 
   // not supported, must throw
   BOOST_CHECK_THROW(container.addElement(0, TruthElement(0)), std::runtime_error);
@@ -76,16 +78,26 @@ BOOST_AUTO_TEST_CASE(MCTruth)
   BOOST_CHECK(view.size() == 1);
   BOOST_CHECK(view[0] == 10);
 
-  // add multiple labels
+  // add multiple labels (to last index which already had a label)
   std::vector newlabels = {101, 102, 103};
-  container.addElements(2, newlabels);
-  view = container.getLabels(2);
+  container.addElements(4, newlabels);
+  view = container.getLabels(4);
   BOOST_CHECK(view.size() == 4);
-  BOOST_CHECK(view[0] == 10);
+  BOOST_CHECK(view[0] == 4);
   BOOST_CHECK(view[1] == 101);
   BOOST_CHECK(view[2] == 102);
   BOOST_CHECK(view[3] == 103);
 
+  // check empty labels case
+  view = container.getLabels(3);
+  BOOST_CHECK(view.size() == 0);
+
+  // add empty label vector
+  std::vector newlabels2 = {};
+  container.addElements(5, newlabels2);
+  view = container.getLabels(5);
+  BOOST_CHECK(view.size() == 0);
+
   // test merging
   {
     dataformats::MCTruthContainer container1;
@@ -329,7 +341,8 @@ BOOST_AUTO_TEST_CASE(MCTruthContainer_ROOTIO)
   using Container = dataformats::MCTruthContainer;
   Container container;
   const size_t BIGSIZE{1000000};
-  for (int i = 0; i < BIGSIZE; ++i) {
+  container.addNoLabelIndex(0); // the first index does not have a label
+  for (int i = 1; i < BIGSIZE; ++i) {
     container.addElement(i, TruthElement(i, i, i));
     container.addElement(i, TruthElement(i + 1, i, i));
   }
@@ -342,6 +355,7 @@ BOOST_AUTO_TEST_CASE(MCTruthContainer_ROOTIO)
     TFile f("tmp2.root", "RECREATE");
     TTree tree("o2sim", "o2sim");
     auto br = tree.Branch("Labels", &io, 32000, 2);
+    tree.Branch("LabelsOriginal", &container, 32000, 2);
     tree.Fill();
     tree.Write();
     f.Close();
@@ -361,14 +375,30 @@ BOOST_AUTO_TEST_CASE(MCTruthContainer_ROOTIO)
   ConstMCTruthContainer cc;
   io2->copyandflatten(cc);
 
-  BOOST_CHECK(cc.getNElements() == BIGSIZE * 2);
+  BOOST_CHECK(cc.getNElements() == (BIGSIZE - 1) * 2);
   BOOST_CHECK(cc.getIndexedSize() == BIGSIZE);
-  BOOST_CHECK(cc.getLabels(0).size() == 2);
-  BOOST_CHECK(cc.getLabels(0)[0] == TruthElement(0, 0, 0));
-  BOOST_CHECK(cc.getLabels(0)[1] == TruthElement(1, 0, 0));
+  BOOST_CHECK(cc.getLabels(0).size() == 0);
+  BOOST_CHECK(cc.getLabels(1).size() == 2);
+  BOOST_CHECK(cc.getLabels(1)[0] == TruthElement(1, 1, 1));
+  BOOST_CHECK(cc.getLabels(1)[1] == TruthElement(2, 1, 1));
   BOOST_CHECK(cc.getLabels(BIGSIZE - 1).size() == 2);
   BOOST_CHECK(cc.getLabels(BIGSIZE - 1)[0] == TruthElement(BIGSIZE - 1, BIGSIZE - 1, BIGSIZE - 1));
   BOOST_CHECK(cc.getLabels(BIGSIZE - 1)[1] == TruthElement(BIGSIZE, BIGSIZE - 1, BIGSIZE - 1));
+
+  // testing convenience API to retrieve a constant label container from a ROOT file, entry 0
+  auto cont = o2::dataformats::MCLabelIOHelper::loadFromTTree(tree2, "Labels", 0);
+  auto cont2 = o2::dataformats::MCLabelIOHelper::loadFromTTree(tree2, "LabelsOriginal", 0);
+
+  BOOST_CHECK(cont);
+  BOOST_CHECK(cont2);
+  BOOST_CHECK(cont->getNElements() == (BIGSIZE - 1) * 2);
+  BOOST_CHECK(cont2->getNElements() == (BIGSIZE - 1) * 2);
+  BOOST_CHECK(cont->getLabels(0).size() == 0);
+  BOOST_CHECK(cont2->getLabels(0).size() == 0);
+  BOOST_CHECK(cont->getLabels(BIGSIZE - 1)[0] == TruthElement(BIGSIZE - 1, BIGSIZE - 1, BIGSIZE - 1));
+  BOOST_CHECK(cont->getLabels(BIGSIZE - 1)[1] == TruthElement(BIGSIZE, BIGSIZE - 1, BIGSIZE - 1));
+  BOOST_CHECK(cont2->getLabels(BIGSIZE - 1)[0] == TruthElement(BIGSIZE - 1, BIGSIZE - 1, BIGSIZE - 1));
+  BOOST_CHECK(cont2->getLabels(BIGSIZE - 1)[1] == TruthElement(BIGSIZE, BIGSIZE - 1, BIGSIZE - 1));
 }
 
 } // namespace o2
diff --git a/Detectors/AOD/CMakeLists.txt b/Detectors/AOD/CMakeLists.txt
index e875770329fc6..827b23b3e4cdd 100644
--- a/Detectors/AOD/CMakeLists.txt
+++ b/Detectors/AOD/CMakeLists.txt
@@ -19,6 +19,7 @@ target_link_libraries(
           O2::FDDWorkflow
           O2::FV0Workflow
           O2::Framework
+          O2::FrameworkAnalysisSupport
           O2::GlobalTracking
           O2::GlobalTrackingWorkflow
           O2::ITSMFTWorkflow
@@ -40,8 +41,16 @@ add_library(internal::AODProducerWorkflow ALIAS AODProducerWorkflow)
 o2_add_executable(
   workflow
   COMPONENT_NAME aod-producer
-  SOURCES src/aod-producer-workflow.cxx src/AODProducerWorkflowSpec.cxx
-  PUBLIC_LINK_LIBRARIES internal::AODProducerWorkflow O2::Version
+  TARGETVARNAME targetName
+  SOURCES src/aod-producer-workflow.cxx src/AODProducerWorkflowSpec.cxx src/AODMcProducerHelpers.cxx
+  PUBLIC_LINK_LIBRARIES internal::AODProducerWorkflow O2::Version nlohmann_json::nlohmann_json
+)
+
+o2_add_executable(
+  workflow
+  COMPONENT_NAME aod-mc-producer
+  SOURCES src/aod-mc-producer-workflow.cxx src/AODMcProducerWorkflowSpec.cxx src/AODMcProducerHelpers.cxx
+  PUBLIC_LINK_LIBRARIES internal::AODProducerWorkflow O2::Version nlohmann_json::nlohmann_json
 )
 
 o2_add_executable(
@@ -67,4 +76,11 @@ o2_add_executable(
         O2::DataFormatsFT0
         O2::Steer
         O2::ZDCBase
-)
+        O2::FrameworkAnalysisSupport
+        nlohmann_json::nlohmann_json
+        )
+
+if (OpenMP_CXX_FOUND)
+  target_compile_definitions(${targetName} PRIVATE WITH_OPENMP)
+  target_link_libraries(${targetName} PRIVATE OpenMP::OpenMP_CXX)
+endif()
diff --git a/Detectors/AOD/README.md b/Detectors/AOD/README.md
new file mode 100644
index 0000000000000..42a9840fabc04
--- /dev/null
+++ b/Detectors/AOD/README.md
@@ -0,0 +1,50 @@
+
+
+# AOD producers
+
+The code in this directory is meant to produce `AO2D.root` output
+files.   There are two such producers:
+
+- [`o2-aod-producer-workflow`](src/AODProducerWorkflowSpec.cxx) that
+  produces `AO2D.root` from reconstructed data (whether from
+  simulations or the experiment).  The output will contain
+  reconstructed tracks "calo" hits, and so on.
+- [`o2-aod-mc-producer-workflow`](src/AODMcProducerWorkflowSpec.cxx)
+  that produces `AO2D.root` from simulation data only. The output will
+  only contain the simulation particles and collision headers.
+
+## Do a full simulation to AO2D chain
+
+You need [`O2DPG`](https://github.com/AliceO2Group/O2DPG/) installed.
+
+Then, a script a la
+
+    #!/bin/bash
+    #
+    # A example workflow MC->RECO->AOD for a simple pp production
+    # excluding ZDC
+    NWORKERS=${NWORKERS:-6}
+    SIMENGINE=${SIMENGINE:-TGeant4}
+
+    ${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py \
+                 -eCM 14000 \
+                 -col pp \
+                 -gen pythia8 \
+                 -proc "cdiff" \
+                 -tf 2  \
+                 -ns 50 \
+                 -e ${SIMENGINE} \
+                 -j ${NWORKERS} \
+                 -run 303000 \
+                 -seed 624 \
+                 -interactionRate 50000
+
+    ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json -tt aod
+
+will set-up a chain and run it.  The above will make 2 time-frames,
+each with 50 pp collisions at sqrt(s)=14TeV generated by Pythia8.
+
+See also [this page](https://aliceo2group.github.io/simulation/) for more on simulations with O2.
+
diff --git a/Detectors/AOD/include/AODProducerWorkflow/AODMcProducerHelpers.h b/Detectors/AOD/include/AODProducerWorkflow/AODMcProducerHelpers.h
new file mode 100644
index 0000000000000..5e9cd445b576b
--- /dev/null
+++ b/Detectors/AOD/include/AODProducerWorkflow/AODMcProducerHelpers.h
@@ -0,0 +1,325 @@
+// Copyright 2023-2099 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// @file   AODMcProducerHelpers.h
+/// @author Christian Holm Christensen 
+/// common helpers for AOD MC producers
+
+#ifndef O2_AODMCPRODUCER_HELPERS
+#define O2_AODMCPRODUCER_HELPERS
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+/**
+ * Utilities to transform simulated data into AO2D tables.
+ *
+ * The function templates below are templated on the cursor type over
+ * the relevant AOD tables.  Such a table can be obtained from the
+ * ProcessingContext @c pc
+ *
+ * @code
+ * auto builder = pc.make(OutputForTable::ref());
+ * auto cursor  = builder->cursor
();
+ * @endcode
+ *
+ * If the task uses the @c Produces
 template,
+ *
+ * @code
+ * Produces
 mTable;
+ * @endcode
+ *
+ * then a cursor is obtained via,
+ *
+ * @code
+ * auto cursor = mTable.cursor;
+ * @endcode
+ *
+ * Note that these functions cannot be moved into a compilation unit,
+ * because that would require deducing the table cursor type, by
+ * f.ex.
+ *
+ * @code
+ * template 
+ * struct TableCursor {
+ *    using cursor_t = decltype(std::declval()
+ *                              .cursor
());
+ * };
+ * using CollisionCursor = TableCursor:cursor_t;
+ * @endcode
+ *
+ * but since cursors are really Lambdas and Lambda types are specific
+ * to the compilation unit, then the implementation file (compilation
+ * unit) of these functions definitions and their use (another
+ * compilation unit) would have different types of the the cursers,
+ * and thus not be able to link.  More information is given at
+ * https://stackoverflow.com/questions/50033797.
+ */
+namespace o2::aodmchelpers
+{
+//==================================================================
+/**
+ * Deduce cursor type and wrap in std::function
+ */
+template 
+struct TableCursor {
+  using type = decltype(framework::FFL(std::declval()
+                                         .cursor
()));
+};
+//==================================================================
+/** Cursor over aod::McCollisions */
+using CollisionCursor = TableCursor::type;
+/** Cursor over aod::McParticles */
+using ParticleCursor = TableCursor::type;
+/** Cursor over aod::HepMCXSections */
+using XSectionCursor = TableCursor::type;
+/** Cursor over aod::HepMCPdfInfos */
+using PdfInfoCursor = TableCursor::type;
+/** Cursor over aod::HepMCHeavyIons */
+using HeavyIonCursor = TableCursor::type;
+//==================================================================
+/** Types of updates on HepMC tables. */
+enum HepMCUpdate {
+  never,
+  always,
+  anyKey,
+  allKeys
+};
+
+//==================================================================
+/**
+ * Check if header has keys.  If the argument @a anyNotAll is true,
+ * then this member function returns true if @e any of the keys
+ * were found.  If @a anyNotAll is false, then return true only if
+ * @a all keys were found.
+ *
+ * @param header    MC event header
+ * @param keys      Keys to look for
+ * @param anyNotAll If true, return true if @e any key was found.
+ *                  If false, return true only if @a all keys were found
+ *
+ * @return true if any or all keys were found
+ */
+bool hasKeys(o2::dataformats::MCEventHeader const& header,
+             const std::vector& keys,
+             bool anyNotall = true);
+//--------------------------------------------------------------------
+/**
+ * Get a property from the header, or if not set or not valid, a
+ * default value.
+ *
+ * @param header  The MC event header
+ * @param key     Key to look for
+ * @param def     Value to return if key is not found
+ *
+ * @return Value of key or def if key is not found
+ */
+template 
+const T getEventInfo(o2::dataformats::MCEventHeader const& header,
+                     std::string const& key,
+                     T const& def)
+{
+  if (not header.hasInfo(key))
+    return def;
+
+  bool isValid = false;
+  const T& val = header.getInfo(key, isValid);
+  if (not isValid)
+    return def;
+
+  return val;
+}
+//====================================================================
+/**
+ * Fill in collision information.  This is read from the passed MC
+ * header and stored in the MCCollision table.  The member function
+ * returns the encoded generator ID.
+ *
+ * @param cursor      Cursor over o2::aod::McCollisions table
+ * @param bcId        Bunch-crossing Identifier
+ * @param time        Time of collisions
+ * @param header      Event header from generator
+ * @param generatorId Default generator
+ * @param sourceId    Identifier of source
+ *
+ * @return encoded generator ID
+ */
+short updateMCCollisions(const CollisionCursor& cursor,
+                         int bcId,
+                         float time,
+                         o2::dataformats::MCEventHeader const& header,
+                         short generatorId = 0,
+                         int sourceId = 0,
+                         unsigned int mask = 0xFFFFFFF0);
+//--------------------------------------------------------------------
+/**
+ * Fill in HepMC cross-section table from event generator header.
+ *
+ * @param cursor      Cursor over o2::aod::HepMCXSections table
+ * @param collisionID Identifier of collision (as given updateMCCollision)
+ * @param generatorID Encoded generator ID
+ * @param header      Event header from generator
+ * @param anyNotAll   If true, then any key present trigger and update.
+ *                    If false, then all keys must be present to update
+ *                    the table.
+ *
+ * @return true if table was updated
+ */
+bool updateHepMCXSection(const XSectionCursor& cursor,
+                         int collisionID,
+                         short generatorID,
+                         o2::dataformats::MCEventHeader const& header,
+                         HepMCUpdate when = HepMCUpdate::anyKey);
+//--------------------------------------------------------------------
+/**
+ * Fill in HepMC parton distribution function table from event
+ * generator header
+ *
+ * @param cursor      Cursor over o2::aod::HepMCXSections table
+ * @param collisionID Identifier of collision (as given updateMCCollision)
+ * @param generatorID Encoded generator ID
+ * @param header      Event header from generator
+ * @param anyNotAll   If true, then any key present trigger and update.
+ *                    If false, then all keys must be present to update
+ *                    the table.
+ *
+ * @return true if table was updated
+ */
+bool updateHepMCPdfInfo(const PdfInfoCursor& cursor,
+                        int collisionID,
+                        short generatorID,
+                        o2::dataformats::MCEventHeader const& header,
+                        HepMCUpdate when = HepMCUpdate::anyKey);
+//--------------------------------------------------------------------
+/**
+ * Fill in HepMC heavy-ion table from generator event header.
+ *
+ * @param cursor      Cursor over o2::aod::HepMCXSections table
+ * @param collisionID Identifier of collision (as given updateMCCollision)
+ * @param generatorID Encoded generator ID
+ * @param header      Event header from generator
+ * @param anyNotAll   If true, then any key present trigger and update.
+ *                    If false, then all keys must be present to update
+ *                    the table.
+ *
+ * @return true if table was updated
+ */
+bool updateHepMCHeavyIon(const HeavyIonCursor& cursor,
+                         int collisionID,
+                         short generatorID,
+                         o2::dataformats::MCEventHeader const& header,
+                         HepMCUpdate when = HepMCUpdate::anyKey);
+//--------------------------------------------------------------------
+/**
+ * Type of mapping from track number to row index
+ */
+using TrackToIndex = std::unordered_map;
+//--------------------------------------------------------------------
+/**
+ * Update aod::McParticles table with information from an MC track.
+ *
+ * @param cursor       Cursor over aod::McParticles table
+ * @param mapping      Maps track number to index in table
+ * @param collisionID  Collision identifier
+ * @param track        Track to update table with
+ * @param tracks       List of all tracks of current collision
+ * @param flags        Base flags of this track
+ * @param weightMask   Mask on weight floating point value
+ * @param momentumMask Mask on momentum floating point values
+ * @param positionMask Mask on position floating point values
+ */
+void updateParticle(const ParticleCursor& cursor,
+                    const TrackToIndex& toStore,
+                    int collisionID,
+                    o2::MCTrack const& track,
+                    std::vector const& tracks,
+                    uint8_t flags = 0,
+                    uint32_t weightMask = 0xFFFFFFF0,
+                    uint32_t momentumMask = 0xFFFFFFF0,
+                    uint32_t positionMask = 0xFFFFFFF0);
+//--------------------------------------------------------------------
+/**
+ * Update aod::McParticles table with tracks from MC.
+ *
+ * To add particles from many events, one will do
+ *
+ * @code
+ * TrackToIndex preselect = findMcTracksToStore(...);
+ *
+ * size_t offset = 0;
+ * for (auto event : events)
+ *    offset = updateParticles(cursor,
+ *                             event.getCollisionID(),
+ *                             event.getTracks(),
+ *                             offset,
+ *                             filter,
+ *                             event.isBackground(),
+ *                             preselect);
+ * @endcode
+ *
+ * Here @a preselect must be a map from track number to a positive
+ * value.  Tracks that are mapped as such in @a preselect are stored
+ * in addition to other tracks selected by the function.  Note that @a
+ * preselect may be empty.
+ *
+ * If @a filter is false, then @a all tracks will be stored.
+ *
+ * If @a filter is true, then tracks that are
+ *
+ * - generated by the generator,
+ * - physical primaries
+ *   (MCTrackNavigator::isPhysicalPrimary),
+ * - to be kept for physics
+ *   (MCTrackNavigator::isKeepPhysics), or
+ * - is listed with a positive value in @a preselect, or
+ * - either a mother or daughter of one such track, then
+ *
+ * that track is kept
+ *
+ * On return, the @a preselect will map from track number (index in
+ * the @a tracks container) to the table row index (including offset
+ * from previous events in the same time-frame).
+ *
+ * @param cursor       Cursor over aod::McParticles
+ * @param int          Collision identifier
+ * @param tracks       List of all tracks of current collision
+ * @param offset       Index just beyond last table entry
+ * @param filter       Filter tracks
+ * @param background   True of from background event
+ * @param preselect    Mapping of preselected tracks
+ * @param weightMask   Mask on weight floating point value
+ * @param momentumMask Mask on momentum floating point values
+ * @param positionMask Mask on position floating point values
+ *
+ * @return Index beyond the last particle added to table
+ */
+uint32_t updateParticles(const ParticleCursor& cursor,
+                         int collisionID,
+                         std::vector const& tracks,
+                         TrackToIndex& preselect,
+                         uint32_t offset = 0,
+                         bool filter = false,
+                         bool background = false,
+                         uint32_t weightMask = 0xFFFFFFF0,
+                         uint32_t momentumMask = 0xFFFFFFF0,
+                         uint32_t positionMask = 0xFFFFFFF0,
+                         bool signalFilter = false);
+} // namespace o2::aodmchelpers
+
+#endif /* O2_AODMCPRODUCER_HELPERS */
+// Local Variables:
+//   mode: C++
+// End:
diff --git a/Detectors/AOD/include/AODProducerWorkflow/AODMcProducerWorkflowSpec.h b/Detectors/AOD/include/AODProducerWorkflow/AODMcProducerWorkflowSpec.h
new file mode 100644
index 0000000000000..674fbf7bfcd05
--- /dev/null
+++ b/Detectors/AOD/include/AODProducerWorkflow/AODMcProducerWorkflowSpec.h
@@ -0,0 +1,147 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// @file   AODMcProducerWorkflowSpec.h
+
+#ifndef O2_AODMCPRODUCER_WORKFLOW_SPEC
+#define O2_AODMCPRODUCER_WORKFLOW_SPEC
+
+#include "AODProducerHelpers.h"
+#include "AODMcProducerHelpers.h"
+#include "CommonDataFormat/InteractionRecord.h"
+#include "Framework/AnalysisDataModel.h"
+#include "Framework/AnalysisHelpers.h"
+#include "Framework/Task.h"
+#include "Framework/DataProcessorSpec.h"
+#include "Steer/MCKinematicsReader.h"
+#include 
+
+#include 
+#include 
+
+using namespace o2::framework;
+
+namespace o2::aodmcproducer
+{
+
+class AODMcProducerWorkflowDPL : public Task
+{
+ public:
+  AODMcProducerWorkflowDPL() = default;
+  ~AODMcProducerWorkflowDPL() override = default;
+  void init(InitContext& ic) final;
+  void run(ProcessingContext& pc) final;
+  void endOfStream(EndOfStreamContext& ec) final;
+
+  /** Some types we will use */
+  using MCEventHeader = dataformats::MCEventHeader;
+  using McCollisions = aod::McCollisions;
+  using McParticles = aod::StoredMcParticles;
+  using Origins = aod::Origins;
+  using XSections = aod::HepMCXSections;
+  using PdfInfos = aod::HepMCPdfInfos;
+  using HeavyIons = aod::HepMCHeavyIons;
+  using MCKinematicsReader = steer::MCKinematicsReader;
+  /** */
+ private:
+  /** some other types we will use */
+  using CollisionCursor = aodmchelpers::CollisionCursor;
+  using XSectionCursor = aodmchelpers::XSectionCursor;
+  using PdfInfoCursor = aodmchelpers::PdfInfoCursor;
+  using HeavyIonCursor = aodmchelpers::HeavyIonCursor;
+  using HepMCUpdate = aodmchelpers::HepMCUpdate;
+  /**
+   * Update the header (collision and HepMC aux) information.
+   *
+   * When updating the HepMC aux tables, we take the relevant policies
+   * into account (mXSectionUpdate, mPdfInfoUpdate, mHeavyIonUpdate).
+   *
+   * - If a policy is "never", then the corresponding table is never
+   *   updated.
+   *
+   * - If the policy is "always", then the table is always
+   *   update.
+   *
+   * - If the policy is either "anyKey" or "allKeys", _and_
+   *   this is the first event, then we check if any or all keys,
+   *   respectively are present in the header.
+   *
+   *   - If that check fails, then we do not update and set the
+   *     corresponding policy to be "never".
+   *
+   *   - If the check succeeds, then we do update the table, and set
+   *     the corresponding policty to "always".
+   *
+   *   In this way, we will let the first event decide what to do for
+   *   subsequent events and thus avoid too many string comparisions.
+   *
+   * @param collisionCursor Cursor over aod::McCollisions
+   * @param xSectionCursor Cursor over aod::HepMCXSections
+   * @param pdfInfoCursor Cursor over aod::HepMCPdfInfos
+   * @param heavyIonCursor Cursor over aod::HepMCHeavyIons
+   * @param header Header to read information from
+   * @param collisionID Index of collision in table
+   * @param bcID Current event identifier (bcID)
+   * @param time Time of event
+   * @param generatorID Generator identifier, if any
+   * @param sourceID Source identifier
+   *
+   */
+  void updateHeader(CollisionCursor& collisionCursor,
+                    XSectionCursor& xSectionCursor,
+                    PdfInfoCursor& pdfInfoCursor,
+                    HeavyIonCursor& heavyIonCursor,
+                    const MCEventHeader& header,
+                    int collisionID, // Index
+                    int bcID,
+                    float time,
+                    short generatorID,
+                    int sourceID);
+  /** Current timeframe number */
+  int64_t mTFNumber{1};
+  /** Whether to filter tracks so that only primary particles,
+      particles from EG, or their parents are written out */
+  int mFilterMC{0};
+  /** Whether to enable embedding */
+  bool mEnableEmbed{false};
+  /** LPM production tag, for anchoring */
+  TString mLPMProdTag{""};
+  /** Pass identifier for anchoring */
+  TString mAnchorPass{""};
+  /** Production identifier for anchoring */
+  TString mAnchorProd{""};
+  /** Reconstruction pass for anchoring */
+  TString mRecoPass{""};
+  /** Timer */
+  TStopwatch mTimer;
+  /** Rules for when to update HepMC tables */
+  HepMCUpdate mXSectionUpdate = HepMCUpdate::anyKey;
+  HepMCUpdate mPdfInfoUpdate = HepMCUpdate::anyKey;
+  HepMCUpdate mHeavyIonUpdate = HepMCUpdate::anyKey;
+
+  std::string mSimPrefix;
+
+  // MC production metadata holder
+  bool mIsMDSent{false};
+
+  // truncation is enabled by default
+  uint32_t mCollisionPosition = 0xFFFFFFF0; // 19 bits mantissa
+  uint32_t mMcParticleW = 0xFFFFFFF0;       // 19 bits
+  uint32_t mMcParticlePos = 0xFFFFFFF0;     // 19 bits
+  uint32_t mMcParticleMom = 0xFFFFFFF0;     // 19 bits
+};
+
+/// create a processor spec
+framework::DataProcessorSpec getAODMcProducerWorkflowSpec();
+
+} // namespace o2::aodmcproducer
+
+#endif /* O2_AODMCPRODUCER_WORKFLOW_SPEC */
diff --git a/Detectors/AOD/include/AODProducerWorkflow/AODProducerHelpers.h b/Detectors/AOD/include/AODProducerWorkflow/AODProducerHelpers.h
new file mode 100644
index 0000000000000..5351504443269
--- /dev/null
+++ b/Detectors/AOD/include/AODProducerWorkflow/AODProducerHelpers.h
@@ -0,0 +1,61 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// @file   AODProducerHelpers.h
+/// common helpers for AOD producers
+
+#ifndef O2_AODPRODUCER_HELPERS
+#define O2_AODPRODUCER_HELPERS
+
+#include 
+#include 
+#include 
+#include 
+
+namespace o2::aodhelpers
+{
+
+typedef boost::tuple Triplet_t;
+
+struct TripletHash {
+  std::size_t operator()(Triplet_t const& e) const
+  {
+    std::size_t seed = 0;
+    boost::hash_combine(seed, e.get<0>());
+    boost::hash_combine(seed, e.get<1>());
+    boost::hash_combine(seed, e.get<2>());
+    return seed;
+  }
+};
+
+struct TripletEqualTo {
+  bool operator()(Triplet_t const& x, Triplet_t const& y) const
+  {
+    return (x.get<0>() == y.get<0>() &&
+            x.get<1>() == y.get<1>() &&
+            x.get<2>() == y.get<2>());
+  }
+};
+
+typedef boost::unordered_map TripletsMap_t;
+
+template 
+auto createTableCursor(framework::ProcessingContext& pc)
+{
+  framework::Produces c;
+  c.resetCursor(pc.outputs()
+                  .make(framework::OutputForTable::ref()));
+  c.setLabel(aod::label());
+  return c;
+}
+} // namespace o2::aodhelpers
+
+#endif /* O2_AODPRODUCER_HELPERS */
diff --git a/Detectors/AOD/include/AODProducerWorkflow/AODProducerWorkflowSpec.h b/Detectors/AOD/include/AODProducerWorkflow/AODProducerWorkflowSpec.h
index 6416a1f390928..2c58db42ed856 100644
--- a/Detectors/AOD/include/AODProducerWorkflow/AODProducerWorkflowSpec.h
+++ b/Detectors/AOD/include/AODProducerWorkflow/AODProducerWorkflowSpec.h
@@ -14,39 +14,29 @@
 #ifndef O2_AODPRODUCER_WORKFLOW_SPEC
 #define O2_AODPRODUCER_WORKFLOW_SPEC
 
-#include "CCDB/BasicCCDBManager.h"
-#include "DataFormatsFT0/RecPoints.h"
-#include "DataFormatsFDD/RecPoint.h"
-#include "DataFormatsFV0/RecPoints.h"
+#include "AODMcProducerHelpers.h"
+#include "DataFormatsEMCAL/Cell.h"
 #include "DataFormatsGlobalTracking/RecoContainer.h"
-#include "DataFormatsITS/TrackITS.h"
-#include "DataFormatsMFT/TrackMFT.h"
-#include "DataFormatsMCH/TrackMCH.h"
-#include "DataFormatsTPC/TrackTPC.h"
+#include "DataFormatsPHOS/Cell.h"
 #include "DataFormatsTRD/TrackTRD.h"
-#include "DataFormatsZDC/BCRecData.h"
-#include "DataFormatsEMCAL/EventHandler.h"
-#include "DataFormatsPHOS/EventHandler.h"
 #include "DetectorsBase/GRPGeomHelper.h"
-#include "Framework/AnalysisDataModel.h"
-#include "Framework/AnalysisHelpers.h"
+#include "DetectorsBase/Propagator.h"
 #include "Framework/DataProcessorSpec.h"
 #include "Framework/Task.h"
 #include "ReconstructionDataFormats/GlobalTrackID.h"
-#include "ReconstructionDataFormats/PrimaryVertex.h"
-#include "ReconstructionDataFormats/TrackTPCITS.h"
 #include "ReconstructionDataFormats/VtxTrackIndex.h"
-#include "SimulationDataFormat/MCCompLabel.h"
 #include "Steer/MCKinematicsReader.h"
-#include "TMap.h"
 #include "TStopwatch.h"
-
-#include 
-#include 
-#include 
-#include 
+#include "ZDCBase/Constants.h"
+#include "GlobalTracking/MatchGlobalFwd.h"
+#include "CommonUtils/TreeStreamRedirector.h"
+#include "CommonUtils/EnumFlags.h"
+
+#include 
+#include 
+#include 
 #include 
-
+#include 
 using namespace o2::framework;
 using GID = o2::dataformats::GlobalTrackID;
 using GIndex = o2::dataformats::VtxTrackIndex;
@@ -54,35 +44,178 @@ using DataRequest = o2::globaltracking::DataRequest;
 
 namespace o2::aodproducer
 {
+/// A structure or container to organize bunch crossing data of a timeframe
+/// and to facilitate fast lookup and search within bunch crossings.
+class BunchCrossings
+{
+ public:
+  /// Constructor initializes the acceleration structure
+  BunchCrossings() = default;
 
-typedef boost::tuple Triplet_t;
+  /// initialize this container (to be ready for lookup/search queries)
+  void init(std::map const& bcs)
+  {
+    clear();
+    // init the structures
+    for (auto& key : bcs) {
+      mBCTimeVector.emplace_back(key.first);
+    }
+    initTimeWindows();
+  }
 
-struct TripletHash : std::unary_function {
-  std::size_t operator()(Triplet_t const& e) const
+  /// return the sorted vector of increaing BC times
+  std::vector const& getBCTimeVector() const { return mBCTimeVector; }
+
+  /// Performs a "lower bound" search for timestamp within the bunch
+  /// crossing data.
+  ///
+  /// Returns the smallest bunch crossing (index and value) equal or
+  /// greater than timestamp.
+  ///
+  /// The functions is expected to perform much better than a binary
+  /// search in the bunch crossing data directly. Expect O(1) instead
+  /// of O(log(N)) at the cost of the additional memory used by this
+  /// class.
+  ///
+  /// This is _not_ O(1).  The loop below makes it at least O(N).  The
+  /// call to std::lower_bound is O(log(N)).
+  std::pair lower_bound(uint64_t timestamp) const
   {
-    std::size_t seed = 0;
-    boost::hash_combine(seed, e.get<0>());
-    boost::hash_combine(seed, e.get<1>());
-    boost::hash_combine(seed, e.get<2>());
-    return seed;
+    // a) determine the timewindow
+    const auto NofWindows = static_cast(mTimeWindows.size());
+    const auto smallestBC = mBCTimeVector[0];
+    const auto largestBC = mBCTimeVector.back();
+    auto timeindex = std::max((int)0, (int)((timestamp - smallestBC) / mWindowSize));
+
+    if (timeindex >= NofWindows) {
+      // do extra check avoid valse positive due to machine precision
+      if (timestamp > largestBC) { // there is no next greater; so the bc index is at the end of the vector
+        return std::make_pair(mBCTimeVector.size(), 0);
+      }
+      timeindex = int(mBCTimeVector.size() - 1);
+    }
+
+    const auto* timewindow = &mTimeWindows[timeindex];
+    while (timeindex < NofWindows && (!timewindow->isOccupied() || mBCTimeVector[timewindow->to] < timestamp)) {
+      timeindex = timewindow->nextOccupiedRight;
+      if (timeindex < NofWindows) {
+        timewindow = &mTimeWindows[timeindex];
+      }
+    }
+    if (timeindex >= NofWindows) {
+      // there is no next greater; so the bc index is at the end of the vector
+      return std::make_pair(mBCTimeVector.size(), 0);
+    }
+    // otherwise we actually do a search now
+    std::pair p;
+    auto iter = std::lower_bound(mBCTimeVector.begin() + timewindow->from, mBCTimeVector.begin() + timewindow->to + 1, timestamp);
+    int k = std::distance(mBCTimeVector.begin(), iter);
+    p.first = k;
+    p.second = mBCTimeVector[k];
+    return p;
   }
-};
 
-struct TripletEqualTo : std::binary_function {
-  bool operator()(Triplet_t const& x, Triplet_t const& y) const
+  /// clear/reset this container
+  void clear()
   {
-    return (x.get<0>() == y.get<0>() &&
-            x.get<1>() == y.get<1>() &&
-            x.get<2>() == y.get<2>());
+    mBCs.clear();
+    mBCTimeVector.clear();
+    mTimeWindows.clear();
   }
-};
 
-typedef boost::unordered_map TripletsMap_t;
+  /// print information about this container
+  void print()
+  {
+    LOG(info) << "Have " << mBCTimeVector.size() << " BCs";
+    for (auto t : mBCTimeVector) {
+      LOG(info) << t;
+    }
+    int twcount = 0;
+    auto wsize = mWindowSize;
+    for (auto& tw : mTimeWindows) {
+      LOG(info) << "TimeWindow " << twcount << " [ " << wsize * twcount << ":" << wsize * (twcount + 1) << " ]  : from " << tw.from << " to " << tw.to << " nextLeft " << tw.nextOccupiedLeft << " nextRight " << tw.nextOccupiedRight;
+      twcount++;
+    }
+  }
+
+ private:
+  std::map mBCs;
+  std::vector mBCTimeVector; // simple sorted vector of BC times
+
+  /// initialize the internal acceleration structure
+  void initTimeWindows()
+  {
+    // on average we want say M bunch crossings per time window
+    const int M = 5;
+    int window_number = mBCTimeVector.size() / M;
+    if (mBCTimeVector.size() % M != 0) {
+      window_number += 1;
+    }
+    auto bcrange = (mBCTimeVector.back() + 1 - mBCTimeVector[0]);
+    if (bcrange > (uint64_t(3564 * 258))) {
+      LOGP(warn, "Attention: BC range {}:{} covers more than 258 orbits", mBCTimeVector[0], mBCTimeVector.back());
+    }
+    mWindowSize = bcrange / (1. * window_number);
+    // now we go through the list of times and bucket them into the correct windows
+    mTimeWindows.resize(window_number);
+    for (auto bcindex = 0U; bcindex < mBCTimeVector.size(); ++bcindex) {
+      auto windowindex = (int)((mBCTimeVector[bcindex] - mBCTimeVector[0]) / mWindowSize);
+      // we add "bcindex" to the TimeWindow windowindex
+      auto& tw = mTimeWindows[windowindex];
+      if (tw.from == -1) {
+        tw.from = bcindex;
+      } else {
+        tw.from = std::min(tw.from, static_cast(bcindex));
+      }
+      if (tw.to == -1) {
+        tw.to = bcindex;
+      } else {
+        tw.to = std::max(tw.to, static_cast(bcindex));
+      }
+    }
+
+    // now we do the neighbourhood linking of time windows
+    int lastoccupied = -1;
+    for (int windowindex = 0; windowindex < window_number; ++windowindex) {
+      mTimeWindows[windowindex].nextOccupiedLeft = lastoccupied;
+      if (mTimeWindows[windowindex].isOccupied()) {
+        lastoccupied = windowindex;
+      }
+    }
+    lastoccupied = window_number;
+    for (int windowindex = window_number - 1; windowindex >= 0; --windowindex) {
+      mTimeWindows[windowindex].nextOccupiedRight = lastoccupied;
+      if (mTimeWindows[windowindex].isOccupied()) {
+        lastoccupied = windowindex;
+      }
+    }
+  }
+
+  /// Internal structure to "cover" the time duration of all BCs with
+  /// constant time intervals to speed up searching for a particular BC.
+  /// The structure keeps indices into mBCTimeVector denoting the BCs contained within.
+  struct TimeWindow {
+    int from = -1;
+    int to = -1;
+    int nextOccupiedRight = -1; // next time window occupied to the right
+    int nextOccupiedLeft = -1;  // next time window which is occupied to the left
+    inline bool size() const { return to - from; }
+    inline bool isOccupied() const { return size() > 0; }
+  }; // end struct
+
+  std::vector mTimeWindows; // the time window structure covering the complete duration of mBCTimeVector
+  double mWindowSize;                   // the size of a single time window
+}; // end internal class
+
+// Steering bits for additional output during AOD production
+enum struct AODProducerStreamerFlags : uint8_t {
+  TrackQA,
+};
 
 class AODProducerWorkflowDPL : public Task
 {
  public:
-  AODProducerWorkflowDPL(GID::mask_t src, std::shared_ptr dataRequest, std::shared_ptr gr, bool enableSV, std::string resFile, bool useMC = true) : mInputSources(src), mDataRequest(dataRequest), mGGCCDBRequest(gr), mEnableSV(enableSV), mResFile{resFile}, mUseMC(useMC) {}
+  AODProducerWorkflowDPL(GID::mask_t src, std::shared_ptr dataRequest, std::shared_ptr gr, bool enableSV, bool useMC = true, bool enableFITextra = false) : mUseMC(useMC), mEnableSV(enableSV), mEnableFITextra(enableFITextra), mInputSources(src), mDataRequest(dataRequest), mGGCCDBRequest(gr) {}
   ~AODProducerWorkflowDPL() override = default;
   void init(InitContext& ic) final;
   void run(ProcessingContext& pc) final;
@@ -101,8 +234,34 @@ class AODProducerWorkflowDPL : public Task
     return std::uint64_t(mStartIR.toLong()) + relativeTime_to_LocalBC(relativeTimeStampInNS);
   }
 
+  bool mThinTracks{false};
+  bool mPropTracks{false};
+  bool mPropMuons{false};
+  float mTrackQCKeepGlobalTracks{false};
+  float mTrackQCRetainOnlydEdx{false};
+  float mTrackQCFraction{0.00};
+  int64_t mTrackQCNTrCut{4};
+  float mTrackQCDCAxy{3.};
+  float mTrackQCPt{0.2};
+  int mTrackQCNCls{80};
+  float mSqrtS{13860.};
+  std::mt19937 mGenerator{}; ///< random generator for trackQA sampling
+  o2::base::Propagator::MatCorrType mMatCorr{o2::base::Propagator::MatCorrType::USEMatCorrLUT};
+  o2::dataformats::MeanVertexObject mVtx;
+  float mMaxPropXiu{5.0f}; // max X_IU for which track is to be propagated if mPropTracks is true. (other option: o2::constants::geom::XTPCInnerRef + 0.1f)
+
+  std::unordered_set mGIDUsedBySVtx;
+  std::unordered_set mGIDUsedByStr;
+
+  o2::utils::EnumFlags mStreamerFlags;
+  std::shared_ptr mStreamer;
+
+  int mNThreads = 1;
   bool mUseMC = true;
-  bool mEnableSV = true;             // enable secondary vertices
+  bool mUseSigFiltMC = false; // enable signal filtering for MC with embedding
+  bool mEnableSV = true; // enable secondary vertices
+  bool mEnableFITextra = false;
+  bool mFieldON = false;
   const float cSpeed = 0.029979246f; // speed of light in TOF units
 
   GID::mask_t mInputSources;
@@ -111,12 +270,14 @@ class AODProducerWorkflowDPL : public Task
   int mTruncate{1};
   int mRecoOnly{0};
   o2::InteractionRecord mStartIR{}; // TF 1st IR
-  TString mResFile{"AO2D"};
   TString mLPMProdTag{""};
   TString mAnchorPass{""};
   TString mAnchorProd{""};
   TString mRecoPass{""};
+  TString mUser{"aliprod"}; // who created this AOD (aliprod, alidaq, individual users)
   TStopwatch mTimer;
+  bool mEMCselectLeading{false};
+  uint64_t mEMCALTrgClassMask = 0;
 
   // unordered map connects global indices and table indices of barrel tracks
   std::unordered_map mGIDToTableID;
@@ -134,6 +295,11 @@ class AODProducerWorkflowDPL : public Task
   std::unordered_map mV0ToTableID;
   int mTableV0ID{0};
 
+  // Strangeness tracking indices lookup tables
+  std::vector mVertexStrLUT;                    /// LUT for accessing strangeness tracks for each vertex
+  std::vector> mCollisionStrTrk; /// collision index and original index of the strangeness track
+  std::vector mStrTrkIndices;                   /// indices of strangeness tracks in the track table
+
   //  std::unordered_map mIndexTableFwd;
   std::vector mIndexTableFwd;
   int mIndexFwdID{0};
@@ -141,10 +307,12 @@ class AODProducerWorkflowDPL : public Task
   std::vector mIndexTableMFT;
   int mIndexMFTID{0};
 
+  BunchCrossings mBCLookup;
+
   // zdc helper maps to avoid a number of "if" statements
   // when filling ZDC table
-  map mZDCEnergyMap; // mapping detector name to a corresponding energy
-  map mZDCTDCMap;    // mapping TDC channel to a corresponding TDC value
+  std::array mZDCEnergyMap; // mapping detector id to a corresponding energy
+  std::array mZDCTDCMap; // mapping TDC channel id to a corresponding TDC value
 
   std::vector mITSTPCTRDTriggers; // mapping from TRD tracks ID to corresponding trigger (for tracks time extraction)
   std::vector mTPCTRDTriggers;    // mapping from TRD tracks ID to corresponding trigger (for tracks time extraction)
@@ -153,18 +321,28 @@ class AODProducerWorkflowDPL : public Task
   std::vector mMCHROFs;           // mapping from MCH tracks ID to corresponding ROF (for SA MCH tracks time extraction)
   double mITSROFrameHalfLengthNS = -1;      // ITS ROF half length
   double mMFTROFrameHalfLengthNS = -1;      // ITS ROF half length
+  double mITSROFBiasNS = 0;                 // ITS ROF start bias
+  double mMFTROFBiasNS = 0;                 // ITS ROF start bias
   double mNSigmaTimeTrack = -1;             // number track errors sigmas (for gaussian errors only) used in track-vertex matching
   double mTimeMarginTrackTime = -1;         // safety margin in NS used for track-vertex matching (additive to track uncertainty)
   double mTPCBinNS = -1;                    // inverse TPC time-bin in ns
 
-  TripletsMap_t mToStore;
+  // Container used to mark MC particles to store/transfer to AOD.
+  // Mapping of eventID, sourceID, trackID to some integer.
+  // The first two indices are not sparse whereas the trackID index is sparse which explains
+  // the combination of vector and map
+  std::vector>> mToStore;
+  o2::steer::MCKinematicsReader* mMCKineReader = nullptr; //!
 
-  // MC production metadata holder
-  TMap mMetaData;
+  // production metadata
+  std::vector mMetaDataKeys;
+  std::vector mMetaDataVals;
 
   std::shared_ptr mDataRequest;
   std::shared_ptr mGGCCDBRequest;
 
+  o2::globaltracking::MatchGlobalFwd mMatching;
+
   static constexpr int TOFTimePrecPS = 16; // required max error in ps for TOF tracks
   // truncation is enabled by default
   uint32_t mCollisionPosition = 0xFFFFFFF0;    // 19 bits mantissa
@@ -179,14 +357,17 @@ class AODProducerWorkflowDPL : public Task
   uint32_t mTrackCovOffDiag = 0xFFFF0000;      // 7 bits
   uint32_t mTrackSignal = 0xFFFFFF00;          // 15 bits
   uint32_t mTrackTime = 0xFFFFFFFF;            // use full float precision for time
+  uint32_t mTPCTime0 = 0xFFFFFFE0;             // 18 bits, providing 14256./(1<<19) = 0.027 TB precision e.g., ~0.13 mm in z
   uint32_t mTrackTimeError = 0xFFFFFF00;       // 15 bits
   uint32_t mTrackPosEMCAL = 0xFFFFFF00;        // 15 bits
   uint32_t mTracklets = 0xFFFFFF00;            // 15 bits
   uint32_t mMcParticleW = 0xFFFFFFF0;          // 19 bits
   uint32_t mMcParticlePos = 0xFFFFFFF0;        // 19 bits
   uint32_t mMcParticleMom = 0xFFFFFFF0;        // 19 bits
-  uint32_t mCaloAmp = 0xFFFFFF00;              // 15 bits
-  uint32_t mCaloTime = 0xFFFFFF00;             // 15 bits
+  uint32_t mCaloAmp = 0xFFFFFF00;              // 15 bits todo check which truncation should actually be used
+  uint32_t mCaloTime = 0xFFFFFF00;             // 15 bits todo check which truncation should actually be used
+  uint32_t mCPVPos = 0xFFFFF800;               // 12 bits
+  uint32_t mCPVAmpl = 0xFFFFFF00;              // 15 bits
   uint32_t mMuonTr1P = 0xFFFFFC00;             // 13 bits
   uint32_t mMuonTrThetaX = 0xFFFFFF00;         // 15 bits
   uint32_t mMuonTrThetaY = 0xFFFFFF00;         // 15 bits
@@ -197,19 +378,25 @@ class AODProducerWorkflowDPL : public Task
   uint32_t mMuonCl = 0xFFFFFF00;               // 15 bits
   uint32_t mMuonClErr = 0xFFFF0000;            // 7 bits
   uint32_t mV0Time = 0xFFFFF000;               // 11 bits
+  uint32_t mV0ChannelTime = 0xFFFFFF00;        // 15 bits
   uint32_t mFDDTime = 0xFFFFF000;              // 11 bits
+  uint32_t mFDDChannelTime = 0xFFFFFF00;       // 15 bits
   uint32_t mT0Time = 0xFFFFFF00;               // 15 bits
+  uint32_t mT0ChannelTime = 0xFFFFFFF0;        // 19 bits
   uint32_t mV0Amplitude = 0xFFFFF000;          // 11 bits
   uint32_t mFDDAmplitude = 0xFFFFF000;         // 11 bits
   uint32_t mT0Amplitude = 0xFFFFF000;          // 11 bits
   int mCTPReadout = 0;                         // 0 = use CTP readout from CTP; 1 = create CTP readout
+  bool mCTPConfigPerRun = false;               // 0 = use common CTPconfig as for MC; 1 = run dependent CTP config
   // helper struct for extra info in fillTrackTablesPerCollision()
   struct TrackExtraInfo {
     float tpcInnerParam = 0.f;
     uint32_t flags = 0;
+    uint32_t itsClusterSizes = 0u;
     uint8_t itsClusterMap = 0;
     uint8_t tpcNClsFindable = 0;
     int8_t tpcNClsFindableMinusFound = 0;
+    int8_t tpcNClsFindableMinusPID = 0;
     int8_t tpcNClsFindableMinusCrossedRows = 0;
     uint8_t tpcNClsShared = 0;
     uint8_t trdPattern = 0;
@@ -225,7 +412,38 @@ class AODProducerWorkflowDPL : public Task
     float trackPhiEMCAL = -999.f;
     float trackTime = -999.f;
     float trackTimeRes = -999.f;
+    int diffBCRef = 0; // offset of time reference BC from the start of the orbit
     int bcSlice[2] = {-1, -1};
+    bool isTPConly = false; // not to be written out
+  };
+
+  struct TrackQA {
+    GID trackID;
+    float tpcTime0{};
+    float tpcdEdxNorm{};
+    int16_t tpcdcaR{};
+    int16_t tpcdcaZ{};
+    uint8_t tpcClusterByteMask{};
+    uint8_t tpcdEdxMax0R{};
+    uint8_t tpcdEdxMax1R{};
+    uint8_t tpcdEdxMax2R{};
+    uint8_t tpcdEdxMax3R{};
+    uint8_t tpcdEdxTot0R{};
+    uint8_t tpcdEdxTot1R{};
+    uint8_t tpcdEdxTot2R{};
+    uint8_t tpcdEdxTot3R{};
+    int8_t dRefContY{std::numeric_limits::min()};
+    int8_t dRefContZ{std::numeric_limits::min()};
+    int8_t dRefContSnp{std::numeric_limits::min()};
+    int8_t dRefContTgl{std::numeric_limits::min()};
+    int8_t dRefContQ2Pt{std::numeric_limits::min()};
+    int8_t dRefGloY{std::numeric_limits::min()};
+    int8_t dRefGloZ{std::numeric_limits::min()};
+    int8_t dRefGloSnp{std::numeric_limits::min()};
+    int8_t dRefGloTgl{std::numeric_limits::min()};
+    int8_t dRefGloQ2Pt{std::numeric_limits::min()};
+    int8_t dTofdX{std::numeric_limits::min()};
+    int8_t dTofdZ{std::numeric_limits::min()};
   };
 
   // helper struct for addToFwdTracksTable()
@@ -276,12 +494,18 @@ class AODProducerWorkflowDPL : public Task
   // using -1 as dummies for AOD
   struct MCLabels {
     uint32_t labelID = -1;
-    uint32_t labelITS = -1;
-    uint32_t labelTPC = -1;
     uint16_t labelMask = 0;
     uint8_t fwdLabelMask = 0;
   };
 
+  // counters for TPC clusters
+  struct TPCCounters {
+    uint8_t shared = 0;
+    uint8_t found = 0;
+    uint8_t crossed = 0;
+  };
+  std::vector mTPCCounters;
+
   void updateTimeDependentParams(ProcessingContext& pc);
 
   void addRefGlobalBCsForTOF(const o2::dataformats::VtxTrackRef& trackRef, const gsl::span& GIndices,
@@ -291,33 +515,38 @@ class AODProducerWorkflowDPL : public Task
                   const std::vector& mcRecords,
                   std::map& bcsMap);
 
-  uint64_t getTFNumber(const o2::InteractionRecord& tfStartIR, int runNumber);
   template 
   void addToTracksTable(TracksCursorType& tracksCursor, TracksCovCursorType& tracksCovCursor,
-                        const o2::track::TrackParCov& track, int collisionID);
+                        const o2::track::TrackParCov& track, int collisionID, aod::track::TrackTypeEnum type = aod::track::TrackIU);
 
   template 
   void addToTracksExtraTable(TracksExtraCursorType& tracksExtraCursor, TrackExtraInfo& extraInfoHolder);
 
+  template 
+  void addToTracksQATable(TracksQACursorType& tracksQACursor, TrackQA& trackQAInfoHolder);
+
   template 
   void addToMFTTracksTable(mftTracksCursorType& mftTracksCursor, AmbigMFTTracksCursorType& ambigMFTTracksCursor,
                            GIndex trackID, const o2::globaltracking::RecoContainer& data, int collisionID,
                            std::uint64_t collisionBC, const std::map& bcsMap);
 
-  template 
-  void addToFwdTracksTable(fwdTracksCursorType& fwdTracksCursor, fwdTracksCovCursorType& fwdTracksCovCursor, AmbigFwdTracksCursorType& ambigFwdTracksCursor,
+  template 
+  void addToFwdTracksTable(fwdTracksCursorType& fwdTracksCursor, fwdTracksCovCursorType& fwdTracksCovCursor, AmbigFwdTracksCursorType& ambigFwdTracksCursor, mftTracksCovCursorType& mftTracksCovCursor,
                            GIndex trackID, const o2::globaltracking::RecoContainer& data, int collisionID, std::uint64_t collisionBC, const std::map& bcsMap);
 
   TrackExtraInfo processBarrelTrack(int collisionID, std::uint64_t collisionBC, GIndex trackIndex, const o2::globaltracking::RecoContainer& data, const std::map& bcsMap);
+  TrackQA processBarrelTrackQA(int collisionID, std::uint64_t collisionBC, GIndex trackIndex, const o2::globaltracking::RecoContainer& data, const std::map& bcsMap);
 
+  bool propagateTrackToPV(o2::track::TrackParametrizationWithError& trackPar, const o2::globaltracking::RecoContainer& data, int colID);
+  void extrapolateToCalorimeters(TrackExtraInfo& extraInfoHolder, const o2::track::TrackPar& track);
   void cacheTriggers(const o2::globaltracking::RecoContainer& recoData);
 
   // helper for track tables
   // * fills tables collision by collision
   // * interaction time is for TOF information
-  template 
+  template 
   void fillTrackTablesPerCollision(int collisionID,
                                    std::uint64_t collisionBC,
                                    const o2::dataformats::VtxTrackRef& trackRef,
@@ -326,61 +555,138 @@ class AODProducerWorkflowDPL : public Task
                                    TracksCursorType& tracksCursor,
                                    TracksCovCursorType& tracksCovCursor,
                                    TracksExtraCursorType& tracksExtraCursor,
+                                   TracksQACursorType& tracksQACursor,
                                    AmbigTracksCursorType& ambigTracksCursor,
                                    MFTTracksCursorType& mftTracksCursor,
+                                   MFTTracksCovCursorType& mftTracksCovCursor,
                                    AmbigMFTTracksCursorType& ambigMFTTracksCursor,
                                    FwdTracksCursorType& fwdTracksCursor,
                                    FwdTracksCovCursorType& fwdTracksCovCursor,
                                    AmbigFwdTracksCursorType& ambigFwdTracksCursor,
+                                   FwdTrkClsCursorType& fwdTrkClsCursor,
                                    const std::map& bcsMap);
 
+  template 
+  void addClustersToFwdTrkClsTable(const o2::globaltracking::RecoContainer& recoData, FwdTrkClsCursorType& fwdTrkClsCursor, GIndex trackID, int fwdTrackId);
+
   void fillIndexTablesPerCollision(const o2::dataformats::VtxTrackRef& trackRef, const gsl::span& GIndices, const o2::globaltracking::RecoContainer& data);
 
-  template 
-  void fillSecondaryVertices(const o2::globaltracking::RecoContainer& data, V0CursorType& v0Cursor, CascadeCursorType& cascadeCursor);
+  template 
+  void fillSecondaryVertices(const o2::globaltracking::RecoContainer& data, V0CursorType& v0Cursor, CascadeCursorType& cascadeCursor, Decay3bodyCursorType& decay3bodyCursor);
+
+  template 
+  void fillHMPID(const o2::globaltracking::RecoContainer& recoData, HMPCursorType& hmpCursor);
+
+  void prepareStrangenessTracking(const o2::globaltracking::RecoContainer& recoData);
+  template 
+  void fillStrangenessTrackingTables(const o2::globaltracking::RecoContainer& data, V0C& v0Cursor, CC& cascadeCursor, D3BC& decay3bodyCursor);
+
+  /** some other types we will use */
+  using MCCollisionCursor = aodmchelpers::CollisionCursor;
+  using XSectionCursor = aodmchelpers::XSectionCursor;
+  using PdfInfoCursor = aodmchelpers::PdfInfoCursor;
+  using HeavyIonCursor = aodmchelpers::HeavyIonCursor;
+  using MCParticlesCursor = aodmchelpers::ParticleCursor;
+  using HepMCUpdate = aodmchelpers::HepMCUpdate;
+  using MCEventHeader = dataformats::MCEventHeader;
+  /** Rules for when to update HepMC tables */
+  HepMCUpdate mXSectionUpdate = HepMCUpdate::anyKey;
+  HepMCUpdate mPdfInfoUpdate = HepMCUpdate::anyKey;
+  HepMCUpdate mHeavyIonUpdate = HepMCUpdate::anyKey;
+  /**
+   * Update the header (collision and HepMC aux) information.
+   *
+   * When updating the HepMC aux tables, we take the relevant policies
+   * into account (mXSectionUpdate, mPdfInfoUpdate, mHeavyIonUpdate).
+   *
+   * - If a policy is "never", then the corresponding table is never
+   *   updated.
+   *
+   * - If the policy is "always", then the table is always
+   *   update.
+   *
+   * - If the policy is either "anyKey" or "allKeys", _and_
+   *   this is the first event, then we check if any or all keys,
+   *   respectively are present in the header.
+   *
+   *   - If that check fails, then we do not update and set the
+   *     corresponding policy to be "never".
+   *
+   *   - If the check succeeds, then we do update the table, and set
+   *     the corresponding policty to "always".
+   *
+   *   In this way, we will let the first event decide what to do for
+   *   subsequent events and thus avoid too many string comparisions.
+   *
+   * @param collisionCursor Cursor over aod::McCollisions
+   * @param xSectionCursor Cursor over aod::HepMCXSections
+   * @param pdfInfoCursor Cursor over aod::HepMCPdfInfos
+   * @param heavyIonCursor Cursor over aod::HepMCHeavyIons
+   * @param header Header to read information from
+   * @param collisionID Index of collision in the table
+   * @param bcID Current event identifier (bcID)
+   * @param time Time of event
+   * @param generatorID Generator identifier, if any
+   * @param sourceID Source identifier
+   *
+   */
+  void updateMCHeader(MCCollisionCursor& collisionCursor,
+                      XSectionCursor& xSectionCursor,
+                      PdfInfoCursor& pdfInfoCursor,
+                      HeavyIonCursor& heavyIonCursor,
+                      const MCEventHeader& header,
+                      int collisionID,
+                      int bcID,
+                      float time,
+                      short generatorID,
+                      int sourceID);
 
-  template 
   void fillMCParticlesTable(o2::steer::MCKinematicsReader& mcReader,
-                            const MCParticlesCursorType& mcParticlesCursor,
+                            MCParticlesCursor& mcParticlesCursor,
                             const gsl::span& primVer2TRefs,
                             const gsl::span& GIndices,
                             const o2::globaltracking::RecoContainer& data,
-                            const std::map, int>& mcColToEvSrc);
+                            const std::vector>& mcColToEvSrc);
 
   template 
-  void fillMCTrackLabelsTable(const MCTrackLabelCursorType& mcTrackLabelCursor,
-                              const MCMFTTrackLabelCursorType& mcMFTTrackLabelCursor,
-                              const MCFwdTrackLabelCursorType& mcFwdTrackLabelCursor,
+  void fillMCTrackLabelsTable(MCTrackLabelCursorType& mcTrackLabelCursor,
+                              MCMFTTrackLabelCursorType& mcMFTTrackLabelCursor,
+                              MCFwdTrackLabelCursorType& mcFwdTrackLabelCursor,
                               const o2::dataformats::VtxTrackRef& trackRef,
                               const gsl::span& primVerGIs,
-                              const o2::globaltracking::RecoContainer& data);
+                              const o2::globaltracking::RecoContainer& data,
+                              int vertexId = -1);
 
   std::uint64_t fillBCSlice(int (&slice)[2], double tmin, double tmax, const std::map& bcsMap) const;
 
+  std::vector fillBCFlags(const o2::globaltracking::RecoContainer& data, std::map& bcsMap) const;
+
   // helper for tpc clusters
-  void countTPCClusters(const o2::tpc::TrackTPC& track,
-                        const gsl::span& tpcClusRefs,
-                        const gsl::span& tpcClusShMap,
-                        const o2::tpc::ClusterNativeAccess& tpcClusAcc,
-                        uint8_t& shared, uint8_t& found, uint8_t& crossed);
+  void countTPCClusters(const o2::globaltracking::RecoContainer& data);
 
   // helper for trd pattern
   uint8_t getTRDPattern(const o2::trd::TrackTRD& track);
 
-  template 
-  void fillCaloTable(TEventHandler* caloEventHandler, const TCaloCells& calocells, const TCaloTriggerRecord& caloCellTRGR,
-                     const TCaloCursor& caloCellCursor, const TCaloTRGTableCursor& caloCellTRGTableCursor,
-                     std::map& bcsMap, int8_t caloType);
+  template 
+  void addToCaloTable(TCaloHandler& caloHandler, TCaloCursor& caloCellCursor, TCaloTRGCursor& caloTRGCursor,
+                      TMCCaloLabelCursor& mcCaloCellLabelCursor, int eventID, int bcID, int8_t caloType);
+
+  template 
+  void fillCaloTable(TCaloCursor& caloCellCursor, TCaloTRGCursor& caloTRGCursor,
+                     TMCCaloLabelCursor& mcCaloCellLabelCursor, const std::map& bcsMap,
+                     const o2::globaltracking::RecoContainer& data);
+
+  std::set filterEMCALIncomplete(const gsl::span triggers);
 };
 
 /// create a processor spec
-framework::DataProcessorSpec getAODProducerWorkflowSpec(GID::mask_t src, bool enableSV, bool useMC, std::string resFile);
+framework::DataProcessorSpec getAODProducerWorkflowSpec(GID::mask_t src, bool enableSV, bool enableST, bool useMC, bool CTPConfigPerRun, bool enableFITextra);
 
 // helper interface for calo cells to "befriend" emcal and phos cells
 class CellHelper
 {
  public:
-  static int8_t getTriggerBits(const o2::emcal::Cell& cell)
+  static int8_t getTriggerBits(const o2::emcal::Cell& /*cell*/)
   {
     return 0; // dummy value
   }
@@ -397,6 +703,9 @@ class CellHelper
 
   static int16_t getCellNumber(const o2::phos::Cell& cell)
   {
+    if (cell.getTRU()) {
+      return cell.getTRUId();
+    }
     return cell.getAbsId();
   }
   // If this cell - trigger one?
@@ -410,7 +719,7 @@ class CellHelper
     return cell.getTRU();
   }
 
-  static int16_t getFastOrAbsID(const o2::emcal::Cell& cell)
+  static int16_t getFastOrAbsID(const o2::emcal::Cell& /*cell*/)
   {
     return 0; // dummy value
   }
@@ -430,7 +739,7 @@ class CellHelper
     return cell.getEnergy();
   }
 
-  static int16_t getLnAmplitude(const o2::emcal::Cell& cell)
+  static int16_t getLnAmplitude(const o2::emcal::Cell& /*cell*/)
   {
     return 0; // dummy value
   }
diff --git a/Detectors/AOD/src/AODMcProducerHelpers.cxx b/Detectors/AOD/src/AODMcProducerHelpers.cxx
new file mode 100644
index 0000000000000..a7093e0048c25
--- /dev/null
+++ b/Detectors/AOD/src/AODMcProducerHelpers.cxx
@@ -0,0 +1,436 @@
+// Copyright 2023-2099 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// @file   AODMcProducerHelpers.h
+/// @author Christian Holm Christensen 
+/// common helpers for AOD MC producers
+#include "AODProducerWorkflow/AODMcProducerHelpers.h"
+#include 
+#include 
+#include 
+#include 
+#define verbosity debug
+
+namespace o2::aodmchelpers
+{
+//==================================================================
+bool hasKeys(o2::dataformats::MCEventHeader const& header,
+             const std::vector& keys,
+             bool anyNotAll)
+{
+  auto check = [&header](const std::string& key) { // Do not format
+    return header.hasInfo(key);
+  };
+  return (anyNotAll ? // Do not format
+            std::any_of(keys.cbegin(), keys.cend(), check)
+                    : // Do not format
+            std::all_of(keys.cbegin(), keys.cend(), check));
+}
+//====================================================================
+short updateMCCollisions(const CollisionCursor& cursor,
+                         int bcId,
+                         float time,
+                         o2::dataformats::MCEventHeader const& header,
+                         short generatorId,
+                         int sourceId,
+                         unsigned int mask)
+{
+  using Key = o2::dataformats::MCInfoKeys;
+  using GenProp = o2::mcgenid::GeneratorProperty;
+  using namespace o2::math_utils;
+
+  int subGenId = getEventInfo(header, GenProp::SUBGENERATORID, -1);
+  int genId = getEventInfo(header, GenProp::GENERATORID,
+                           int(generatorId));
+  float weight = getEventInfo(header, Key::weight, 1.f);
+
+  auto encodedGeneratorId = o2::mcgenid::getEncodedGenId(genId,
+                                                         sourceId,
+                                                         subGenId);
+
+  LOG(verbosity) << "Updating MC Collisions table w/bcId=" << bcId;
+  cursor(0,
+         bcId,
+         encodedGeneratorId,
+         truncateFloatFraction(header.GetX(), mask),
+         truncateFloatFraction(header.GetY(), mask),
+         truncateFloatFraction(header.GetZ(), mask),
+         truncateFloatFraction(time, mask),
+         truncateFloatFraction(weight, mask),
+         header.GetB(),
+         getEventInfo(header, Key::planeAngle, header.GetRotZ()));
+  return encodedGeneratorId;
+}
+//--------------------------------------------------------------------
+bool updateHepMCXSection(const XSectionCursor& cursor,
+                         int collisionID,
+                         short generatorID,
+                         o2::dataformats::MCEventHeader const& header,
+                         HepMCUpdate when)
+{
+  using Key = o2::dataformats::MCInfoKeys;
+
+  if (when == HepMCUpdate::never or
+      (when != HepMCUpdate::always and
+       not hasKeys(header, {                      // Do not
+                            Key::acceptedEvents,  // mess with
+                            Key::attemptedEvents, // with
+                            Key::xSection,        // my
+                            Key::xSectionError},  // formatting
+                   when == HepMCUpdate::anyKey))) {
+    return false;
+  }
+
+  LOG(verbosity) << "Updating HepMC cross-section table w/collisionId "
+                 << collisionID;
+  cursor(0,
+         collisionID,
+         generatorID,
+         getEventInfo(header, Key::acceptedEvents, 0),
+         getEventInfo(header, Key::attemptedEvents, 0),
+         getEventInfo(header, Key::xSection, 0.f),
+         getEventInfo(header, Key::xSectionError, 0.f),
+         getEventInfo(header, Key::eventScale, 1.f),
+         getEventInfo(header, Key::mpi, -1),
+         getEventInfo(header, Key::processCode, -1));
+  return true;
+}
+//--------------------------------------------------------------------
+bool updateHepMCPdfInfo(const PdfInfoCursor& cursor,
+                        int collisionID,
+                        short generatorID,
+                        o2::dataformats::MCEventHeader const& header,
+                        HepMCUpdate when)
+{
+  using Key = o2::dataformats::MCInfoKeys;
+
+  if (when == HepMCUpdate::never or
+      (when != HepMCUpdate::always and         // Do
+       not hasKeys(header, {Key::pdfParton1Id, // not
+                            Key::pdfParton2Id, // mess
+                            Key::pdfCode1,     // with
+                            Key::pdfCode2,     // my
+                            Key::pdfX1,        // formatting
+                            Key::pdfX2,        // .
+                            Key::pdfScale,     // It
+                            Key::pdfXF1,       // is
+                            Key::pdfXF2},      // better
+                   when == HepMCUpdate::anyKey))) {
+    return false;
+  }
+
+  LOG(verbosity) << "Updating HepMC PDF table w/collisionId " << collisionID;
+  cursor(0,
+         collisionID,
+         generatorID,
+         getEventInfo(header, Key::pdfParton1Id, int(0)),
+         getEventInfo(header, Key::pdfParton2Id, int(0)),
+         getEventInfo(header, Key::pdfCode1, int(0)),
+         getEventInfo(header, Key::pdfCode2, int(0)),
+         getEventInfo(header, Key::pdfX1, 0.f),
+         getEventInfo(header, Key::pdfX2, 0.f),
+         getEventInfo(header, Key::pdfScale, 1.f),
+         getEventInfo(header, Key::pdfXF1, 0.f),
+         getEventInfo(header, Key::pdfXF2, 0.f));
+
+  return true;
+}
+//--------------------------------------------------------------------
+bool updateHepMCHeavyIon(const HeavyIonCursor& cursor,
+                         int collisionID,
+                         short generatorID,
+                         o2::dataformats::MCEventHeader const& header,
+                         HepMCUpdate when)
+{
+  using Key = dataformats::MCInfoKeys;
+
+  if (when == HepMCUpdate::never or
+      (when != HepMCUpdate::always and                   // clang
+       not hasKeys(header, {Key::nCollHard,              // format
+                            Key::nPartProjectile,        // is
+                            Key::nPartTarget,            // so
+                            Key::nColl,                  // annoying
+                            Key::nCollNNWounded,         // .
+                            Key::nCollNWoundedN,         // It
+                            Key::nCollNWoundedNwounded,  // messes
+                            Key::nSpecProjectileNeutron, // up
+                            Key::nSpecTargetNeutron,     // the
+                            Key::nSpecProjectileProton,  // clarity
+                            Key::nSpecTargetProton,      // of
+                            Key::planeAngle,             // the
+                            "eccentricity",              // code
+                            Key::sigmaInelNN,            // to
+                            Key::centrality},            // noavail
+                   when == HepMCUpdate::anyKey))) {
+    return false;
+  }
+
+  int specNeutrons = (getEventInfo(header, Key::nSpecProjectileNeutron, -1) +
+                      getEventInfo(header, Key::nSpecTargetNeutron, -1));
+  int specProtons = (getEventInfo(header, Key::nSpecProjectileProton, -1) +
+                     getEventInfo(header, Key::nSpecTargetProton, -1));
+
+  LOG(verbosity) << "Updating HepMC heavy-ion table w/collisionID "
+                 << collisionID;
+  cursor(0,
+         collisionID,
+         generatorID,
+         getEventInfo(header, Key::nCollHard, -1),
+         getEventInfo(header, Key::nPartProjectile, -1),
+         getEventInfo(header, Key::nPartTarget, -1),
+         getEventInfo(header, Key::nColl, -1),
+         getEventInfo(header, Key::nCollNNWounded, -1),
+         getEventInfo(header, Key::nCollNWoundedN, -1),
+         getEventInfo(header, Key::nCollNWoundedNwounded, -1),
+         specNeutrons,
+         specProtons,
+         header.GetB(),
+         getEventInfo(header, Key::planeAngle, header.GetRotZ()),
+         getEventInfo(header, "eccentricity", 0),
+         getEventInfo(header, Key::sigmaInelNN, 0.),
+         getEventInfo(header, Key::centrality, -1));
+
+  return true;
+}
+//--------------------------------------------------------------------
+void updateParticle(const ParticleCursor& cursor,
+                    const TrackToIndex& toStore,
+                    int collisionID,
+                    o2::MCTrack const& track,
+                    std::vector const& tracks,
+                    uint8_t flags,
+                    uint32_t weightMask,
+                    uint32_t momentumMask,
+                    uint32_t positionMask)
+{
+  using o2::mcutils::MCTrackNavigator;
+  using namespace o2::aod::mcparticle::enums;
+  using namespace o2::math_utils;
+  using namespace o2::mcgenstatus;
+
+  auto mapping = [&toStore](int trackNo) {
+    auto iter = toStore.find(trackNo);
+    if (iter == toStore.end()) {
+      return -1;
+    }
+    return iter->second;
+  };
+
+  auto statusCode = track.getStatusCode().fullEncoding;
+  auto hepmc = getHepMCStatusCode(track.getStatusCode());
+  if (not track.isPrimary()) {
+    flags |= ProducedByTransport;
+    statusCode = track.getProcess();
+  }
+  if (MCTrackNavigator::isPhysicalPrimary(track, tracks)) {
+    flags |= PhysicalPrimary;
+  }
+
+  int daughters[2] = {-1, -1};
+  std::vector mothers;
+  int id;
+  if ((id = mapping(track.getMotherTrackId())) >= 0) {
+    mothers.push_back(id);
+  }
+  if ((id = mapping(track.getSecondMotherTrackId())) >= 0) {
+    mothers.push_back(id);
+  }
+  if ((id = mapping(track.getFirstDaughterTrackId())) >= 0) {
+    daughters[0] = id;
+  }
+  if ((id = mapping(track.getLastDaughterTrackId())) >= 0) {
+    daughters[1] = id;
+  } else {
+    daughters[1] = daughters[0];
+  }
+  if (daughters[0] < 0 and daughters[1] >= 0) {
+    LOG(error) << "Problematic daughters: " << daughters[0] << " and "
+               << daughters[1];
+    daughters[0] = daughters[1];
+  }
+  if (daughters[0] > daughters[1]) {
+    std::swap(daughters[0], daughters[1]);
+  }
+
+  float weight = track.getWeight();
+  float pX = float(track.Px());
+  float pY = float(track.Py());
+  float pZ = float(track.Pz());
+  float energy = float(track.GetEnergy());
+  float vX = float(track.Vx());
+  float vY = float(track.Vy());
+  float vZ = float(track.Vz());
+  float time = float(track.T());
+
+  LOG(verbosity) << "McParticle collisionId=" << collisionID << ","
+                 << "status=" << statusCode << ","
+                 << "hepmc=" << hepmc << ","
+                 << "pdg=" << track.GetPdgCode() << ","
+                 << "nMothers=" << mothers.size() << ","
+                 << "daughters=" << daughters[0] << ","
+                 << daughters[1];
+
+  cursor(0,
+         collisionID,
+         track.GetPdgCode(),
+         statusCode,
+         flags,
+         mothers,
+         daughters,
+         truncateFloatFraction(weight, weightMask),
+         truncateFloatFraction(pX, momentumMask),
+         truncateFloatFraction(pY, momentumMask),
+         truncateFloatFraction(pZ, momentumMask),
+         truncateFloatFraction(energy, momentumMask),
+         truncateFloatFraction(vX, positionMask),
+         truncateFloatFraction(vY, positionMask),
+         truncateFloatFraction(vZ, positionMask),
+         truncateFloatFraction(time, positionMask));
+}
+//--------------------------------------------------------------------
+uint32_t updateParticles(const ParticleCursor& cursor,
+                         int collisionID,
+                         std::vector const& tracks,
+                         TrackToIndex& preselect,
+                         uint32_t offset,
+                         bool filter,
+                         bool background,
+                         uint32_t weightMask,
+                         uint32_t momentumMask,
+                         uint32_t positionMask,
+                         bool signalFilter)
+{
+  using o2::mcutils::MCTrackNavigator;
+  using namespace o2::aod::mcparticle::enums;
+  using namespace o2::mcgenstatus;
+
+  // First loop over particles to find out which to store
+  // TrackToIndex toStore(preselect.begin(), preselect.end());
+  //
+  // Guess we need to modifiy the passed in mapping so that MC labels
+  // can be set correctly
+  TrackToIndex& toStore = preselect;
+
+  // Mapping lambda.  This maps the track number to the index into
+  // the table exported.
+  auto mapping = [&toStore](int trackNo) {
+    auto iter = toStore.find(trackNo);
+    if (iter == toStore.end()) {
+      return -1;
+    }
+    return iter->second;
+  };
+
+  LOG(verbosity) << "Got a total of " << tracks.size();
+  for (int trackNo = tracks.size() - 1; trackNo >= 0; trackNo--) {
+    auto& track = tracks[trackNo];
+    auto hepmc = getHepMCStatusCode(track.getStatusCode());
+    if (filter) {
+      if (toStore.find(trackNo) == toStore.end() and
+          /* The above test is in-correct.  The track may be stored in
+             the list, but with a negative value.  In that case, the
+             above test will still check mothers and daughters, and
+             possible store them in the output.  This is however how
+             it is done in current `dev` branch, and so to enable
+             comparison on closure, we do this test for now.  The
+             correct way it commented out below. */
+          // mapping(trackNo) < 0 and
+          not track.isPrimary() and
+          not MCTrackNavigator::isPhysicalPrimary(track, tracks) and
+          not MCTrackNavigator::isKeepPhysics(track, tracks)) {
+        LOG(verbosity) << "Skipping track " << trackNo << " " << hepmc << " "
+                       << mapping(trackNo) << " "
+                       << track.isPrimary() << " "
+                       << MCTrackNavigator::isPhysicalPrimary(track, tracks)
+                       << " "
+                       << MCTrackNavigator::isKeepPhysics(track, tracks);
+        continue;
+      }
+    }
+    if (background && signalFilter) {
+      continue;
+    }
+
+    // Store this particle.  We mark that putting a 1 in the
+    // `toStore` mapping. This will later on be updated with the
+    // actual index into the table
+    toStore[trackNo] = 1;
+
+    // If we're filtering tracks, then also mark mothers and
+    // daughters(?) to be stored.
+    if (filter) {
+      int id;
+      if ((id = track.getMotherTrackId()) >= 0) {
+        toStore[id] = 1;
+      }
+      if ((id = track.getSecondMotherTrackId()) >= 0) {
+        toStore[id] = 1;
+      }
+      if ((id = track.getFirstDaughterTrackId()) >= 0) {
+        toStore[id] = 1;
+      }
+      if ((id = track.getLastDaughterTrackId()) >= 0) {
+        toStore[id] = 1;
+      }
+    }
+  }
+
+  // Second loop to set indexes.  This is needed to be done before
+  // we actually update the table, because a particle may point to a
+  // later particle.
+  LOG(verbosity) << "Will store " << toStore.size() << " particles";
+  size_t index = 0;
+
+  for (size_t trackNo = 0U; trackNo < tracks.size(); trackNo++) {
+    auto storeIt = mapping(trackNo);
+    if (storeIt < 0) {
+      continue;
+    }
+
+    toStore[trackNo] = offset + index;
+    index++;
+  }
+
+  // Make sure we have the right number
+  assert(index == toStore.size());
+  LOG(verbosity) << "Starting index " << offset << ", last index "
+                 << offset + index << " "
+                 << "Selected " << toStore.size() << " tracks out of "
+                 << tracks.size() << " "
+                 << "Collision # " << collisionID;
+  // Third loop to actually store the particles in the order given
+  for (size_t trackNo = 0U; trackNo < tracks.size(); trackNo++) {
+    auto storeIt = mapping(trackNo);
+    if (storeIt < 0) {
+      continue;
+    }
+
+    auto& track = tracks[trackNo];
+    auto hepmc = getHepMCStatusCode(track.getStatusCode());
+    uint8_t flags = (background ? FromBackgroundEvent : 0);
+    updateParticle(cursor,
+                   toStore,
+                   collisionID,
+                   track,
+                   tracks,
+                   flags,
+                   weightMask,
+                   momentumMask,
+                   positionMask);
+  }
+  LOG(verbosity) << "Return new offset " << offset + index;
+  return offset + index;
+}
+} // namespace o2::aodmchelpers
+
+// Local Variables:
+//   mode: C++
+// End:
diff --git a/Detectors/AOD/src/AODMcProducerWorkflowSpec.cxx b/Detectors/AOD/src/AODMcProducerWorkflowSpec.cxx
new file mode 100644
index 0000000000000..8136bd3e6d268
--- /dev/null
+++ b/Detectors/AOD/src/AODMcProducerWorkflowSpec.cxx
@@ -0,0 +1,329 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// @file   AODMcProducerWorkflowSpec.cxx
+
+#include "AODProducerWorkflow/AODMcProducerWorkflowSpec.h"
+#include "AODProducerWorkflow/AODProducerHelpers.h"
+#include "Framework/ControlService.h"
+#include "Framework/DataTypes.h"
+#include "SimulationDataFormat/MCUtils.h"
+#include "O2Version.h"
+#include "TString.h"
+
+using namespace o2::framework;
+
+namespace o2::aodmcproducer
+{
+
+//------------------------------------------------------------------
+void AODMcProducerWorkflowDPL::init(InitContext& ic)
+{
+  mTimer.Stop();
+  mLPMProdTag = ic.options().get("lpmp-prod-tag");
+  mAnchorPass = ic.options().get("anchor-pass");
+  mAnchorProd = ic.options().get("anchor-prod");
+  mRecoPass = ic.options().get("reco-pass");
+  mTFNumber = ic.options().get("aod-timeframe-id");
+  mFilterMC = ic.options().get("filter-mctracks");
+  int truncate = ic.options().get("enable-truncation");
+  if (mTFNumber == -1L) {
+    LOG(info) << "TFNumber will be obtained from CCDB";
+  }
+
+  // set no truncation if selected by user
+  if (truncate == 0) {
+    LOG(info) << "Truncation is not used!";
+    mCollisionPosition = 0xFFFFFFFF;
+    mMcParticleW = 0xFFFFFFFF;
+    mMcParticlePos = 0xFFFFFFFF;
+    mMcParticleMom = 0xFFFFFFFF;
+  }
+
+  mEnableEmbed = ic.options().get("enable-embedding");
+
+  if (!mEnableEmbed) {
+    // parse list of sim prefixes into vector
+    mSimPrefix = ic.options().get("mckine-fname");
+  }
+  std::string hepmcUpdate = ic.options().get("hepmc-update");
+  HepMCUpdate when = (hepmcUpdate == "never"           //
+                        ? HepMCUpdate::never           //
+                        : hepmcUpdate == "always"      //
+                            ? HepMCUpdate::always      //
+                            : hepmcUpdate == "all"     //
+                                ? HepMCUpdate::allKeys //
+                                : HepMCUpdate::anyKey);
+  mXSectionUpdate = when;
+  mPdfInfoUpdate = when;
+  mHeavyIonUpdate = when;
+
+  mTimer.Reset();
+}
+
+//------------------------------------------------------------------
+void AODMcProducerWorkflowDPL::updateHeader(CollisionCursor& collisionCursor,
+                                            XSectionCursor& xSectionCursor,
+                                            PdfInfoCursor& pdfInfoCursor,
+                                            HeavyIonCursor& heavyIonCursor,
+                                            const MCEventHeader& header,
+                                            int collisionID, // Index
+                                            int bcID,
+                                            float time,
+                                            short generatorID,
+                                            int sourceID)
+{
+  using aodmchelpers::updateHepMCHeavyIon;
+  using aodmchelpers::updateHepMCPdfInfo;
+  using aodmchelpers::updateHepMCXSection;
+  using aodmchelpers::updateMCCollisions;
+
+  auto genID = updateMCCollisions(collisionCursor,
+                                  bcID,
+                                  time,
+                                  header,
+                                  generatorID,
+                                  sourceID,
+                                  mCollisionPosition);
+  mXSectionUpdate = (updateHepMCXSection(xSectionCursor,  //
+                                         collisionID,     //
+                                         genID,           //
+                                         header,          //
+                                         mXSectionUpdate) //
+                       ? HepMCUpdate::always              //
+                       : HepMCUpdate::never);
+  mPdfInfoUpdate = (updateHepMCPdfInfo(pdfInfoCursor,  //
+                                       collisionID,    //
+                                       genID,          //
+                                       header,         //
+                                       mPdfInfoUpdate) //
+                      ? HepMCUpdate::always            //
+                      : HepMCUpdate::never);
+  mHeavyIonUpdate = (updateHepMCHeavyIon(heavyIonCursor,  //
+                                         collisionID,     //
+                                         genID,           //
+                                         header,          //
+                                         mHeavyIonUpdate) //
+                       ? HepMCUpdate::always              //
+                       : HepMCUpdate::never);
+}
+
+//------------------------------------------------------------------
+void AODMcProducerWorkflowDPL::run(ProcessingContext& pc)
+{
+  mTimer.Start(false);
+
+  uint64_t tfNumber = mTFNumber;
+
+  using namespace o2::aodmchelpers;
+  using namespace o2::aodhelpers;
+
+  auto collisionsCursor = createTableCursor(pc);
+  auto particlesCursor = createTableCursor(pc);
+  auto originCursor = createTableCursor(pc);
+  auto xSectionCursor = createTableCursor(pc);
+  auto pdfInfoCursor = createTableCursor(pc);
+  auto heavyIonCursor = createTableCursor(pc);
+
+  // --- Create our reader -------------------------------------------
+  std::unique_ptr reader;
+  if (not mEnableEmbed) {
+    reader =
+      std::make_unique(mSimPrefix,
+                                           MCKinematicsReader::Mode::kMCKine);
+  } else {
+    reader = std::make_unique("collisioncontext.root");
+  }
+
+  // --- Container of event indexes ---------------------------------
+  using EventInfo = std::vector>;
+  EventInfo eventInfo;
+
+  // --- Fill collision and HepMC aux tables ------------------------
+  // dummy time information
+  float time = 0;
+
+  if (not mEnableEmbed) {
+    // simply store all MC events into table
+    int icol = 0;
+    int nSources = reader->getNSources();
+    for (int isrc = 0; isrc < nSources; isrc++) {
+      short generatorID = isrc;
+      int nEvents = reader->getNEvents(isrc);
+      for (int ievt = 0; ievt < nEvents; ievt++) {
+        auto& header = reader->getMCEventHeader(isrc, ievt);
+        updateHeader(collisionsCursor.cursor,
+                     xSectionCursor.cursor,
+                     pdfInfoCursor.cursor,
+                     heavyIonCursor.cursor,
+                     header,
+                     ievt,
+                     ievt, // BC is the same as collision index
+                     time,
+                     generatorID,
+                     isrc);
+
+        eventInfo.emplace_back(std::make_tuple(icol, isrc, ievt));
+        icol++;
+      }
+    }
+  } else {
+    // treat embedded events using collisioncontext: injected events
+    // will be stored together with background events into the same
+    // collisions
+    int nCollisions = reader->getDigitizationContext()->getNCollisions();
+    const auto& records = reader->getDigitizationContext()->getEventRecords();
+    const auto& parts = reader->getDigitizationContext()->getEventParts();
+    for (int icol = 0; icol < nCollisions; icol++) {
+      auto& colParts = parts[icol];
+      auto nParts = colParts.size();
+      for (auto colPart : colParts) {
+        auto eventID = colPart.entryID;
+        auto sourceID = colPart.sourceID;
+        // enable embedding: if several colParts exist, then they are
+        // saved as one collision
+        if (nParts == 1 || sourceID == 0) {
+          // Make collision header from first source only
+          short generatorID = sourceID;
+          auto& header = reader->getMCEventHeader(sourceID, eventID);
+
+          updateHeader(collisionsCursor.cursor,
+                       xSectionCursor.cursor,
+                       pdfInfoCursor.cursor,
+                       heavyIonCursor.cursor,
+                       header,
+                       icol,
+                       icol, // BC is the same as collision index
+                       time,
+                       generatorID,
+                       sourceID);
+        }
+        // point background and injected signal events to one collision
+        eventInfo.emplace_back(std::make_tuple(icol, sourceID, eventID));
+      }
+    }
+  }
+
+  // Sort the event information
+  std::sort(eventInfo.begin(), eventInfo.end(),
+            [](typename EventInfo::const_reference left,
+               typename EventInfo::const_reference right) { //
+              return (std::get<0>(left) < std::get<0>(right));
+            });
+
+  // Loop over available events and update the tracks table
+  size_t offset = 0;
+  for (auto& colInfo : eventInfo) {
+    int event = std::get<2>(colInfo);
+    int source = std::get<1>(colInfo);
+    int collisionID = std::get<0>(colInfo);
+    auto tracks = reader->getTracks(source, event);
+
+    TrackToIndex preselect;
+    offset = updateParticles(particlesCursor.cursor,
+                             collisionID,
+                             tracks,
+                             preselect,
+                             offset,
+                             mFilterMC,
+                             source == 0,
+                             mMcParticleW,
+                             mMcParticleMom,
+                             mMcParticlePos);
+
+    reader->releaseTracksForSourceAndEvent(source, event);
+  }
+
+  // --- Update the origin and the time-frame ------------------------
+  originCursor(tfNumber);
+
+  // --- Sending metadata to writer if not done already --------------
+  if (not mIsMDSent) {
+    TString o2Version = o2::fullVersion();
+    std::vector metaDataKeys = {"DataType",
+                                         "Run",
+                                         "O2Version",
+                                         "ROOTVersion",
+                                         "RecoPassName",
+                                         "AnchorProduction",
+                                         "AnchorPassName",
+                                         "LPMProductionTag"};
+    std::vector metaDataVals = {"MC",
+                                         "3",
+                                         TString{o2::fullVersion()},
+                                         ROOT_RELEASE,
+                                         mRecoPass,
+                                         mAnchorProd,
+                                         mAnchorPass,
+                                         mLPMProdTag};
+    pc.outputs().snapshot(Output{"AMD", "AODMetadataKeys", 0}, metaDataKeys);
+    pc.outputs().snapshot(Output{"AMD", "AODMetadataVals", 0}, metaDataVals);
+    mIsMDSent = true;
+  }
+
+  pc.outputs().snapshot(Output{"TFN", "TFNumber", 0}, tfNumber);
+  pc.outputs().snapshot(Output{"TFF", "TFFilename", 0}, "");
+
+  pc.services().get().endOfStream();
+  pc.services().get().readyToQuit(QuitRequest::Me);
+
+  mTimer.Stop();
+}
+
+void AODMcProducerWorkflowDPL::endOfStream(EndOfStreamContext&)
+{
+  LOGF(info, "aod producer dpl total timing: Cpu: %.3e Real: %.3e s in %d slots",
+       mTimer.CpuTime(), mTimer.RealTime(), mTimer.Counter() - 1);
+}
+
+DataProcessorSpec getAODMcProducerWorkflowSpec()
+{
+  using McCollisions = AODMcProducerWorkflowDPL::McCollisions;
+  using McParticles = AODMcProducerWorkflowDPL::McParticles;
+  using Origins = AODMcProducerWorkflowDPL::Origins;
+  using XSections = AODMcProducerWorkflowDPL::XSections;
+  using PdfInfos = AODMcProducerWorkflowDPL::PdfInfos;
+  using HeavyIons = AODMcProducerWorkflowDPL::HeavyIons;
+
+  std::vector outputs{
+    OutputForTable::spec(),
+    OutputForTable::spec(),
+    OutputForTable::spec(),
+    OutputForTable::spec(),
+    OutputForTable::spec(),
+    OutputForTable::spec(),
+    OutputSpec{"TFN", "TFNumber"},
+    OutputSpec{"TFF", "TFFilename"},
+    OutputSpec{"AMD", "AODMetadataKeys"},
+    OutputSpec{"AMD", "AODMetadataVals"}};
+
+  return DataProcessorSpec{
+    "aod-mc-producer-workflow",
+    Inputs{},
+    outputs,
+    AlgorithmSpec{adaptFromTask()},
+    Options{
+      ConfigParamSpec{"aod-timeframe-id", VariantType::Int64, 1L, {"Set timeframe number"}},
+      ConfigParamSpec{"enable-truncation", VariantType::Int, 1, {"Truncation parameter: 1 -- on, != 1 -- off"}},
+      ConfigParamSpec{"lpmp-prod-tag", VariantType::String, "", {"LPMProductionTag"}},
+      ConfigParamSpec{"anchor-pass", VariantType::String, "", {"AnchorPassName"}},
+      ConfigParamSpec{"anchor-prod", VariantType::String, "", {"AnchorProduction"}},
+      ConfigParamSpec{"reco-pass", VariantType::String, "", {"RecoPassName"}},
+      ConfigParamSpec{"filter-mctracks", VariantType::Int, 1, {"Store only physical primary MC tracks and their mothers/daughters. 0 -- off, != 0 -- on"}},
+      ConfigParamSpec{"enable-embedding", VariantType::Int, 0, {"Use collisioncontext.root to process embedded events"}},
+      ConfigParamSpec{"mckine-fname", VariantType::String, "o2sim", {"MC kinematics file name prefix: e.g. 'o2sim', 'bkg', 'sgn_1'. Used only if 'enable-embedding' is 0"}},
+      ConfigParamSpec{"hepmc-update", VariantType::String, "always", {"When to update HepMC Aux tables: always - force update, never - never update, all - if all keys are present, any - when any key is present (not valid yet)"}}}};
+}
+
+} // namespace o2::aodmcproducer
+//
+// EOF
+//
diff --git a/Detectors/AOD/src/AODProducerWorkflowSpec.cxx b/Detectors/AOD/src/AODProducerWorkflowSpec.cxx
index 1da3dd9054d8d..852419a9895eb 100644
--- a/Detectors/AOD/src/AODProducerWorkflowSpec.cxx
+++ b/Detectors/AOD/src/AODProducerWorkflowSpec.cxx
@@ -12,26 +12,35 @@
 /// @file   AODProducerWorkflowSpec.cxx
 
 #include "AODProducerWorkflow/AODProducerWorkflowSpec.h"
+#include "AODProducerWorkflow/AODMcProducerHelpers.h"
+#include "AODProducerWorkflow/AODProducerHelpers.h"
+#include "DataFormatsEMCAL/TriggerRecord.h"
+#include "DataFormatsEMCAL/EventHandler.h"
 #include "DataFormatsFT0/RecPoints.h"
 #include "DataFormatsFDD/RecPoint.h"
+#include "DataFormatsFV0/RecPoints.h"
 #include "DataFormatsGlobalTracking/RecoContainer.h"
 #include "DataFormatsCTP/Digits.h"
 #include "DataFormatsCTP/Configuration.h"
+#include "DataFormatsCPV/Cluster.h"
+#include "DataFormatsCPV/TriggerRecord.h"
 #include "DataFormatsITS/TrackITS.h"
 #include "DataFormatsMCH/ROFRecord.h"
 #include "DataFormatsMCH/TrackMCH.h"
 #include "DataFormatsMCH/Cluster.h"
 #include "DataFormatsMID/Track.h"
 #include "DataFormatsMFT/TrackMFT.h"
+#include "DataFormatsPHOS/TriggerRecord.h"
+#include "DataFormatsPHOS/EventHandler.h"
 #include "DataFormatsTPC/TrackTPC.h"
 #include "DataFormatsTRD/TriggerRecord.h"
+#include "DataFormatsZDC/BCRecData.h"
 #include "DataFormatsZDC/ZDCEnergy.h"
 #include "DataFormatsZDC/ZDCTDCData.h"
 #include "DataFormatsParameters/GRPECSObject.h"
-#include "CommonUtils/NameConf.h"
 #include "MathUtils/Utils.h"
-#include "DetectorsBase/GeometryManager.h"
 #include "CCDB/BasicCCDBManager.h"
+#include "CommonConstants/Triggers.h"
 #include "CommonConstants/PhysicsConstants.h"
 #include "CommonDataFormat/InteractionRecord.h"
 #include "DataFormatsTRD/TrackTRD.h"
@@ -40,25 +49,25 @@
 #include "Framework/AnalysisDataModel.h"
 #include "Framework/ConfigParamRegistry.h"
 #include "Framework/DataTypes.h"
-#include "Framework/InputRecordWalker.h"
-#include "Framework/Logger.h"
 #include "Framework/TableBuilder.h"
-#include "Framework/TableTreeHelpers.h"
 #include "Framework/CCDBParamSpec.h"
-#include "FDDBase/Constants.h"
+#include "CommonUtils/TreeStreamRedirector.h"
 #include "FT0Base/Geometry.h"
-#include "FV0Base/Geometry.h"
 #include "GlobalTracking/MatchTOF.h"
 #include "ReconstructionDataFormats/Cascade.h"
+#include "GlobalTracking/MatchGlobalFwd.h"
 #include "MCHTracking/TrackExtrap.h"
 #include "MCHTracking/TrackParam.h"
 #include "ITSMFTBase/DPLAlpideParam.h"
 #include "DetectorsVertexing/PVertexerParams.h"
+#include "ReconstructionDataFormats/GlobalFwdTrack.h"
 #include "ReconstructionDataFormats/GlobalTrackID.h"
+#include "ReconstructionDataFormats/PrimaryVertex.h"
+#include "ReconstructionDataFormats/StrangeTrack.h"
 #include "ReconstructionDataFormats/Track.h"
 #include "ReconstructionDataFormats/TrackTPCITS.h"
 #include "ReconstructionDataFormats/TrackMCHMID.h"
-#include "ReconstructionDataFormats/GlobalFwdTrack.h"
+#include "ReconstructionDataFormats/MatchInfoHMP.h"
 #include "ReconstructionDataFormats/V0.h"
 #include "ReconstructionDataFormats/VtxTrackIndex.h"
 #include "ReconstructionDataFormats/VtxTrackRef.h"
@@ -67,6 +76,7 @@
 #include "SimulationDataFormat/MCTrack.h"
 #include "SimulationDataFormat/MCTruthContainer.h"
 #include "SimulationDataFormat/MCUtils.h"
+#include "SimulationDataFormat/MCGenProperties.h"
 #include "ZDCBase/Constants.h"
 #include "TPCBase/ParameterElectronics.h"
 #include "GPUTPCGMMergedTrackHit.h"
@@ -75,13 +85,25 @@
 #include "TMath.h"
 #include "MathUtils/Utils.h"
 #include "Math/SMatrix.h"
-#include "TMatrixD.h"
 #include "TString.h"
-#include "TObjString.h"
+#include 
 #include 
+#include 
+#include 
 #include 
+#include 
 #include 
 #include 
+#include 
+#include "TLorentzVector.h"
+#include "TVector3.h"
+#include "MathUtils/Tsallis.h"
+#include 
+#ifdef WITH_OPENMP
+#include 
+#endif
+#include 
+#include 
 
 using namespace o2::framework;
 using namespace o2::math_utils::detail;
@@ -89,6 +111,7 @@ using PVertex = o2::dataformats::PrimaryVertex;
 using GIndex = o2::dataformats::VtxTrackIndex;
 using DataRequest = o2::globaltracking::DataRequest;
 using GID = o2::dataformats::GlobalTrackID;
+using DetID = o2::detectors::DetID;
 using SMatrix55Sym = ROOT::Math::SMatrix>;
 
 namespace o2::aodproducer
@@ -98,50 +121,98 @@ void AODProducerWorkflowDPL::createCTPReadout(const o2::globaltracking::RecoCont
 {
   // Extraxt CTP Config from CCDB
   const auto ctpcfg = pc.inputs().get("ctpconfig");
+  ctpcfg->printStream(std::cout);
   // o2::ctp::CTPConfiguration ctpcfg = o2::ctp::CTPRunManager::getConfigFromCCDB(-1, std::to_string(runNumber)); // how to get run
   //  Extract inputs from recoData
-  std::map bcsMapT0triggers;
-  std::map bcsMapTRDreadout;
+  uint64_t classMaskEMCAL = 0, classMaskTRD = 0, classMaskPHOSCPV = 0;
+  for (const auto& trgclass : ctpcfg->getCTPClasses()) {
+    if (trgclass.cluster->getClusterDetNames().find("EMC") != std::string::npos) {
+      classMaskEMCAL = trgclass.classMask;
+    }
+    if (trgclass.cluster->getClusterDetNames().find("PHS") != std::string::npos) {
+      classMaskPHOSCPV = trgclass.classMask;
+    }
+    if (trgclass.cluster->getClusterDetNames().find("TRD") != std::string::npos) {
+      classMaskTRD = trgclass.classMask;
+    }
+  }
+  LOG(info) << "createCTPReadout: Class Mask EMCAL -> " << classMaskEMCAL;
+  LOG(info) << "createCTPReadout: Class Mask PHOS/CPV -> " << classMaskPHOSCPV;
+  LOG(info) << "createCTPReadout: Class Mask TRD -> " << classMaskTRD;
+
   // const auto& fddRecPoints = recoData.getFDDRecPoints();
   // const auto& fv0RecPoints = recoData.getFV0RecPoints();
-  const auto& caloEMCCellsTRGR = recoData.getEMCALTriggers();
-  const auto& caloPHOSCellsTRGR = recoData.getPHOSTriggers();
+  const auto& triggerrecordEMCAL = recoData.getEMCALTriggers();
+  const auto& triggerrecordPHOSCPV = recoData.getPHOSTriggers();
   const auto& triggerrecordTRD = recoData.getTRDTriggerRecords();
+  // For EMCAL filter remove calibration triggers
+  std::vector triggerRecordEMCALPhys;
+  for (const auto& trg : triggerrecordEMCAL) {
+    if (trg.getTriggerBits() & o2::trigger::Cal) {
+      continue;
+    }
+    triggerRecordEMCALPhys.push_back(trg);
+  }
+  // const auto& triggerrecordTRD =recoData.getITSTPCTRDTriggers()
   //
+
+  // Find TVX triggers, only TRD/EMCAL/PHOS/CPV triggers in coincidence will be accepted
+  std::set bcsMapT0triggers;
   const auto& ft0RecPoints = recoData.getFT0RecPoints();
   for (auto& ft0RecPoint : ft0RecPoints) {
     auto t0triggers = ft0RecPoint.getTrigger();
     if (t0triggers.getVertex()) {
       uint64_t globalBC = ft0RecPoint.getInteractionRecord().toLong();
-      uint64_t classmask = ctpcfg->getClassMaskForInput("MTVX");
-      bcsMapT0triggers[globalBC] = classmask;
-    }
-  }
-  // find trd redaout and add CTPDigit if trigger there
-  int cntwarnings = 0;
-  uint32_t orbitPrev = 0;
-  uint16_t bcPrev = 0;
-  for (auto& trdrec : triggerrecordTRD) {
-    auto orbitPrevT = orbitPrev;
-    auto bcPrevT = bcPrev;
-    bcPrev = trdrec.getBCData().bc;
-    orbitPrev = trdrec.getBCData().orbit;
-    if (orbitPrev < orbitPrevT || bcPrev >= o2::constants::lhc::LHCMaxBunches || (orbitPrev == orbitPrevT && bcPrev < bcPrevT)) {
-      cntwarnings++;
-      // LOGP(warning, "Bogus TRD trigger at bc:{}/orbit:{} (previous was {}/{}), with {} tracklets and {} digits",bcPrev, orbitPrev, bcPrevT, orbitPrevT, trig.getNumberOfTracklets(), trig.getNumberOfDigits());
-    } else {
-      uint64_t globalBC = trdrec.getBCData().toLong();
-      auto t0entry = bcsMapT0triggers.find(globalBC);
-      if (t0entry != bcsMapT0triggers.end()) {
-        auto& ctpdig = ctpDigits.emplace_back();
-        ctpdig.intRecord.setFromLong(globalBC);
-        ctpdig.CTPClassMask = t0entry->second;
+      bcsMapT0triggers.insert(globalBC);
+    }
+  }
+
+  auto genericCTPDigitizer = [&bcsMapT0triggers, &ctpDigits](auto triggerrecords, uint64_t classmask) -> int {
+    // Strategy:
+    // find detector trigger based on trigger record from readout and add CTPDigit if trigger there
+    int cntwarnings = 0;
+    uint32_t orbitPrev = 0;
+    uint16_t bcPrev = 0;
+    for (auto& trigger : triggerrecords) {
+      auto orbitPrevT = orbitPrev;
+      auto bcPrevT = bcPrev;
+      bcPrev = trigger.getBCData().bc;
+      orbitPrev = trigger.getBCData().orbit;
+      // dedicated for TRD: remove bogus triggers
+      if (orbitPrev < orbitPrevT || bcPrev >= o2::constants::lhc::LHCMaxBunches || (orbitPrev == orbitPrevT && bcPrev < bcPrevT)) {
+        cntwarnings++;
+        // LOGP(warning, "Bogus TRD trigger at bc:{}/orbit:{} (previous was {}/{}), with {} tracklets and {} digits",bcPrev, orbitPrev, bcPrevT, orbitPrevT, trig.getNumberOfTracklets(), trig.getNumberOfDigits());
       } else {
-        LOG(warning) << "Found trd and no MTVX:" << globalBC;
+        uint64_t globalBC = trigger.getBCData().toLong();
+        auto t0entry = bcsMapT0triggers.find(globalBC);
+        if (t0entry != bcsMapT0triggers.end()) {
+          auto ctpdig = std::find_if(ctpDigits.begin(), ctpDigits.end(), [globalBC](const o2::ctp::CTPDigit& dig) { return static_cast(dig.intRecord.toLong()) == globalBC; });
+          if (ctpdig != ctpDigits.end()) {
+            // CTP digit existing from other trigger, merge detector class mask
+            ctpdig->CTPClassMask |= std::bitset<64>(classmask);
+            LOG(debug) << "createCTPReadout: Merging " << classmask << " CTP digits with existing digit, CTP mask " << ctpdig->CTPClassMask;
+          } else {
+            // New CTP digit needed
+            LOG(debug) << "createCTPReadout: New CTP digit needed for class " << classmask << std::endl;
+            auto& ctpdigNew = ctpDigits.emplace_back();
+            ctpdigNew.intRecord.setFromLong(globalBC);
+            ctpdigNew.CTPClassMask = classmask;
+          }
+        } else {
+          LOG(warning) << "createCTPReadout: Found " << classmask << " and no MTVX:" << globalBC;
+        }
       }
     }
-  }
-  LOG(info) << "# of TRD bogus triggers:" << cntwarnings;
+    return cntwarnings;
+  };
+
+  auto warningsTRD = genericCTPDigitizer(triggerrecordTRD, classMaskTRD);
+  auto warningsEMCAL = genericCTPDigitizer(triggerRecordEMCALPhys, classMaskEMCAL);
+  auto warningsPHOSCPV = genericCTPDigitizer(triggerrecordPHOSCPV, classMaskPHOSCPV);
+
+  LOG(info) << "createCTPReadout:# of TRD bogus triggers:" << warningsTRD;
+  LOG(info) << "createCTPReadout:# of EMCAL bogus triggers:" << warningsEMCAL;
+  LOG(info) << "createCTPReadout:# of PHOS/CPV bogus triggers:" << warningsPHOSCPV;
 }
 
 void AODProducerWorkflowDPL::collectBCs(const o2::globaltracking::RecoContainer& data,
@@ -154,6 +225,7 @@ void AODProducerWorkflowDPL::collectBCs(const o2::globaltracking::RecoContainer&
   const auto& fv0RecPoints = data.getFV0RecPoints();
   const auto& caloEMCCellsTRGR = data.getEMCALTriggers();
   const auto& caloPHOSCellsTRGR = data.getPHOSTriggers();
+  const auto& cpvTRGR = data.getCPVTriggers();
   const auto& ctpDigits = data.getCTPDigits();
   const auto& zdcBCRecData = data.getZDCBCRecData();
 
@@ -202,6 +274,11 @@ void AODProducerWorkflowDPL::collectBCs(const o2::globaltracking::RecoContainer&
     bcsMap[globalBC] = 1;
   }
 
+  for (auto& cpvtrg : cpvTRGR) {
+    uint64_t globalBC = cpvtrg.getBCData().toLong();
+    bcsMap[globalBC] = 1;
+  }
+
   for (auto& ctpDigit : ctpDigits) {
     uint64_t globalBC = ctpDigit.intRecord.toLong();
     bcsMap[globalBC] = 1;
@@ -214,44 +291,12 @@ void AODProducerWorkflowDPL::collectBCs(const o2::globaltracking::RecoContainer&
   }
 }
 
-uint64_t AODProducerWorkflowDPL::getTFNumber(const o2::InteractionRecord& tfStartIR, int runNumber)
-{
-  auto& mgr = o2::ccdb::BasicCCDBManager::instance();
-  o2::ccdb::CcdbApi ccdb_api;
-  const std::string rct_path = "RCT/Info/RunInformation/";
-  const std::string start_orbit_path = "Trigger/StartOrbit";
-
-  mgr.setURL(o2::base::NameConf::getCCDBServer());
-  ccdb_api.init(o2::base::NameConf::getCCDBServer());
-
-  std::map* mapStartOrbit = mgr.get>(start_orbit_path);
-  int64_t ts = 0;
-  std::map metadata;
-  std::map headers;
-  const std::string run_path = Form("%s/%i", rct_path.data(), runNumber);
-  headers = ccdb_api.retrieveHeaders(run_path, metadata, -1);
-  ts = atol(headers["SOR"].c_str());
-
-  // ccdb returns timestamp in mus
-  // mus to ms
-  ts = ts / 1000;
-
-  uint32_t initialOrbit = mapStartOrbit->at(runNumber);
-  uint16_t firstRecBC = tfStartIR.bc;
-  uint32_t firstRecOrbit = tfStartIR.orbit;
-  const o2::InteractionRecord firstRec(firstRecBC, firstRecOrbit);
-  ts += firstRec.bc2ns() / 1000000;
-
-  return ts;
-};
-
 template 
 void AODProducerWorkflowDPL::addToTracksTable(TracksCursorType& tracksCursor, TracksCovCursorType& tracksCovCursor,
-                                              const o2::track::TrackParCov& track, int collisionID)
+                                              const o2::track::TrackParCov& track, int collisionID, aod::track::TrackTypeEnum type)
 {
-  tracksCursor(0,
-               collisionID,
-               o2::aod::track::Track,
+  tracksCursor(collisionID,
+               type,
                truncateFloatFraction(track.getX(), mTrackX),
                truncateFloatFraction(track.getAlpha(), mTrackAlpha),
                track.getY(),
@@ -262,8 +307,7 @@ void AODProducerWorkflowDPL::addToTracksTable(TracksCursorType& tracksCursor, Tr
   // trackscov
   float sY = TMath::Sqrt(track.getSigmaY2()), sZ = TMath::Sqrt(track.getSigmaZ2()), sSnp = TMath::Sqrt(track.getSigmaSnp2()),
         sTgl = TMath::Sqrt(track.getSigmaTgl2()), sQ2Pt = TMath::Sqrt(track.getSigma1Pt2());
-  tracksCovCursor(0,
-                  truncateFloatFraction(sY, mTrackCovDiag),
+  tracksCovCursor(truncateFloatFraction(sY, mTrackCovDiag),
                   truncateFloatFraction(sZ, mTrackCovDiag),
                   truncateFloatFraction(sSnp, mTrackCovDiag),
                   truncateFloatFraction(sTgl, mTrackCovDiag),
@@ -283,13 +327,20 @@ void AODProducerWorkflowDPL::addToTracksTable(TracksCursorType& tracksCursor, Tr
 template 
 void AODProducerWorkflowDPL::addToTracksExtraTable(TracksExtraCursorType& tracksExtraCursor, TrackExtraInfo& extraInfoHolder)
 {
+  // In case of TPC-only tracks, do not truncate the time error since we encapsulate there a special encoding of
+  // the deltaFwd/Bwd times
+  auto trackTimeRes = extraInfoHolder.trackTimeRes;
+  if (!extraInfoHolder.isTPConly) {
+    trackTimeRes = truncateFloatFraction(trackTimeRes, mTrackTimeError);
+  }
+
   // extra
-  tracksExtraCursor(0,
-                    truncateFloatFraction(extraInfoHolder.tpcInnerParam, mTrack1Pt),
+  tracksExtraCursor(truncateFloatFraction(extraInfoHolder.tpcInnerParam, mTrack1Pt),
                     extraInfoHolder.flags,
-                    extraInfoHolder.itsClusterMap,
+                    extraInfoHolder.itsClusterSizes,
                     extraInfoHolder.tpcNClsFindable,
                     extraInfoHolder.tpcNClsFindableMinusFound,
+                    extraInfoHolder.tpcNClsFindableMinusPID,
                     extraInfoHolder.tpcNClsFindableMinusCrossedRows,
                     extraInfoHolder.tpcNClsShared,
                     extraInfoHolder.trdPattern,
@@ -304,7 +355,39 @@ void AODProducerWorkflowDPL::addToTracksExtraTable(TracksExtraCursorType& tracks
                     truncateFloatFraction(extraInfoHolder.trackEtaEMCAL, mTrackPosEMCAL),
                     truncateFloatFraction(extraInfoHolder.trackPhiEMCAL, mTrackPosEMCAL),
                     truncateFloatFraction(extraInfoHolder.trackTime, mTrackTime),
-                    truncateFloatFraction(extraInfoHolder.trackTimeRes, mTrackTimeError));
+                    trackTimeRes);
+}
+
+template 
+void AODProducerWorkflowDPL::addToTracksQATable(TracksQACursorType& tracksQACursor, TrackQA& trackQAInfoHolder)
+{
+  tracksQACursor(
+    trackQAInfoHolder.trackID,
+    mTrackQCRetainOnlydEdx ? 0.0f : truncateFloatFraction(trackQAInfoHolder.tpcTime0, mTPCTime0),
+    truncateFloatFraction(trackQAInfoHolder.tpcdEdxNorm, mTrackSignal),
+    mTrackQCRetainOnlydEdx ? std::numeric_limits::min() : trackQAInfoHolder.tpcdcaR,
+    mTrackQCRetainOnlydEdx ? std::numeric_limits::min() : trackQAInfoHolder.tpcdcaZ,
+    trackQAInfoHolder.tpcClusterByteMask,
+    trackQAInfoHolder.tpcdEdxMax0R,
+    trackQAInfoHolder.tpcdEdxMax1R,
+    trackQAInfoHolder.tpcdEdxMax2R,
+    trackQAInfoHolder.tpcdEdxMax3R,
+    trackQAInfoHolder.tpcdEdxTot0R,
+    trackQAInfoHolder.tpcdEdxTot1R,
+    trackQAInfoHolder.tpcdEdxTot2R,
+    trackQAInfoHolder.tpcdEdxTot3R,
+    mTrackQCRetainOnlydEdx ? std::numeric_limits::min() : trackQAInfoHolder.dRefContY,
+    mTrackQCRetainOnlydEdx ? std::numeric_limits::min() : trackQAInfoHolder.dRefContZ,
+    mTrackQCRetainOnlydEdx ? std::numeric_limits::min() : trackQAInfoHolder.dRefContSnp,
+    mTrackQCRetainOnlydEdx ? std::numeric_limits::min() : trackQAInfoHolder.dRefContTgl,
+    mTrackQCRetainOnlydEdx ? std::numeric_limits::min() : trackQAInfoHolder.dRefContQ2Pt,
+    mTrackQCRetainOnlydEdx ? std::numeric_limits::min() : trackQAInfoHolder.dRefGloY,
+    mTrackQCRetainOnlydEdx ? std::numeric_limits::min() : trackQAInfoHolder.dRefGloZ,
+    mTrackQCRetainOnlydEdx ? std::numeric_limits::min() : trackQAInfoHolder.dRefGloSnp,
+    mTrackQCRetainOnlydEdx ? std::numeric_limits::min() : trackQAInfoHolder.dRefGloTgl,
+    mTrackQCRetainOnlydEdx ? std::numeric_limits::min() : trackQAInfoHolder.dRefGloQ2Pt,
+    mTrackQCRetainOnlydEdx ? std::numeric_limits::min() : trackQAInfoHolder.dTofdX,
+    mTrackQCRetainOnlydEdx ? std::numeric_limits::min() : trackQAInfoHolder.dTofdZ);
 }
 
 template 
@@ -316,9 +399,9 @@ void AODProducerWorkflowDPL::addToMFTTracksTable(mftTracksCursorType& mftTracksC
   int bcSlice[2] = {-1, -1};
   const auto& track = data.getMFTTrack(trackID);
   const auto& rof = data.getMFTTracksROFRecords()[mMFTROFs[trackID.getIndex()]];
-  float trackTime = rof.getBCData().differenceInBC(mStartIR) * o2::constants::lhc::LHCBunchSpacingNS + mMFTROFrameHalfLengthNS;
+  float trackTime = rof.getBCData().differenceInBC(mStartIR) * o2::constants::lhc::LHCBunchSpacingNS + mMFTROFrameHalfLengthNS + mMFTROFBiasNS;
   float trackTimeRes = mMFTROFrameHalfLengthNS;
-  bool needBCSlice = collisionID < 0 || trackID.isAmbiguous();
+  bool needBCSlice = collisionID < 0;
   std::uint64_t bcOfTimeRef;
   if (needBCSlice) {
     double error = mTimeMarginTrackTime + trackTimeRes;
@@ -328,26 +411,29 @@ void AODProducerWorkflowDPL::addToMFTTracksTable(mftTracksCursorType& mftTracksC
   }
   trackTime -= bcOfTimeRef * o2::constants::lhc::LHCBunchSpacingNS;
 
-  mftTracksCursor(0,
-                  collisionID,
+  // the Cellular Automaton track-finding algorithm flag is stored in first of the 4 bits not used for the cluster size
+  uint64_t mftClusterSizesAndTrackFlags = track.getClusterSizes();
+  mftClusterSizesAndTrackFlags |= (track.isCA()) ? (1ULL << (60)) : 0;
+
+  mftTracksCursor(collisionID,
                   track.getX(),
                   track.getY(),
                   truncateFloatFraction(track.getZ(), mTrackX), // for the forward tracks Z has the same role as X in barrel
                   truncateFloatFraction(track.getPhi(), mTrackAlpha),
                   truncateFloatFraction(track.getTanl(), mTrackTgl),
                   truncateFloatFraction(track.getInvQPt(), mTrack1Pt),
-                  track.getNumberOfPoints(),
+                  mftClusterSizesAndTrackFlags,
                   truncateFloatFraction(track.getTrackChi2(), mTrackChi2),
                   truncateFloatFraction(trackTime, mTrackTime),
                   truncateFloatFraction(trackTimeRes, mTrackTimeError));
   if (needBCSlice) {
-    ambigMFTTracksCursor(0, mTableTrMFTID, bcSlice);
+    ambigMFTTracksCursor(mTableTrMFTID, bcSlice);
   }
 }
 
-template 
+template 
 void AODProducerWorkflowDPL::fillTrackTablesPerCollision(int collisionID,
                                                          std::uint64_t collisionBC,
                                                          const o2::dataformats::VtxTrackRef& trackRef,
@@ -356,12 +442,15 @@ void AODProducerWorkflowDPL::fillTrackTablesPerCollision(int collisionID,
                                                          TracksCursorType& tracksCursor,
                                                          TracksCovCursorType& tracksCovCursor,
                                                          TracksExtraCursorType& tracksExtraCursor,
+                                                         TracksQACursorType& tracksQACursor,
                                                          AmbigTracksCursorType& ambigTracksCursor,
                                                          MFTTracksCursorType& mftTracksCursor,
+                                                         MFTTracksCovCursorType& mftTracksCovCursor,
                                                          AmbigMFTTracksCursorType& ambigMFTTracksCursor,
                                                          FwdTracksCursorType& fwdTracksCursor,
                                                          FwdTracksCovCursorType& fwdTracksCovCursor,
                                                          AmbigFwdTracksCursorType& ambigFwdTracksCursor,
+                                                         FwdTrkClsCursorType& fwdTrkClsCursor,
                                                          const std::map& bcsMap)
 {
   for (int src = GIndex::NSources; src--;) {
@@ -370,8 +459,22 @@ void AODProducerWorkflowDPL::fillTrackTablesPerCollision(int collisionID,
     }
     int start = trackRef.getFirstEntryOfSource(src);
     int end = start + trackRef.getEntriesOfSource(src);
+    int nToReserve = end - start; // + last index for a given table
+    if (src == GIndex::Source::MFT) {
+      mftTracksCursor.reserve(nToReserve + mftTracksCursor.lastIndex());
+    } else if (src == GIndex::Source::MCH || src == GIndex::Source::MFTMCH || src == GIndex::Source::MCHMID) {
+      fwdTracksCursor.reserve(nToReserve + fwdTracksCursor.lastIndex());
+      fwdTracksCovCursor.reserve(nToReserve + fwdTracksCovCursor.lastIndex());
+      if (src == GIndex::Source::MFTMCH) {
+        mftTracksCovCursor.reserve(nToReserve + mftTracksCovCursor.lastIndex());
+      }
+    } else {
+      tracksCursor.reserve(nToReserve + tracksCursor.lastIndex());
+      tracksCovCursor.reserve(nToReserve + tracksCovCursor.lastIndex());
+      tracksExtraCursor.reserve(nToReserve + tracksExtraCursor.lastIndex());
+    }
     for (int ti = start; ti < end; ti++) {
-      auto& trackIndex = GIndices[ti];
+      const auto& trackIndex = GIndices[ti];
       if (GIndex::includesSource(src, mInputSources)) {
         if (src == GIndex::Source::MFT) {                                                                // MFT tracks are treated separately since they are stored in a different table
           if (trackIndex.isAmbiguous() && mGIDToTableMFTID.find(trackIndex) != mGIDToTableMFTID.end()) { // was it already stored ?
@@ -384,25 +487,76 @@ void AODProducerWorkflowDPL::fillTrackTablesPerCollision(int collisionID,
           if (trackIndex.isAmbiguous() && mGIDToTableFwdID.find(trackIndex) != mGIDToTableFwdID.end()) {           // was it already stored ?
             continue;
           }
-          addToFwdTracksTable(fwdTracksCursor, fwdTracksCovCursor, ambigFwdTracksCursor, trackIndex, data, collisionID, collisionBC, bcsMap);
+          addToFwdTracksTable(fwdTracksCursor, fwdTracksCovCursor, ambigFwdTracksCursor, mftTracksCovCursor, trackIndex, data, collisionID, collisionBC, bcsMap);
           mGIDToTableFwdID.emplace(trackIndex, mTableTrFwdID);
+          addClustersToFwdTrkClsTable(data, fwdTrkClsCursor, trackIndex, mTableTrFwdID);
           mTableTrFwdID++;
         } else {
           // barrel track: normal tracks table
           if (trackIndex.isAmbiguous() && mGIDToTableID.find(trackIndex) != mGIDToTableID.end()) { // was it already stored ?
             continue;
           }
+
+          float weight = 0;
+          static std::uniform_real_distribution<> distr(0., 1.);
+          bool writeQAData = o2::math_utils::Tsallis::downsampleTsallisCharged(data.getTrackParam(trackIndex).getPt(), mTrackQCFraction, mSqrtS, weight, distr(mGenerator)) || ((src != GIndex::TPC || mGIDUsedBySVtx.find(trackIndex) != mGIDUsedBySVtx.end() || mGIDUsedByStr.find(trackIndex) != mGIDUsedByStr.end()) && mTrackQCKeepGlobalTracks);
           auto extraInfoHolder = processBarrelTrack(collisionID, collisionBC, trackIndex, data, bcsMap);
-          if (extraInfoHolder.trackTimeRes < 0.f) { // failed or rejected?
+
+          if (writeQAData) {
+            auto trackQAInfoHolder = processBarrelTrackQA(collisionID, collisionBC, trackIndex, data, bcsMap);
+            if (std::bitset<8>(trackQAInfoHolder.tpcClusterByteMask).count() >= mTrackQCNTrCut) {
+              trackQAInfoHolder.trackID = mTableTrID;
+              // LOGP(info, "orig time0 in bc: {} diffBCRef: {}, ttime: {} -> {}", trackQAInfoHolder.tpcTime0*8, extraInfoHolder.diffBCRef, extraInfoHolder.trackTime, (trackQAInfoHolder.tpcTime0 * 8 - extraInfoHolder.diffBCRef) * o2::constants::lhc::LHCBunchSpacingNS - extraInfoHolder.trackTime);
+              trackQAInfoHolder.tpcTime0 = (trackQAInfoHolder.tpcTime0 * 8 - extraInfoHolder.diffBCRef) * o2::constants::lhc::LHCBunchSpacingNS - extraInfoHolder.trackTime;
+              // difference between TPC track time0 and stored track nominal time in ns instead of TF start
+              addToTracksQATable(tracksQACursor, trackQAInfoHolder);
+            } else {
+              writeQAData = false;
+            }
+          }
+
+          // include specific selection of tpc standalone tracks if thinning is active
+          if (mThinTracks && extraInfoHolder.isTPConly && !writeQAData) { // if trackQA is written then no check has to be done
+            auto trk = data.getTPCTrack(trackIndex);
+            if (trk.getNClusters() >= mTrackQCNCls && trk.getPt() >= mTrackQCPt) {
+              o2::dataformats::DCA dcaInfo{999.f, 999.f, 999.f, 999.f, 999.f};
+              o2::dataformats::VertexBase v = mVtx.getMeanVertex(collisionID < 0 ? 0.f : data.getPrimaryVertex(collisionID).getZ());
+              if (o2::base::Propagator::Instance()->propagateToDCABxByBz(v, trk, 2., mMatCorr, &dcaInfo) && std::abs(dcaInfo.getY()) < mTrackQCDCAxy) {
+                writeQAData = true; // just setting this to not thin the track
+              }
+            }
+          }
+
+          // Skip thinning if not enabled or track is not tpc standalone or assoc. to a V0 or qa'ed
+          if (mThinTracks && src == GIndex::Source::TPC && mGIDUsedBySVtx.find(trackIndex) == mGIDUsedBySVtx.end() && mGIDUsedByStr.find(trackIndex) == mGIDUsedByStr.end() && !writeQAData) {
+            mGIDToTableID.emplace(trackIndex, -1); // pretend skipped tracks are stored; this is safe since they are are not written to disk and -1 indicates to all users to not use this track
+            continue;
+          }
+
+          if (!extraInfoHolder.isTPConly && extraInfoHolder.trackTimeRes < 0.f) { // failed or rejected?
             LOG(warning) << "Barrel track " << trackIndex << " has no time set, rejection is not expected : time=" << extraInfoHolder.trackTime
                          << " timeErr=" << extraInfoHolder.trackTimeRes << " BCSlice: " << extraInfoHolder.bcSlice[0] << ":" << extraInfoHolder.bcSlice[1];
             continue;
           }
-          addToTracksTable(tracksCursor, tracksCovCursor, data.getTrackParam(trackIndex), collisionID);
+          const auto& trOrig = data.getTrackParam(trackIndex);
+          bool isProp = false;
+          if (mPropTracks && trOrig.getX() < mMaxPropXiu &&
+              mGIDUsedBySVtx.find(trackIndex) == mGIDUsedBySVtx.end() &&
+              mGIDUsedByStr.find(trackIndex) == mGIDUsedByStr.end()) { // Do not propagate track assoc. to V0s and str. tracking
+            auto trackPar(trOrig);
+            isProp = propagateTrackToPV(trackPar, data, collisionID);
+            if (isProp) {
+              addToTracksTable(tracksCursor, tracksCovCursor, trackPar, collisionID, aod::track::Track);
+            }
+          }
+          if (!isProp) {
+            addToTracksTable(tracksCursor, tracksCovCursor, trOrig, collisionID, aod::track::TrackIU);
+          }
           addToTracksExtraTable(tracksExtraCursor, extraInfoHolder);
-          // collecting table indices of barrel tracks for V0s table
-          if (extraInfoHolder.bcSlice[0] >= 0) {
-            ambigTracksCursor(0, mTableTrID, extraInfoHolder.bcSlice);
+
+          //  collecting table indices of barrel tracks for V0s table
+          if (extraInfoHolder.bcSlice[0] >= 0 && collisionID < 0) {
+            ambigTracksCursor(mTableTrID, extraInfoHolder.bcSlice);
           }
           mGIDToTableID.emplace(trackIndex, mTableTrID);
           mTableTrID++;
@@ -410,6 +564,25 @@ void AODProducerWorkflowDPL::fillTrackTablesPerCollision(int collisionID,
       }
     }
   }
+  if (collisionID < 0) {
+    return;
+  }
+  /// Add strangeness tracks to the table
+  auto sTracks = data.getStrangeTracks();
+  tracksCursor.reserve(mVertexStrLUT[collisionID + 1] + tracksCursor.lastIndex());
+  tracksCovCursor.reserve(mVertexStrLUT[collisionID + 1] + tracksCovCursor.lastIndex());
+  tracksExtraCursor.reserve(mVertexStrLUT[collisionID + 1] + tracksExtraCursor.lastIndex());
+  for (int iS{mVertexStrLUT[collisionID]}; iS < mVertexStrLUT[collisionID + 1]; ++iS) {
+    auto& collStrTrk = mCollisionStrTrk[iS];
+    auto& sTrk = sTracks[collStrTrk.second];
+    TrackExtraInfo extraInfo;
+    extraInfo.itsChi2NCl = sTrk.mTopoChi2; // TODO: this is the total chi2 of adding the ITS clusters, the topology chi2 meaning might change in the future
+    extraInfo.itsClusterSizes = sTrk.getClusterSizes();
+    addToTracksTable(tracksCursor, tracksCovCursor, sTrk.mMother, collisionID, aod::track::StrangeTrack);
+    addToTracksExtraTable(tracksExtraCursor, extraInfo);
+    mStrTrkIndices[collStrTrk.second] = mTableTrID;
+    mTableTrID++;
+  }
 }
 
 void AODProducerWorkflowDPL::fillIndexTablesPerCollision(const o2::dataformats::VtxTrackRef& trackRef, const gsl::span& GIndices, const o2::globaltracking::RecoContainer& data)
@@ -448,9 +621,9 @@ void AODProducerWorkflowDPL::fillIndexTablesPerCollision(const o2::dataformats::
   }
 }
 
-template 
+template 
 void AODProducerWorkflowDPL::addToFwdTracksTable(FwdTracksCursorType& fwdTracksCursor, FwdTracksCovCursorType& fwdTracksCovCursor,
-                                                 AmbigFwdTracksCursorType& ambigFwdTracksCursor, GIndex trackID,
+                                                 AmbigFwdTracksCursorType& ambigFwdTracksCursor, mftTracksCovCursorType& mftTracksCovCursor, GIndex trackID,
                                                  const o2::globaltracking::RecoContainer& data, int collisionID, std::uint64_t collisionBC,
                                                  const std::map& bcsMap)
 {
@@ -503,10 +676,12 @@ void AODProducerWorkflowDPL::addToFwdTracksTable(FwdTracksCursorType& fwdTracksC
     }
 
     o2::mch::TrackParam trackParamAtVertex(track.getZ(), track.getParameters(), track.getCovariances());
-    double errVtx{0.0}; // FIXME: get errors associated with vertex if available
-    double errVty{0.0};
-    if (!o2::mch::TrackExtrap::extrapToVertex(trackParamAtVertex, vx, vy, vz, errVtx, errVty)) {
-      return false;
+    if (mPropMuons) {
+      double errVtx{0.0}; // FIXME: get errors associated with vertex if available
+      double errVty{0.0};
+      if (!o2::mch::TrackExtrap::extrapToVertex(trackParamAtVertex, vx, vy, vz, errVtx, errVty)) {
+        return false;
+      }
     }
 
     // extrapolate to DCA
@@ -528,43 +703,37 @@ void AODProducerWorkflowDPL::addToFwdTracksTable(FwdTracksCursorType& fwdTracksC
     double xAbs = trackParamAtRAbs.getNonBendingCoor();
     double yAbs = trackParamAtRAbs.getBendingCoor();
 
-    double px = trackParamAtVertex.px();
-    double py = trackParamAtVertex.py();
-    double pz = trackParamAtVertex.pz();
-
-    double pt = std::sqrt(px * px + py * py);
-    double dphi = std::atan2(py, px);
-    double dtanl = pz / pt;
-    double dinvqpt = 1.0 / (trackParamAtVertex.getCharge() * pt);
     double dpdca = track.getP() * dca;
     double dchi2 = track.getChi2OverNDF();
 
-    fwdInfo.x = trackParamAtVertex.getNonBendingCoor();
-    fwdInfo.y = trackParamAtVertex.getBendingCoor();
-    fwdInfo.z = trackParamAtVertex.getZ();
+    auto fwdmuon = mMatching.MCHtoFwd(trackParamAtVertex);
+
+    fwdInfo.x = fwdmuon.getX();
+    fwdInfo.y = fwdmuon.getY();
+    fwdInfo.z = fwdmuon.getZ();
+    fwdInfo.phi = fwdmuon.getPhi();
+    fwdInfo.tanl = fwdmuon.getTgl();
+    fwdInfo.invqpt = fwdmuon.getInvQPt();
     fwdInfo.rabs = std::sqrt(xAbs * xAbs + yAbs * yAbs);
-    fwdInfo.phi = dphi;
-    fwdInfo.tanl = dtanl;
-    fwdInfo.invqpt = dinvqpt;
     fwdInfo.chi2 = dchi2;
     fwdInfo.pdca = dpdca;
     fwdInfo.nClusters = track.getNClusters();
 
-    fwdCovInfo.sigX = TMath::Sqrt(trackParamAtVertex.getCovariances()(0, 0));
-    fwdCovInfo.sigY = TMath::Sqrt(trackParamAtVertex.getCovariances()(1, 1));
-    fwdCovInfo.sigPhi = TMath::Sqrt(trackParamAtVertex.getCovariances()(2, 2));
-    fwdCovInfo.sigTgl = TMath::Sqrt(trackParamAtVertex.getCovariances()(3, 3));
-    fwdCovInfo.sig1Pt = TMath::Sqrt(trackParamAtVertex.getCovariances()(4, 4));
-    fwdCovInfo.rhoXY = (Char_t)(128. * trackParamAtVertex.getCovariances()(0, 1) / (fwdCovInfo.sigX * fwdCovInfo.sigY));
-    fwdCovInfo.rhoPhiX = (Char_t)(128. * trackParamAtVertex.getCovariances()(0, 2) / (fwdCovInfo.sigPhi * fwdCovInfo.sigX));
-    fwdCovInfo.rhoPhiY = (Char_t)(128. * trackParamAtVertex.getCovariances()(1, 2) / (fwdCovInfo.sigPhi * fwdCovInfo.sigY));
-    fwdCovInfo.rhoTglX = (Char_t)(128. * trackParamAtVertex.getCovariances()(0, 3) / (fwdCovInfo.sigTgl * fwdCovInfo.sigX));
-    fwdCovInfo.rhoTglY = (Char_t)(128. * trackParamAtVertex.getCovariances()(1, 3) / (fwdCovInfo.sigTgl * fwdCovInfo.sigY));
-    fwdCovInfo.rhoTglPhi = (Char_t)(128. * trackParamAtVertex.getCovariances()(2, 3) / (fwdCovInfo.sigTgl * fwdCovInfo.sigPhi));
-    fwdCovInfo.rho1PtX = (Char_t)(128. * trackParamAtVertex.getCovariances()(0, 4) / (fwdCovInfo.sig1Pt * fwdCovInfo.sigX));
-    fwdCovInfo.rho1PtY = (Char_t)(128. * trackParamAtVertex.getCovariances()(1, 4) / (fwdCovInfo.sig1Pt * fwdCovInfo.sigY));
-    fwdCovInfo.rho1PtPhi = (Char_t)(128. * trackParamAtVertex.getCovariances()(2, 4) / (fwdCovInfo.sig1Pt * fwdCovInfo.sigPhi));
-    fwdCovInfo.rho1PtTgl = (Char_t)(128. * trackParamAtVertex.getCovariances()(3, 4) / (fwdCovInfo.sig1Pt * fwdCovInfo.sigTgl));
+    fwdCovInfo.sigX = TMath::Sqrt(fwdmuon.getCovariances()(0, 0));
+    fwdCovInfo.sigY = TMath::Sqrt(fwdmuon.getCovariances()(1, 1));
+    fwdCovInfo.sigPhi = TMath::Sqrt(fwdmuon.getCovariances()(2, 2));
+    fwdCovInfo.sigTgl = TMath::Sqrt(fwdmuon.getCovariances()(3, 3));
+    fwdCovInfo.sig1Pt = TMath::Sqrt(fwdmuon.getCovariances()(4, 4));
+    fwdCovInfo.rhoXY = (Char_t)(128. * fwdmuon.getCovariances()(0, 1) / (fwdCovInfo.sigX * fwdCovInfo.sigY));
+    fwdCovInfo.rhoPhiX = (Char_t)(128. * fwdmuon.getCovariances()(0, 2) / (fwdCovInfo.sigPhi * fwdCovInfo.sigX));
+    fwdCovInfo.rhoPhiY = (Char_t)(128. * fwdmuon.getCovariances()(1, 2) / (fwdCovInfo.sigPhi * fwdCovInfo.sigY));
+    fwdCovInfo.rhoTglX = (Char_t)(128. * fwdmuon.getCovariances()(0, 3) / (fwdCovInfo.sigTgl * fwdCovInfo.sigX));
+    fwdCovInfo.rhoTglY = (Char_t)(128. * fwdmuon.getCovariances()(1, 3) / (fwdCovInfo.sigTgl * fwdCovInfo.sigY));
+    fwdCovInfo.rhoTglPhi = (Char_t)(128. * fwdmuon.getCovariances()(2, 3) / (fwdCovInfo.sigTgl * fwdCovInfo.sigPhi));
+    fwdCovInfo.rho1PtX = (Char_t)(128. * fwdmuon.getCovariances()(0, 4) / (fwdCovInfo.sig1Pt * fwdCovInfo.sigX));
+    fwdCovInfo.rho1PtY = (Char_t)(128. * fwdmuon.getCovariances()(1, 4) / (fwdCovInfo.sig1Pt * fwdCovInfo.sigY));
+    fwdCovInfo.rho1PtPhi = (Char_t)(128. * fwdmuon.getCovariances()(2, 4) / (fwdCovInfo.sig1Pt * fwdCovInfo.sigPhi));
+    fwdCovInfo.rho1PtTgl = (Char_t)(128. * fwdmuon.getCovariances()(3, 4) / (fwdCovInfo.sig1Pt * fwdCovInfo.sigTgl));
 
     return true;
   };
@@ -596,6 +765,11 @@ void AODProducerWorkflowDPL::addToFwdTracksTable(FwdTracksCursorType& fwdTracksC
     fwdInfo.trackTimeRes = time.getTimeStampError() * 1.e3;
   } else { // This is a GlobalMuonTrack or a GlobalForwardTrack
     const auto& track = data.getGlobalFwdTrack(trackID);
+    const auto& mftTracks = data.getMFTTracks();
+    const auto& mfttrack = mftTracks[track.getMFTTrackID()];
+    if (!extrapMCHTrack(track.getMCHTrackID())) {
+      LOGF(warn, "Unable to extrapolate MCH track with ID %d! Dummy parameters will be used", track.getMCHTrackID());
+    }
     fwdInfo.x = track.getX();
     fwdInfo.y = track.getY();
     fwdInfo.z = track.getZ();
@@ -632,20 +806,39 @@ void AODProducerWorkflowDPL::addToFwdTracksTable(FwdTracksCursorType& fwdTracksC
     fwdCovInfo.rho1PtTgl = (Char_t)(128. * track.getCovariances()(3, 4) / (fwdCovInfo.sig1Pt * fwdCovInfo.sigTgl));
 
     fwdInfo.trackTypeId = (fwdInfo.chi2matchmchmid >= 0) ? o2::aod::fwdtrack::GlobalMuonTrack : o2::aod::fwdtrack::GlobalForwardTrack;
+
+    float sX = TMath::Sqrt(mfttrack.getSigma2X()), sY = TMath::Sqrt(mfttrack.getSigma2Y()), sPhi = TMath::Sqrt(mfttrack.getSigma2Phi()),
+          sTgl = TMath::Sqrt(mfttrack.getSigma2Tanl()), sQ2Pt = TMath::Sqrt(mfttrack.getSigma2InvQPt());
+
+    mftTracksCovCursor(fwdInfo.matchmfttrackid,
+                       truncateFloatFraction(sX, mTrackCovDiag),
+                       truncateFloatFraction(sY, mTrackCovDiag),
+                       truncateFloatFraction(sPhi, mTrackCovDiag),
+                       truncateFloatFraction(sTgl, mTrackCovDiag),
+                       truncateFloatFraction(sQ2Pt, mTrackCovDiag),
+                       (Char_t)(128. * mfttrack.getCovariances()(0, 1) / (sX * sY)),
+                       (Char_t)(128. * mfttrack.getCovariances()(0, 2) / (sPhi * sX)),
+                       (Char_t)(128. * mfttrack.getCovariances()(1, 2) / (sPhi * sY)),
+                       (Char_t)(128. * mfttrack.getCovariances()(0, 3) / (sTgl * sX)),
+                       (Char_t)(128. * mfttrack.getCovariances()(1, 3) / (sTgl * sY)),
+                       (Char_t)(128. * mfttrack.getCovariances()(2, 3) / (sTgl * sPhi)),
+                       (Char_t)(128. * mfttrack.getCovariances()(0, 4) / (sQ2Pt * sX)),
+                       (Char_t)(128. * mfttrack.getCovariances()(1, 4) / (sQ2Pt * sY)),
+                       (Char_t)(128. * mfttrack.getCovariances()(2, 4) / (sQ2Pt * sPhi)),
+                       (Char_t)(128. * mfttrack.getCovariances()(3, 4) / (sQ2Pt * sTgl)));
   }
 
   std::uint64_t bcOfTimeRef;
-  bool needBCSlice = trackID.isAmbiguous() || collisionID < 0;
+  bool needBCSlice = collisionID < 0;
   if (needBCSlice) { // need to store BC slice
     float err = mTimeMarginTrackTime + fwdInfo.trackTimeRes;
     bcOfTimeRef = fillBCSlice(bcSlice, fwdInfo.trackTime - err, fwdInfo.trackTime + err, bcsMap);
   } else {
-    bcOfTimeRef = collisionBC - mStartIR.toLong(); // by default (unambiguous) track time is wrt collision BC
+    bcOfTimeRef = collisionBC - mStartIR.toLong(); // by default track time is wrt collision BC (unless no collision assigned)
   }
   fwdInfo.trackTime -= bcOfTimeRef * o2::constants::lhc::LHCBunchSpacingNS;
 
-  fwdTracksCursor(0,
-                  collisionID,
+  fwdTracksCursor(collisionID,
                   fwdInfo.trackTypeId,
                   fwdInfo.x,
                   fwdInfo.y,
@@ -668,8 +861,7 @@ void AODProducerWorkflowDPL::addToFwdTracksTable(FwdTracksCursorType& fwdTracksC
                   truncateFloatFraction(fwdInfo.trackTime, mTrackTime),
                   truncateFloatFraction(fwdInfo.trackTimeRes, mTrackTimeError));
 
-  fwdTracksCovCursor(0,
-                     truncateFloatFraction(fwdCovInfo.sigX, mTrackCovDiag),
+  fwdTracksCovCursor(truncateFloatFraction(fwdCovInfo.sigX, mTrackCovDiag),
                      truncateFloatFraction(fwdCovInfo.sigY, mTrackCovDiag),
                      truncateFloatFraction(fwdCovInfo.sigPhi, mTrackCovDiag),
                      truncateFloatFraction(fwdCovInfo.sigTgl, mTrackCovDiag),
@@ -686,18 +878,117 @@ void AODProducerWorkflowDPL::addToFwdTracksTable(FwdTracksCursorType& fwdTracksC
                      fwdCovInfo.rho1PtTgl);
 
   if (needBCSlice) {
-    ambigFwdTracksCursor(0, mTableTrFwdID, bcSlice);
+    ambigFwdTracksCursor(mTableTrFwdID, bcSlice);
+  }
+}
+
+//------------------------------------------------------------------
+void AODProducerWorkflowDPL::updateMCHeader(MCCollisionCursor& collisionCursor,
+                                            XSectionCursor& xSectionCursor,
+                                            PdfInfoCursor& pdfInfoCursor,
+                                            HeavyIonCursor& heavyIonCursor,
+                                            const MCEventHeader& header,
+                                            int collisionID,
+                                            int bcID,
+                                            float time,
+                                            short generatorID,
+                                            int sourceID)
+{
+  using aodmchelpers::updateHepMCHeavyIon;
+  using aodmchelpers::updateHepMCPdfInfo;
+  using aodmchelpers::updateHepMCXSection;
+  using aodmchelpers::updateMCCollisions;
+
+  auto genID = updateMCCollisions(collisionCursor,
+                                  bcID,
+                                  time,
+                                  header,
+                                  generatorID,
+                                  sourceID,
+                                  mCollisionPosition);
+  mXSectionUpdate = (updateHepMCXSection(xSectionCursor, //
+                                         collisionID,    //
+                                         genID,          //
+                                         header,         //
+                                         mXSectionUpdate)
+                       ? HepMCUpdate::always
+                       : HepMCUpdate::never);
+  mPdfInfoUpdate = (updateHepMCPdfInfo(pdfInfoCursor, //
+                                       collisionID,   //
+                                       genID,         //
+                                       header,        //
+                                       mPdfInfoUpdate)
+                      ? HepMCUpdate::always
+                      : HepMCUpdate::never);
+  mHeavyIonUpdate = (updateHepMCHeavyIon(heavyIonCursor, //
+                                         collisionID,    //
+                                         genID,          //
+                                         header,
+                                         mHeavyIonUpdate)
+                       ? HepMCUpdate::always
+                       : HepMCUpdate::never);
+}
+
+void dimensionMCKeepStore(std::vector>>& store, int Nsources, int NEvents)
+{
+  store.resize(Nsources);
+  for (int s = 0; s < Nsources; ++s) {
+    store[s].resize(NEvents);
+  }
+}
+
+void clearMCKeepStore(std::vector>>& store)
+{
+  for (auto s = 0U; s < store.size(); ++s) {
+    for (auto e = 0U; e < store[s].size(); ++e) {
+      store[s][e].clear();
+    }
+  }
+}
+
+// helper function to add a particle/track to the MC keep store
+void keepMCParticle(std::vector>>& store, int source, int event, int track, int value = 1, bool useSigFilt = false)
+{
+  if (track < 0) {
+    LOG(warn) << "trackID is smaller than 0. Neglecting";
+    return;
+  }
+  if (useSigFilt && source == 0) {
+    store[source][event][track] = -1;
+  } else {
+    store[source][event][track] = value;
   }
 }
 
-template 
 void AODProducerWorkflowDPL::fillMCParticlesTable(o2::steer::MCKinematicsReader& mcReader,
-                                                  const MCParticlesCursorType& mcParticlesCursor,
+                                                  MCParticlesCursor& mcParticlesCursor,
                                                   const gsl::span& primVer2TRefs,
                                                   const gsl::span& GIndices,
                                                   const o2::globaltracking::RecoContainer& data,
-                                                  const std::map, int>& mcColToEvSrc)
+                                                  const std::vector>& mcColToEvSrc)
 {
+  int NSources = 0;
+  int NEvents = 0;
+  for (auto& p : mcColToEvSrc) {
+    NSources = std::max(p[1], NSources);
+    NEvents = std::max(p[2], NEvents);
+  }
+  NSources++; // 0 - indexed
+  NEvents++;
+  LOG(info) << " number of events " << NEvents;
+  LOG(info) << " number of sources " << NSources;
+  dimensionMCKeepStore(mToStore, NSources, NEvents);
+
+  std::vector particleIDsToKeep;
+
+  auto markMCTrackForSrc = [&](std::array& contributorsGID, uint8_t src) {
+    auto mcLabel = data.getTrackMCLabel(contributorsGID[src]);
+    if (!mcLabel.isValid()) {
+      return;
+    }
+    keepMCParticle(mToStore, mcLabel.getSourceID(), mcLabel.getEventID(), mcLabel.getTrackID(), 1, mUseSigFiltMC);
+  };
+
   // mark reconstructed MC particles to store them into the table
   for (auto& trackRef : primVer2TRefs) {
     for (int src = GIndex::NSources; src--;) {
@@ -710,184 +1001,96 @@ void AODProducerWorkflowDPL::fillMCParticlesTable(o2::steer::MCKinematicsReader&
           if (!mcTruth.isValid()) {
             continue;
           }
-          int source = mcTruth.getSourceID();
-          int event = mcTruth.getEventID();
-          int particle = mcTruth.getTrackID();
-          mToStore[Triplet_t(source, event, particle)] = 1;
+          keepMCParticle(mToStore, mcTruth.getSourceID(), mcTruth.getEventID(), mcTruth.getTrackID(), 1, mUseSigFiltMC);
           // treating contributors of global tracks
           auto contributorsGID = data.getSingleDetectorRefs(trackIndex);
-          if (contributorsGID[GIndex::Source::ITS].isIndexSet() && contributorsGID[GIndex::Source::TPC].isIndexSet()) {
-            auto mcTruthITS = data.getTrackMCLabel(contributorsGID[GIndex::Source::ITS]);
-            if (mcTruthITS.isValid()) {
-              source = mcTruthITS.getSourceID();
-              event = mcTruthITS.getEventID();
-              particle = mcTruthITS.getTrackID();
-              mToStore[Triplet_t(source, event, particle)] = 1;
-            }
-            auto mcTruthTPC = data.getTrackMCLabel(contributorsGID[GIndex::Source::TPC]);
-            if (mcTruthTPC.isValid()) {
-              source = mcTruthTPC.getSourceID();
-              event = mcTruthTPC.getEventID();
-              particle = mcTruthTPC.getTrackID();
-              mToStore[Triplet_t(source, event, particle)] = 1;
+          if (contributorsGID[GIndex::Source::TPC].isIndexSet()) {
+            markMCTrackForSrc(contributorsGID, GIndex::Source::TPC);
+          }
+          if (contributorsGID[GIndex::Source::ITS].isIndexSet()) {
+            markMCTrackForSrc(contributorsGID, GIndex::Source::ITS);
+          }
+          if (contributorsGID[GIndex::Source::TOF].isIndexSet()) {
+            const auto& labelsTOF = data.getTOFClustersMCLabels()->getLabels(contributorsGID[GIndex::Source::TOF]);
+            for (auto& mcLabel : labelsTOF) {
+              if (!mcLabel.isValid()) {
+                continue;
+              }
+              keepMCParticle(mToStore, mcLabel.getSourceID(), mcLabel.getEventID(), mcLabel.getTrackID(), 1, mUseSigFiltMC);
             }
           }
         }
       }
     }
   }
-  int tableIndex = 1;
-  for (auto& colInfo : mcColToEvSrc) { // loop over " <-> combined MC col. ID" key pairs
-    int event = colInfo.first.first;
-    int source = colInfo.first.second;
-    int mcColId = colInfo.second;
-    std::vector const& mcParticles = mcReader.getTracks(source, event);
-    // mark tracks to be stored per event
-    // loop over stack of MC particles from end to beginning: daughters are stored after mothers
-    if (mRecoOnly) {
-      for (int particle = mcParticles.size() - 1; particle >= 0; particle--) {
-        // we store all primary particles == particles put by generator
-        if (mcParticles[particle].isPrimary()) {
-          mToStore[Triplet_t(source, event, particle)] = 1;
-        } else if (o2::mcutils::MCTrackNavigator::isPhysicalPrimary(mcParticles[particle], mcParticles)) {
-          mToStore[Triplet_t(source, event, particle)] = 1;
-        } else if (o2::mcutils::MCTrackNavigator::isKeepPhysics(mcParticles[particle], mcParticles)) {
-          mToStore[Triplet_t(source, event, particle)] = 1;
-        }
-
-        // skip treatment if this particle has not been marked during reconstruction
-        // or based on criteria just above
-        if (mToStore.find(Triplet_t(source, event, particle)) == mToStore.end()) {
-          continue;
-        }
-
-        int mother0 = mcParticles[particle].getMotherTrackId();
-        // we store mothers and daughters of particles that are reconstructed
-        if (mother0 != -1) {
-          mToStore[Triplet_t(source, event, mother0)] = 1;
-        }
-        int mother1 = mcParticles[particle].getSecondMotherTrackId();
-        if (mother1 != -1) {
-          mToStore[Triplet_t(source, event, mother1)] = 1;
-        }
-        int daughter0 = mcParticles[particle].getFirstDaughterTrackId();
-        if (daughter0 != -1) {
-          mToStore[Triplet_t(source, event, daughter0)] = 1;
-        }
-        int daughterL = mcParticles[particle].getLastDaughterTrackId();
-        if (daughterL != -1) {
-          mToStore[Triplet_t(source, event, daughterL)] = 1;
-        }
-      }
-      // enumerate reconstructed mc particles and their relatives to get mother/daughter relations
-      for (int particle = 0; particle < mcParticles.size(); particle++) {
-        auto mapItem = mToStore.find(Triplet_t(source, event, particle));
-        if (mapItem != mToStore.end()) {
-          mapItem->second = tableIndex - 1;
-          tableIndex++;
-        }
-      }
-    } else {
-      // if all mc particles are stored, all mc particles will be enumerated
-      for (int particle = 0; particle < mcParticles.size(); particle++) {
-        mToStore[Triplet_t(source, event, particle)] = tableIndex - 1;
-        tableIndex++;
+  // mark calorimeter signals as reconstructed particles
+  if (mInputSources[GIndex::EMC]) {
+    auto& mcCaloEMCCellLabels = data.getEMCALCellsMCLabels()->getTruthArray();
+    for (auto& mcTruth : mcCaloEMCCellLabels) {
+      if (!mcTruth.isValid()) {
+        continue;
       }
+      keepMCParticle(mToStore, mcTruth.getSourceID(), mcTruth.getEventID(), mcTruth.getTrackID(), 1, mUseSigFiltMC);
     }
-
-    // second part: fill survived mc tracks into the AOD table
-    for (int particle = 0; particle < mcParticles.size(); particle++) {
-      if (mToStore.find(Triplet_t(source, event, particle)) == mToStore.end()) {
+  }
+  if (mInputSources[GIndex::PHS]) {
+    auto& mcCaloPHOSCellLabels = data.getPHOSCellsMCLabels()->getTruthArray();
+    for (auto& mcTruth : mcCaloPHOSCellLabels) {
+      if (!mcTruth.isValid()) {
         continue;
       }
-      int statusCode = 0;
-      uint8_t flags = 0;
-      if (!mcParticles[particle].isPrimary()) {
-        flags |= o2::aod::mcparticle::enums::ProducedByTransport; // mark as produced by transport
-        statusCode = mcParticles[particle].getProcess();
-      } else {
-        statusCode = mcParticles[particle].getStatusCode();
-      }
-      if (source == 0) {
-        flags |= o2::aod::mcparticle::enums::FromBackgroundEvent; // mark as particle from background event
-      }
-      if (o2::mcutils::MCTrackNavigator::isPhysicalPrimary(mcParticles[particle], mcParticles)) {
-        flags |= o2::aod::mcparticle::enums::PhysicalPrimary; // mark as physical primary
-      }
-      float weight = 0.f;
-      std::vector mothers;
-      int mcMother0 = mcParticles[particle].getMotherTrackId();
-      auto item = mToStore.find(Triplet_t(source, event, mcMother0));
-      if (item != mToStore.end()) {
-        mothers.push_back(item->second);
-      }
-      int mcMother1 = mcParticles[particle].getSecondMotherTrackId();
-      item = mToStore.find(Triplet_t(source, event, mcMother1));
-      if (item != mToStore.end()) {
-        mothers.push_back(item->second);
-      }
-      int daughters[2] = {-1, -1}; // slice
-      int mcDaughter0 = mcParticles[particle].getFirstDaughterTrackId();
-      item = mToStore.find(Triplet_t(source, event, mcDaughter0));
-      if (item != mToStore.end()) {
-        daughters[0] = item->second;
-      }
-      int mcDaughterL = mcParticles[particle].getLastDaughterTrackId();
-      item = mToStore.find(Triplet_t(source, event, mcDaughterL));
-      if (item != mToStore.end()) {
-        daughters[1] = item->second;
-        if (daughters[0] < 0) {
-          LOG(error) << "AOD problematic daughter case observed";
-          daughters[0] = daughters[1]; /// Treat the case of first negative label (pruned in the kinematics)
-        }
-      } else {
-        daughters[1] = daughters[0];
-      }
-      if (daughters[0] > daughters[1]) {
-        std::swap(daughters[0], daughters[1]);
-      }
-      auto pX = (float)mcParticles[particle].Px();
-      auto pY = (float)mcParticles[particle].Py();
-      auto pZ = (float)mcParticles[particle].Pz();
-      auto energy = (float)mcParticles[particle].GetEnergy();
-      mcParticlesCursor(0,
-                        mcColId,
-                        mcParticles[particle].GetPdgCode(),
-                        statusCode,
-                        flags,
-                        mothers,
-                        daughters,
-                        truncateFloatFraction(weight, mMcParticleW),
-                        truncateFloatFraction(pX, mMcParticleMom),
-                        truncateFloatFraction(pY, mMcParticleMom),
-                        truncateFloatFraction(pZ, mMcParticleMom),
-                        truncateFloatFraction(energy, mMcParticleMom),
-                        truncateFloatFraction((float)mcParticles[particle].Vx(), mMcParticlePos),
-                        truncateFloatFraction((float)mcParticles[particle].Vy(), mMcParticlePos),
-                        truncateFloatFraction((float)mcParticles[particle].Vz(), mMcParticlePos),
-                        truncateFloatFraction((float)mcParticles[particle].T(), mMcParticlePos));
+      keepMCParticle(mToStore, mcTruth.getSourceID(), mcTruth.getEventID(), mcTruth.getTrackID(), 1, mUseSigFiltMC);
     }
+  }
+  using namespace aodmchelpers;
+  using MCTrackNavigator = o2::mcutils::MCTrackNavigator;
+
+  size_t offset = 0;
+  for (auto& colInfo : mcColToEvSrc) { // loop over " <-> combined MC col. ID" key pairs
+    int event = colInfo[2];
+    int source = colInfo[1];
+    int mcColId = colInfo[0];
+    std::vector const& mcParticles = mcReader.getTracks(source, event);
+    LOG(debug) << "Event=" << event << " source=" << source << " collision=" << mcColId;
+    auto& preselect = mToStore[source][event];
+
+    offset = updateParticles(mcParticlesCursor,
+                             mcColId,
+                             mcParticles,
+                             preselect,
+                             offset,
+                             mRecoOnly,
+                             source == 0, // background
+                             mMcParticleW,
+                             mMcParticleMom,
+                             mMcParticlePos,
+                             mUseSigFiltMC);
+
     mcReader.releaseTracksForSourceAndEvent(source, event);
   }
 }
 
 template 
-void AODProducerWorkflowDPL::fillMCTrackLabelsTable(const MCTrackLabelCursorType& mcTrackLabelCursor,
-                                                    const MCMFTTrackLabelCursorType& mcMFTTrackLabelCursor,
-                                                    const MCFwdTrackLabelCursorType& mcFwdTrackLabelCursor,
+void AODProducerWorkflowDPL::fillMCTrackLabelsTable(MCTrackLabelCursorType& mcTrackLabelCursor,
+                                                    MCMFTTrackLabelCursorType& mcMFTTrackLabelCursor,
+                                                    MCFwdTrackLabelCursorType& mcFwdTrackLabelCursor,
                                                     const o2::dataformats::VtxTrackRef& trackRef,
                                                     const gsl::span& primVerGIs,
-                                                    const o2::globaltracking::RecoContainer& data)
+                                                    const o2::globaltracking::RecoContainer& data,
+                                                    int vertexId)
 {
   // labelMask (temporary) usage:
-  //   bit 13 -- ITS and TPC labels are not equal
+  //   bit 13 -- ITS/TPC with ITS label (track of AB tracklet) different from TPC
   //   bit 14 -- isNoise() == true
-  //   bit 15 -- isFake() == true
+  //   bit 15 -- isFake() == true (defined by the fakeness of the top level global track, i.e. if TOF is present, fake means that the track of the TPC label does not contribute to TOF cluster)
   // labelID = -1 -- label is not set
 
   for (int src = GIndex::NSources; src--;) {
     int start = trackRef.getFirstEntryOfSource(src);
     int end = start + trackRef.getEntriesOfSource(src);
+    mcMFTTrackLabelCursor.reserve(end - start + mcMFTTrackLabelCursor.lastIndex());
+    mcFwdTrackLabelCursor.reserve(end - start + mcFwdTrackLabelCursor.lastIndex());
+    mcTrackLabelCursor.reserve(end - start + mcTrackLabelCursor.lastIndex());
     for (int ti = start; ti < end; ti++) {
       const auto trackIndex = primVerGIs[ti];
 
@@ -903,13 +1106,13 @@ void AODProducerWorkflowDPL::fillMCTrackLabelsTable(const MCTrackLabelCursorType
 
       if (GIndex::includesSource(src, mInputSources)) {
         auto mcTruth = data.getTrackMCLabel(trackIndex);
-        MCLabels labelHolder;
+        MCLabels labelHolder{};
         if ((src == GIndex::Source::MFT) || (src == GIndex::Source::MFTMCH) || (src == GIndex::Source::MCH) || (src == GIndex::Source::MCHMID)) { // treating mft and fwd labels separately
           if (!needToStore(src == GIndex::Source::MFT ? mGIDToTableMFTID : mGIDToTableFwdID)) {
             continue;
           }
           if (mcTruth.isValid()) { // if not set, -1 will be stored
-            labelHolder.labelID = mToStore.at(Triplet_t(mcTruth.getSourceID(), mcTruth.getEventID(), mcTruth.getTrackID()));
+            labelHolder.labelID = (mToStore[mcTruth.getSourceID()][mcTruth.getEventID()])[mcTruth.getTrackID()];
           }
           if (mcTruth.isFake()) {
             labelHolder.fwdLabelMask |= (0x1 << 7);
@@ -918,65 +1121,84 @@ void AODProducerWorkflowDPL::fillMCTrackLabelsTable(const MCTrackLabelCursorType
             labelHolder.fwdLabelMask |= (0x1 << 6);
           }
           if (src == GIndex::Source::MFT) {
-            mcMFTTrackLabelCursor(0,
-                                  labelHolder.labelID,
+            mcMFTTrackLabelCursor(labelHolder.labelID,
                                   labelHolder.fwdLabelMask);
           } else {
-            mcFwdTrackLabelCursor(0,
-                                  labelHolder.labelID,
+            mcFwdTrackLabelCursor(labelHolder.labelID,
                                   labelHolder.fwdLabelMask);
           }
         } else {
           if (!needToStore(mGIDToTableID)) {
             continue;
           }
-          if (mcTruth.isValid()) { // if not set, -1 will be stored
-            labelHolder.labelID = mToStore.at(Triplet_t(mcTruth.getSourceID(), mcTruth.getEventID(), mcTruth.getTrackID()));
-          }
-          // treating possible mismatches for global tracks
-          auto contributorsGID = data.getSingleDetectorRefs(trackIndex);
-          if (contributorsGID[GIndex::Source::ITS].isIndexSet() && contributorsGID[GIndex::Source::TPC].isIndexSet()) {
-            auto mcTruthITS = data.getTrackMCLabel(contributorsGID[GIndex::Source::ITS]);
-            if (mcTruthITS.isValid()) {
-              labelHolder.labelITS = mToStore.at(Triplet_t(mcTruthITS.getSourceID(), mcTruthITS.getEventID(), mcTruthITS.getTrackID()));
+          if (mcTruth.isValid()) {                                                                               // if not set, -1 will be stored
+            labelHolder.labelID = (mToStore[mcTruth.getSourceID()][mcTruth.getEventID()])[mcTruth.getTrackID()]; // defined by TPC if it contributes, otherwise: by ITS
+            if (mcTruth.isFake()) {
+              labelHolder.labelMask |= (0x1 << 15);
             }
-            auto mcTruthTPC = data.getTrackMCLabel(contributorsGID[GIndex::Source::TPC]);
-            if (mcTruthTPC.isValid()) {
-              labelHolder.labelTPC = mToStore.at(Triplet_t(mcTruthTPC.getSourceID(), mcTruthTPC.getEventID(), mcTruthTPC.getTrackID()));
-              labelHolder.labelID = labelHolder.labelTPC;
+            if (trackIndex.includesDet(DetID::TPC) && trackIndex.getSource() != GIndex::Source::TPC) { // this is global track
+              auto contributorsGID = data.getSingleDetectorRefs(trackIndex);
+              if (contributorsGID[GIndex::Source::ITSTPC].isIndexSet()) { // there is a match to ITS tracks or ITSAB tracklet!
+                if (data.getTrackMCLabel(contributorsGID[GIndex::Source::ITSTPC]).isFake()) {
+                  labelHolder.labelMask |= (0x1 << 13);
+                }
+              }
             }
-            if (labelHolder.labelITS != labelHolder.labelTPC) {
-              LOG(debug) << "ITS-TPC MCTruth: labelIDs do not match at " << trackIndex.getIndex() << ", src = " << src;
-              labelHolder.labelMask |= (0x1 << 13);
+            if (trackIndex.includesDet(DetID::ITS)) {
+              auto itsGID = data.getITSContributorGID(trackIndex);
+              auto itsSource = itsGID.getSource();
+              if (itsSource == GIndex::ITS) {
+                auto& itsTrack = data.getITSTrack(itsGID);
+                for (unsigned int iL = 0; iL < 7; ++iL) {
+                  if (itsTrack.isFakeOnLayer(iL)) {
+                    labelHolder.labelMask |= (0x1 << iL);
+                  }
+                }
+              } else if (itsSource == GIndex::ITSAB) {
+                labelHolder.labelMask |= (data.getTrackMCLabel(itsGID).isFake() << 12);
+              }
             }
-          }
-          if (mcTruth.isFake()) {
-            labelHolder.labelMask |= (0x1 << 15);
-          }
-          if (mcTruth.isNoise()) {
+
+          } else if (mcTruth.isNoise()) {
             labelHolder.labelMask |= (0x1 << 14);
           }
-          mcTrackLabelCursor(0,
-                             labelHolder.labelID,
-                             labelHolder.labelMask);
+          mcTrackLabelCursor(labelHolder.labelID, labelHolder.labelMask);
         }
       }
     }
   }
+
+  // filling the tables with the strangeness tracking labels
+  auto sTrackLabels = data.getStrangeTracksMCLabels();
+  // check if vertexId and vertexId + 1 maps into mVertexStrLUT
+  if (!(vertexId < 0 || vertexId >= mVertexStrLUT.size() - 1)) {
+    mcTrackLabelCursor.reserve(mVertexStrLUT[vertexId + 1] + mcTrackLabelCursor.lastIndex());
+    for (int iS{mVertexStrLUT[vertexId]}; iS < mVertexStrLUT[vertexId + 1]; ++iS) {
+      auto& collStrTrk = mCollisionStrTrk[iS];
+      auto& label = sTrackLabels[collStrTrk.second];
+      MCLabels labelHolder;
+      labelHolder.labelID = label.isValid() ? (mToStore[label.getSourceID()][label.getEventID()])[label.getTrackID()] : -1;
+      labelHolder.labelMask = (label.isFake() << 15) | (label.isNoise() << 14);
+      mcTrackLabelCursor(labelHolder.labelID, labelHolder.labelMask);
+    }
+  }
 }
 
-template 
-void AODProducerWorkflowDPL::fillSecondaryVertices(const o2::globaltracking::RecoContainer& recoData, V0CursorType& v0Cursor, CascadeCursorType& cascadeCursor)
+template 
+void AODProducerWorkflowDPL::fillSecondaryVertices(const o2::globaltracking::RecoContainer& recoData, V0CursorType& v0Cursor, CascadeCursorType& cascadeCursor, Decay3BodyCursorType& decay3BodyCursor)
 {
 
-  auto v0s = recoData.getV0s();
-  auto cascades = recoData.getCascades();
+  auto v0s = recoData.getV0sIdx();
+  auto cascades = recoData.getCascadesIdx();
+  auto decays3Body = recoData.getDecays3BodyIdx();
 
+  v0Cursor.reserve(v0s.size());
   // filling v0s table
   for (size_t iv0 = 0; iv0 < v0s.size(); iv0++) {
     const auto& v0 = v0s[iv0];
     auto trPosID = v0.getProngID(0);
     auto trNegID = v0.getProngID(1);
+    uint8_t v0flags = v0.getBits();
     int posTableIdx = -1, negTableIdx = -1, collID = -1;
     auto item = mGIDToTableID.find(trPosID);
     if (item != mGIDToTableID.end()) {
@@ -997,12 +1219,13 @@ void AODProducerWorkflowDPL::fillSecondaryVertices(const o2::globaltracking::Rec
       collID = itemV->second;
     }
     if (posTableIdx != -1 and negTableIdx != -1 and collID != -1) {
-      v0Cursor(0, collID, posTableIdx, negTableIdx);
+      v0Cursor(collID, posTableIdx, negTableIdx, v0flags);
       mV0ToTableID[int(iv0)] = mTableV0ID++;
     }
   }
 
   // filling cascades table
+  cascadeCursor.reserve(cascades.size());
   for (auto& cascade : cascades) {
     auto itemV0 = mV0ToTableID.find(cascade.getV0ID());
     if (itemV0 == mV0ToTableID.end()) {
@@ -1024,50 +1247,259 @@ void AODProducerWorkflowDPL::fillSecondaryVertices(const o2::globaltracking::Rec
       LOG(warn) << "Could not find cascade collisionID for the vertex ID " << cascade.getVertexID();
       continue;
     }
-    cascadeCursor(0, collID, v0tableID, bachTableIdx);
+    cascadeCursor(collID, v0tableID, bachTableIdx);
+  }
+
+  // filling 3 body decays table
+  decay3BodyCursor.reserve(decays3Body.size());
+  for (size_t i3Body = 0; i3Body < decays3Body.size(); i3Body++) {
+    const auto& decay3Body = decays3Body[i3Body];
+    GIndex trIDs[3]{
+      decay3Body.getProngID(0),
+      decay3Body.getProngID(1),
+      decay3Body.getProngID(2)};
+    int tableIdx[3]{-1, -1, -1}, collID = -1;
+    bool missing{false};
+    for (int i{0}; i < 3; ++i) {
+      auto item = mGIDToTableID.find(trIDs[i]);
+      if (item != mGIDToTableID.end()) {
+        tableIdx[i] = item->second;
+      } else {
+        LOG(warn) << fmt::format("Could not find a track index for prong ID {}", (int)trIDs[i]);
+        missing = true;
+      }
+    }
+    auto itemV = mVtxToTableCollID.find(decay3Body.getVertexID());
+    if (itemV == mVtxToTableCollID.end()) {
+      LOG(warn) << "Could not find 3 body collisionID for the vertex ID " << decay3Body.getVertexID();
+      missing = true;
+    } else {
+      collID = itemV->second;
+    }
+    if (missing) {
+      continue;
+    }
+    decay3BodyCursor(collID, tableIdx[0], tableIdx[1], tableIdx[2]);
   }
 }
 
-void AODProducerWorkflowDPL::countTPCClusters(const o2::tpc::TrackTPC& track,
-                                              const gsl::span& tpcClusRefs,
-                                              const gsl::span& tpcClusShMap,
-                                              const o2::tpc::ClusterNativeAccess& tpcClusAcc,
-                                              uint8_t& shared, uint8_t& found, uint8_t& crossed)
+template 
+void AODProducerWorkflowDPL::addClustersToFwdTrkClsTable(const o2::globaltracking::RecoContainer& recoData, FwdTrkClsCursorType& fwdTrkClsCursor, GIndex trackID, int fwdTrackId)
 {
-  constexpr int maxRows = 152;
-  constexpr int neighbour = 2;
-  std::array clMap{}, shMap{};
-  uint8_t sectorIndex;
-  uint8_t rowIndex;
-  uint32_t clusterIndex;
-  shared = 0;
-  for (int i = 0; i < track.getNClusterReferences(); i++) {
-    o2::tpc::TrackTPC::getClusterReference(tpcClusRefs, i, sectorIndex, rowIndex, clusterIndex, track.getClusterRef());
-    unsigned int absoluteIndex = tpcClusAcc.clusterOffset[sectorIndex][rowIndex] + clusterIndex;
-    clMap[rowIndex] = true;
-    if (tpcClusShMap[absoluteIndex] & GPUCA_NAMESPACE::gpu::GPUTPCGMMergedTrackHit::flagShared) {
-      if (!shMap[rowIndex]) {
-        shared++;
-      }
-      shMap[rowIndex] = true;
+  const auto& mchTracks = recoData.getMCHTracks();
+  const auto& mchmidMatches = recoData.getMCHMIDMatches();
+  const auto& mchClusters = recoData.getMCHTrackClusters();
+
+  int mchTrackID = -1;
+  if (trackID.getSource() == GIndex::MCH) { // This is an MCH track
+    mchTrackID = trackID.getIndex();
+  } else if (trackID.getSource() == GIndex::MCHMID) { // This is an MCH-MID track
+    auto mchmidMatch = mchmidMatches[trackID.getIndex()];
+    mchTrackID = mchmidMatch.getMCHRef().getIndex();
+  } // Others are Global Forward Tracks, their clusters will be or were added with the corresponding MCH track
+
+  if (mchTrackID > -1 && mchTrackID < mchTracks.size()) {
+    const auto& mchTrack = mchTracks[mchTrackID];
+    fwdTrkClsCursor.reserve(mchTrack.getNClusters() + fwdTrkClsCursor.lastIndex());
+    int first = mchTrack.getFirstClusterIdx();
+    int last = mchTrack.getLastClusterIdx();
+    for (int i = first; i <= last; i++) {
+      const auto& cluster = mchClusters[i];
+      fwdTrkClsCursor(fwdTrackId,
+                      truncateFloatFraction(cluster.x, mMuonCl),
+                      truncateFloatFraction(cluster.y, mMuonCl),
+                      truncateFloatFraction(cluster.z, mMuonCl),
+                      (((cluster.ey < 5.) & 0x1) << 12) | (((cluster.ex < 5.) & 0x1) << 11) | cluster.getDEId());
     }
   }
+}
+
+template 
+void AODProducerWorkflowDPL::fillHMPID(const o2::globaltracking::RecoContainer& recoData, HMPCursorType& hmpCursor)
+{
+  auto hmpMatches = recoData.getHMPMatches();
+
+  hmpCursor.reserve(hmpMatches.size());
+
+  // filling HMPs table
+  for (size_t iHmp = 0; iHmp < hmpMatches.size(); iHmp++) {
 
-  crossed = 0;
-  found = 0;
-  int last = -1;
-  for (int i = 0; i < maxRows; i++) {
-    if (clMap[i]) {
-      crossed++;
-      found++;
-      last = i;
-    } else if ((i - last) <= neighbour) {
-      crossed++;
+    const auto& match = hmpMatches[iHmp];
+
+    float xTrk, yTrk, theta, phi;
+    float xMip, yMip;
+    int charge, nph;
+
+    match.getHMPIDtrk(xTrk, yTrk, theta, phi);
+    match.getHMPIDmip(xMip, yMip, charge, nph);
+
+    auto photChargeVec = match.getPhotCharge();
+
+    float photChargeVec2[10]; // = {0.,0.,0.,0.,0.,0.,0.,0.,0.,0.};
+
+    for (Int_t i = 0; i < 10; i++) {
+      photChargeVec2[i] = photChargeVec[i];
+    }
+    auto tref = mGIDToTableID.find(match.getTrackRef());
+    if (tref != mGIDToTableID.end()) {
+      hmpCursor(tref->second, match.getHMPsignal(), xTrk, yTrk, xMip, yMip, nph, charge, match.getIdxHMPClus(), match.getHmpMom(), photChargeVec2);
+    } else {
+      LOG(error) << "Could not find AOD track table entry for HMP-matched track " << match.getTrackRef().asString();
+    }
+  }
+}
+
+void AODProducerWorkflowDPL::prepareStrangenessTracking(const o2::globaltracking::RecoContainer& recoData)
+{
+  auto v0s = recoData.getV0sIdx();
+  auto cascades = recoData.getCascadesIdx();
+  auto decays3Body = recoData.getDecays3BodyIdx();
+
+  int sTrkID = 0;
+  mCollisionStrTrk.clear();
+  mCollisionStrTrk.reserve(recoData.getStrangeTracks().size());
+  mVertexStrLUT.clear();
+  mVertexStrLUT.resize(recoData.getPrimaryVertices().size() + 1, 0);
+  for (auto& sTrk : recoData.getStrangeTracks()) {
+    auto ITSIndex = GIndex{sTrk.mITSRef, GIndex::ITS};
+    int vtxId{0};
+    if (sTrk.mPartType == dataformats::kStrkV0) {
+      vtxId = v0s[sTrk.mDecayRef].getVertexID();
+    } else if (sTrk.mPartType == dataformats::kStrkCascade) {
+      vtxId = cascades[sTrk.mDecayRef].getVertexID();
+    } else {
+      vtxId = decays3Body[sTrk.mDecayRef].getVertexID();
+    }
+    mCollisionStrTrk.emplace_back(vtxId, sTrkID++);
+    mVertexStrLUT[vtxId]++;
+  }
+  std::exclusive_scan(mVertexStrLUT.begin(), mVertexStrLUT.end(), mVertexStrLUT.begin(), 0);
+
+  // sort by collision ID
+  std::sort(mCollisionStrTrk.begin(), mCollisionStrTrk.end(), [](const auto& a, const auto& b) { return a.first < b.first; });
+  mStrTrkIndices.clear();
+  mStrTrkIndices.resize(mCollisionStrTrk.size(), -1);
+}
+
+template 
+void AODProducerWorkflowDPL::fillStrangenessTrackingTables(const o2::globaltracking::RecoContainer& recoData, V0C& v0Curs, CC& cascCurs, D3BC& d3BodyCurs)
+{
+  int itsTableIdx = -1;
+  int sTrkID = 0;
+  int nV0 = 0;
+  int nCasc = 0;
+  int nD3Body = 0;
+
+  for (const auto& sTrk : recoData.getStrangeTracks()) {
+    if (sTrk.mPartType == dataformats::kStrkV0) {
+      nV0++;
+    } else if (sTrk.mPartType == dataformats::kStrkCascade) {
+      nCasc++;
     } else {
-      int lim = std::min(i + 1 + neighbour, maxRows);
-      for (int j = i + 1; j < lim; j++) {
-        if (clMap[j]) {
-          crossed++;
+      nD3Body++;
+    }
+  }
+
+  v0Curs.reserve(nV0);
+  cascCurs.reserve(nCasc);
+  d3BodyCurs.reserve(nD3Body);
+
+  for (const auto& sTrk : recoData.getStrangeTracks()) {
+    auto ITSIndex = GIndex{sTrk.mITSRef, GIndex::ITS};
+    auto item = mGIDToTableID.find(ITSIndex);
+    if (item != mGIDToTableID.end()) {
+      itsTableIdx = item->second;
+    } else {
+      LOG(warn) << "Could not find a ITS strange track index " << ITSIndex;
+      continue;
+    }
+    if (sTrk.mPartType == dataformats::kStrkV0) {
+      v0Curs(mStrTrkIndices[sTrkID++],
+             itsTableIdx,
+             sTrk.mDecayRef,
+             sTrk.mDecayVtx[0],
+             sTrk.mDecayVtx[1],
+             sTrk.mDecayVtx[2],
+             sTrk.mMasses[0],
+             sTrk.mMasses[1],
+             sTrk.mMatchChi2,
+             sTrk.mTopoChi2,
+             sTrk.getAverageClusterSize());
+    } else if (sTrk.mPartType == dataformats::kStrkCascade) {
+      cascCurs(mStrTrkIndices[sTrkID++],
+               itsTableIdx,
+               sTrk.mDecayRef,
+               sTrk.mDecayVtx[0],
+               sTrk.mDecayVtx[1],
+               sTrk.mDecayVtx[2],
+               sTrk.mMasses[0],
+               sTrk.mMasses[1],
+               sTrk.mMatchChi2,
+               sTrk.mTopoChi2,
+               sTrk.getAverageClusterSize());
+    } else {
+      d3BodyCurs(mStrTrkIndices[sTrkID++],
+                 itsTableIdx,
+                 sTrk.mDecayRef,
+                 sTrk.mDecayVtx[0],
+                 sTrk.mDecayVtx[1],
+                 sTrk.mDecayVtx[2],
+                 sTrk.mMasses[0],
+                 sTrk.mMasses[1],
+                 sTrk.mMatchChi2,
+                 sTrk.mTopoChi2,
+                 sTrk.getAverageClusterSize());
+    }
+  }
+}
+
+void AODProducerWorkflowDPL::countTPCClusters(const o2::globaltracking::RecoContainer& data)
+{
+  const auto& tpcTracks = data.getTPCTracks();
+  const auto& tpcClusRefs = data.getTPCTracksClusterRefs();
+  const auto& tpcClusShMap = data.clusterShMapTPC;
+  const auto& tpcClusAcc = data.getTPCClusters();
+  constexpr int maxRows = 152;
+  constexpr int neighbour = 2;
+  int ntr = tpcTracks.size();
+  mTPCCounters.clear();
+  mTPCCounters.resize(ntr);
+#ifdef WITH_OPENMP
+  int ngroup = std::min(50, std::max(1, ntr / mNThreads));
+#pragma omp parallel for schedule(dynamic, ngroup) num_threads(mNThreads)
+#endif
+  for (int itr = 0; itr < ntr; itr++) {
+    std::array clMap{}, shMap{};
+    uint8_t sectorIndex, rowIndex;
+    uint32_t clusterIndex;
+    auto& counters = mTPCCounters[itr];
+    const auto& track = tpcTracks[itr];
+    for (int i = 0; i < track.getNClusterReferences(); i++) {
+      o2::tpc::TrackTPC::getClusterReference(tpcClusRefs, i, sectorIndex, rowIndex, clusterIndex, track.getClusterRef());
+      unsigned int absoluteIndex = tpcClusAcc.clusterOffset[sectorIndex][rowIndex] + clusterIndex;
+      clMap[rowIndex] = true;
+      if (tpcClusShMap[absoluteIndex] & o2::gpu::GPUTPCGMMergedTrackHit::flagShared) {
+        if (!shMap[rowIndex]) {
+          counters.shared++;
+        }
+        shMap[rowIndex] = true;
+      }
+    }
+    int last = -1;
+    for (int i = 0; i < maxRows; i++) {
+      if (clMap[i]) {
+        counters.crossed++;
+        counters.found++;
+        last = i;
+      } else if ((i - last) <= neighbour) {
+        counters.crossed++;
+      } else {
+        int lim = std::min(i + 1 + neighbour, maxRows);
+        for (int j = i + 1; j < lim; j++) {
+          if (clMap[j]) {
+            counters.crossed++;
+            break;
+          }
         }
       }
     }
@@ -1077,34 +1509,169 @@ void AODProducerWorkflowDPL::countTPCClusters(const o2::tpc::TrackTPC& track,
 uint8_t AODProducerWorkflowDPL::getTRDPattern(const o2::trd::TrackTRD& track)
 {
   uint8_t pattern = 0;
-  for (int il = o2::trd::TrackTRD::EGPUTRDTrack::kNLayers; il >= 0; il--) {
+  for (int il = o2::trd::TrackTRD::EGPUTRDTrack::kNLayers - 1; il >= 0; il--) {
     if (track.getTrackletIndex(il) != -1) {
       pattern |= 0x1 << il;
     }
   }
+  if (track.getHasNeighbor()) {
+    pattern |= 0x1 << 6;
+  }
+  if (track.getHasPadrowCrossing()) {
+    pattern |= 0x1 << 7;
+  }
   return pattern;
 }
 
+template 
+void AODProducerWorkflowDPL::addToCaloTable(TCaloHandler& caloHandler, TCaloCursor& caloCellCursor, TCaloTRGCursor& caloTRGCursor,
+                                            TMCCaloLabelCursor& mcCaloCellLabelCursor, int eventID, int bcID, int8_t caloType)
+{
+  auto inputEvent = caloHandler.buildEvent(eventID);
+  auto cellsInEvent = inputEvent.mCells;        // get cells belonging to current event
+  auto cellMClabels = inputEvent.mMCCellLabels; // get MC labels belonging to current event (only implemented for EMCal currently!)
+  caloCellCursor.reserve(cellsInEvent.size() + caloCellCursor.lastIndex());
+  caloTRGCursor.reserve(cellsInEvent.size() + caloTRGCursor.lastIndex());
+  if (mUseMC) {
+    mcCaloCellLabelCursor.reserve(cellsInEvent.size() + mcCaloCellLabelCursor.lastIndex());
+  }
+  for (auto iCell = 0U; iCell < cellsInEvent.size(); iCell++) {
+    caloCellCursor(bcID,
+                   CellHelper::getCellNumber(cellsInEvent[iCell]),
+                   truncateFloatFraction(CellHelper::getAmplitude(cellsInEvent[iCell]), mCaloAmp),
+                   truncateFloatFraction(CellHelper::getTimeStamp(cellsInEvent[iCell]), mCaloTime),
+                   cellsInEvent[iCell].getType(),
+                   caloType); // 1 = emcal, -1 = undefined, 0 = phos
+
+    // todo: fix dummy values in CellHelper when it is clear what is filled for trigger information
+    if (CellHelper::isTRU(cellsInEvent[iCell])) { // Write only trigger cells into this table
+      caloTRGCursor(bcID,
+                    CellHelper::getFastOrAbsID(cellsInEvent[iCell]),
+                    CellHelper::getLnAmplitude(cellsInEvent[iCell]),
+                    CellHelper::getTriggerBits(cellsInEvent[iCell]),
+                    caloType);
+    }
+    if (mUseMC) {
+      // Common for PHOS and EMCAL
+      //  loop over all MC Labels for the current cell
+      std::vector particleIds;
+      std::vector amplitudeFraction;
+      if (!mEMCselectLeading) {
+        particleIds.reserve(cellMClabels.size());
+        amplitudeFraction.reserve(cellMClabels.size());
+      }
+      float tmpMaxAmplitude = 0;
+      int32_t tmpindex = 0;
+      for (auto& mclabel : cellMClabels[iCell]) {
+        // do not fill noise lables!
+        if (mclabel.isValid()) {
+          if (mEMCselectLeading) {
+            if (mclabel.getAmplitudeFraction() > tmpMaxAmplitude) {
+              // Check if this MCparticle added to be kept?
+              if (mToStore.at(mclabel.getSourceID()).at(mclabel.getEventID()).find(mclabel.getTrackID()) !=
+                  mToStore.at(mclabel.getSourceID()).at(mclabel.getEventID()).end()) {
+                tmpMaxAmplitude = mclabel.getAmplitudeFraction();
+                tmpindex = (mToStore.at(mclabel.getSourceID()).at(mclabel.getEventID())).at(mclabel.getTrackID());
+              }
+            }
+          } else {
+            auto trackStore = mToStore.at(mclabel.getSourceID()).at(mclabel.getEventID());
+            auto iter = trackStore.find(mclabel.getTrackID());
+            if (iter != trackStore.end()) {
+              amplitudeFraction.emplace_back(mclabel.getAmplitudeFraction());
+              particleIds.emplace_back(iter->second);
+            } else {
+              particleIds.emplace_back(-1); // should the mc particle not be in mToStore make sure something (e.g. -1) is saved in particleIds so the length of particleIds is the same es amplitudeFraction!
+              amplitudeFraction.emplace_back(0.f);
+              LOG(warn) << "CaloTable: Could not find track for mclabel (" << mclabel.getSourceID() << "," << mclabel.getEventID() << "," << mclabel.getTrackID() << ") in the AOD MC store";
+              if (mMCKineReader) {
+                auto mctrack = mMCKineReader->getTrack(mclabel);
+                TVector3 vec;
+                mctrack->GetStartVertex(vec);
+                LOG(warn) << " ... this track is of PDG " << mctrack->GetPdgCode() << " produced by " << mctrack->getProdProcessAsString() << " at (" << vec.X() << "," << vec.Y() << "," << vec.Z() << ")";
+              }
+            }
+          }
+        }
+      } // end of loop over all MC Labels for the current cell
+      if (mEMCselectLeading) {
+        amplitudeFraction.emplace_back(tmpMaxAmplitude);
+        particleIds.emplace_back(tmpindex);
+      }
+      if (particleIds.size() == 0) {
+        particleIds.emplace_back(-1);
+        amplitudeFraction.emplace_back(0.f);
+      }
+      mcCaloCellLabelCursor(particleIds,
+                            amplitudeFraction);
+    }
+  } // end of loop over cells in current event
+}
+
 // fill calo related tables (cells and calotrigger table)
-template 
-void AODProducerWorkflowDPL::fillCaloTable(TEventHandler* caloEventHandler, const TCaloCells& calocells, const TCaloTriggerRecord& caloCellTRGR,
-                                           const TCaloCursor& caloCellCursor, const TCaloTRGTableCursor& caloCellTRGTableCursor,
-                                           std::map& bcsMap, int8_t caloType)
+template 
+void AODProducerWorkflowDPL::fillCaloTable(TCaloCursor& caloCellCursor, TCaloTRGCursor& caloTRGCursor,
+                                           TMCCaloLabelCursor& mcCaloCellLabelCursor, const std::map& bcsMap,
+                                           const o2::globaltracking::RecoContainer& data)
 {
-  uint64_t globalBC = 0;    // global BC ID
-  uint64_t globalBCRel = 0; // BC id reltive to minGlBC (from FIT)
+  // get calo information
+  auto caloEMCCells = data.getEMCALCells();
+  auto caloEMCCellsTRGR = data.getEMCALTriggers();
+  auto mcCaloEMCCellLabels = data.getEMCALCellsMCLabels();
+
+  auto caloPHOSCells = data.getPHOSCells();
+  auto caloPHOSCellsTRGR = data.getPHOSTriggers();
+  auto mcCaloPHOSCellLabels = data.getPHOSCellsMCLabels();
+
+  if (!mInputSources[GIndex::PHS]) {
+    caloPHOSCells = {};
+    caloPHOSCellsTRGR = {};
+    mcCaloPHOSCellLabels = {};
+  }
+
+  if (!mInputSources[GIndex::EMC]) {
+    caloEMCCells = {};
+    caloEMCCellsTRGR = {};
+    mcCaloEMCCellLabels = {};
+  }
+
+  o2::emcal::EventHandler emcEventHandler;
+  o2::phos::EventHandler phsEventHandler;
 
   // get cell belonging to an eveffillnt instead of timeframe
-  caloEventHandler->reset();
-  caloEventHandler->setCellData(calocells, caloCellTRGR);
+  emcEventHandler.reset();
+  emcEventHandler.setCellData(caloEMCCells, caloEMCCellsTRGR);
+  emcEventHandler.setCellMCTruthContainer(mcCaloEMCCellLabels);
 
-  // loop over events
-  for (int iev = 0; iev < caloEventHandler->getNumberOfEvents(); iev++) {
-    auto inputEvent = caloEventHandler->buildEvent(iev);
-    auto cellsInEvent = inputEvent.mCells;                  // get cells belonging to current event
-    auto interactionRecord = inputEvent.mInteractionRecord; // get interaction records belonging to current event
+  phsEventHandler.reset();
+  phsEventHandler.setCellData(caloPHOSCells, caloPHOSCellsTRGR);
+  phsEventHandler.setCellMCTruthContainer(mcCaloPHOSCellLabels);
 
-    globalBC = interactionRecord.toLong();
+  int emcNEvents = emcEventHandler.getNumberOfEvents();
+  int phsNEvents = phsEventHandler.getNumberOfEvents();
+
+  std::vector> caloEvents; // 
+
+  caloEvents.reserve(emcNEvents + phsNEvents);
+
+  for (int iev = 0; iev < emcNEvents; ++iev) {
+    uint64_t bc = emcEventHandler.getInteractionRecordForEvent(iev).toLong();
+    caloEvents.emplace_back(std::make_tuple(bc, 1, iev));
+  }
+
+  for (int iev = 0; iev < phsNEvents; ++iev) {
+    uint64_t bc = phsEventHandler.getInteractionRecordForEvent(iev).toLong();
+    caloEvents.emplace_back(std::make_tuple(bc, 0, iev));
+  }
+
+  std::sort(caloEvents.begin(), caloEvents.end(),
+            [](const auto& left, const auto& right) { return std::get<0>(left) < std::get<0>(right); });
+
+  // loop over events
+  for (int i = 0; i < emcNEvents + phsNEvents; ++i) {
+    uint64_t globalBC = std::get<0>(caloEvents[i]);
+    int8_t caloType = std::get<1>(caloEvents[i]);
+    int eventID = std::get<2>(caloEvents[i]);
     auto item = bcsMap.find(globalBC);
     int bcID = -1;
     if (item != bcsMap.end()) {
@@ -1112,44 +1679,70 @@ void AODProducerWorkflowDPL::fillCaloTable(TEventHandler* caloEventHandler, cons
     } else {
       LOG(warn) << "Error: could not find a corresponding BC ID for a calo point; globalBC = " << globalBC << ", caloType = " << (int)caloType;
     }
-
-    // loop over all cells in collision
-    for (auto& cell : cellsInEvent) {
-      caloCellCursor(0,
-                     bcID,
-                     CellHelper::getCellNumber(cell),
-                     truncateFloatFraction(CellHelper::getAmplitude(cell), mCaloAmp),
-                     truncateFloatFraction(CellHelper::getTimeStamp(cell), mCaloTime),
-                     cell.getType(),
-                     caloType); // 1 = emcal, -1 = undefined, 0 = phos
-
-      // todo: fix dummy values in CellHelper when it is clear what is filled for trigger information
-      if (CellHelper::isTRU(cell)) { // Write only trigger cells into this table
-        caloCellTRGTableCursor(0,
-                               bcID,
-                               CellHelper::getFastOrAbsID(cell),
-                               CellHelper::getLnAmplitude(cell),
-                               CellHelper::getTriggerBits(cell),
-                               caloType);
-      }
+    if (caloType == 0) { // phos
+      addToCaloTable(phsEventHandler, caloCellCursor, caloTRGCursor, mcCaloCellLabelCursor, eventID, bcID, caloType);
+    }
+    if (caloType == 1) { // emc
+      addToCaloTable(emcEventHandler, caloCellCursor, caloTRGCursor, mcCaloCellLabelCursor, eventID, bcID, caloType);
     }
   }
+
+  caloEvents.clear();
 }
 
 void AODProducerWorkflowDPL::init(InitContext& ic)
 {
   mTimer.Stop();
   o2::base::GRPGeomHelper::instance().setRequest(mGGCCDBRequest);
-  mLPMProdTag = ic.options().get("lpmp-prod-tag");
-  mAnchorPass = ic.options().get("anchor-pass");
-  mAnchorProd = ic.options().get("anchor-prod");
-  mRecoPass = ic.options().get("reco-pass");
+  mLPMProdTag = ic.options().get("lpmp-prod-tag");
+  mAnchorPass = ic.options().get("anchor-pass");
+  mAnchorProd = ic.options().get("anchor-prod");
+  mUser = ic.options().get("created-by");
+  mRecoPass = ic.options().get("reco-pass");
   mTFNumber = ic.options().get("aod-timeframe-id");
   mRecoOnly = ic.options().get("reco-mctracks-only");
   mTruncate = ic.options().get("enable-truncation");
   mRunNumber = ic.options().get("run-number");
   mCTPReadout = ic.options().get("ctpreadout-create");
-
+  mNThreads = std::max(1, ic.options().get("nthreads"));
+  mEMCselectLeading = ic.options().get("emc-select-leading");
+  mThinTracks = ic.options().get("thin-tracks");
+  mPropTracks = ic.options().get("propagate-tracks");
+  mMaxPropXiu = ic.options().get("propagate-tracks-max-xiu");
+  mPropMuons = ic.options().get("propagate-muons");
+  if (auto s = ic.options().get("with-streamers"); !s.empty()) {
+    mStreamerFlags.set(s);
+    if (mStreamerFlags) {
+      LOGP(info, "Writing streamer data with mask:");
+      LOG(info) << mStreamerFlags;
+    } else {
+      LOGP(warn, "Specified non-default empty streamer mask!");
+    }
+  }
+  mTrackQCKeepGlobalTracks = ic.options().get("trackqc-keepglobaltracks");
+  mTrackQCRetainOnlydEdx = ic.options().get("trackqc-retainonlydedx");
+  mTrackQCFraction = ic.options().get("trackqc-fraction");
+  mTrackQCNTrCut = ic.options().get("trackqc-NTrCut");
+  mTrackQCDCAxy = ic.options().get("trackqc-tpc-dca");
+  mTrackQCPt = ic.options().get("trackqc-tpc-pt");
+  mTrackQCNCls = ic.options().get("trackqc-tpc-cls");
+  if (auto seed = ic.options().get("seed"); seed == 0) {
+    LOGP(info, "Using random device for seeding");
+    std::random_device rd;
+    std::array seed_data{};
+    std::generate(std::begin(seed_data), std::end(seed_data), std::ref(rd));
+    std::seed_seq seq(std::begin(seed_data), std::end(seed_data));
+    mGenerator = std::mt19937(seq);
+  } else {
+    LOGP(info, "Using seed {} for sampling", seed);
+    mGenerator.seed(seed);
+  }
+#ifdef WITH_OPENMP
+  LOGP(info, "Multi-threaded parts will run with {} OpenMP threads", mNThreads);
+#else
+  mNThreads = 1;
+  LOG(info) << "OpenMP is disabled";
+#endif
   if (mTFNumber == -1L) {
     LOG(info) << "TFNumber will be obtained from CCDB";
   }
@@ -1157,6 +1750,8 @@ void AODProducerWorkflowDPL::init(InitContext& ic)
     LOG(info) << "The Run number will be obtained from DPL headers";
   }
 
+  mUseSigFiltMC = ic.options().get("mc-signal-filt");
+
   // set no truncation if selected by user
   if (mTruncate != 1) {
     LOG(info) << "Truncation is not used!";
@@ -1172,14 +1767,17 @@ void AODProducerWorkflowDPL::init(InitContext& ic)
     mTrackCovOffDiag = 0xFFFFFFFF;
     mTrackSignal = 0xFFFFFFFF;
     mTrackTime = 0xFFFFFFFF;
+    mTPCTime0 = 0xFFFFFFFF;
     mTrackTimeError = 0xFFFFFFFF;
     mTrackPosEMCAL = 0xFFFFFFFF;
     mTracklets = 0xFFFFFFFF;
     mMcParticleW = 0xFFFFFFFF;
     mMcParticlePos = 0xFFFFFFFF;
     mMcParticleMom = 0xFFFFFFFF;
-    mCaloAmp = 0xFFFFFFFF;  // todo check which truncation should actually be used
-    mCaloTime = 0xFFFFFFFF; // todo check which truncation should actually be used
+    mCaloAmp = 0xFFFFFFFF;
+    mCaloTime = 0xFFFFFFFF;
+    mCPVPos = 0xFFFFFFFF;
+    mCPVAmpl = 0xFFFFFFFF;
     mMuonTr1P = 0xFFFFFFFF;
     mMuonTrThetaX = 0xFFFFFFFF;
     mMuonTrThetaY = 0xFFFFFFFF;
@@ -1190,49 +1788,69 @@ void AODProducerWorkflowDPL::init(InitContext& ic)
     mMuonCl = 0xFFFFFFFF;
     mMuonClErr = 0xFFFFFFFF;
     mV0Time = 0xFFFFFFFF;
+    mV0ChannelTime = 0xFFFFFFFF;
     mFDDTime = 0xFFFFFFFF;
+    mFDDChannelTime = 0xFFFFFFFF;
     mT0Time = 0xFFFFFFFF;
+    mT0ChannelTime = 0xFFFFFFFF;
     mV0Amplitude = 0xFFFFFFFF;
     mFDDAmplitude = 0xFFFFFFFF;
     mT0Amplitude = 0xFFFFFFFF;
   }
-
-  // initialize zdc helper maps
-  for (auto& ChannelName : o2::zdc::ChannelNames) {
-    mZDCEnergyMap[(string)ChannelName] = 0;
-    mZDCTDCMap[(string)ChannelName] = 999;
+  // Initialize ZDC helper maps
+  for (int ic = 0; ic < o2::zdc::NChannels; ic++) {
+    mZDCEnergyMap[ic] = -std::numeric_limits::infinity();
   }
-
-  // writing metadata if it's not yet in AOD file
-  // note: `--aod-writer-resmode "UPDATE"` has to be used,
-  //       so that metadata is not overwritten
-  mResFile += ".root";
-  auto* fResFile = TFile::Open(mResFile, "UPDATE");
-  if (!fResFile) {
-    LOGF(fatal, "Could not open file %s", mResFile);
+  for (int ic = 0; ic < o2::zdc::NTDCChannels; ic++) {
+    mZDCTDCMap[ic] = -std::numeric_limits::infinity();
   }
-  if (fResFile->FindObjectAny("metaData")) {
-    LOGF(warning, "Metadata: target file %s already has metadata, preserving it", mResFile);
-  } else {
-    // populating metadata map
-    TString dataType = mUseMC ? "MC" : "RAW";
-    mMetaData.Add(new TObjString("DataType"), new TObjString(dataType));
-    mMetaData.Add(new TObjString("Run"), new TObjString("3"));
-    TString O2Version = o2::fullVersion();
-    TString ROOTVersion = ROOT_RELEASE;
-    mMetaData.Add(new TObjString("O2Version"), new TObjString(O2Version));
-    mMetaData.Add(new TObjString("ROOTVersion"), new TObjString(ROOTVersion));
-    mMetaData.Add(new TObjString("RecoPassName"), new TObjString(mRecoPass));
-    mMetaData.Add(new TObjString("AnchorProduction"), new TObjString(mAnchorProd));
-    mMetaData.Add(new TObjString("AnchorPassName"), new TObjString(mAnchorPass));
-    mMetaData.Add(new TObjString("LPMProductionTag"), new TObjString(mLPMProdTag));
-    LOGF(info, "Metadata: writing into %s", mResFile);
-    fResFile->WriteObject(&mMetaData, "metaData", "Overwrite");
-  }
-  fResFile->Close();
+
+  std::string hepmcUpdate = ic.options().get("hepmc-update");
+  HepMCUpdate when = (hepmcUpdate == "never" ? HepMCUpdate::never : hepmcUpdate == "always" ? HepMCUpdate::always
+                                                                  : hepmcUpdate == "all"    ? HepMCUpdate::allKeys
+                                                                                            : HepMCUpdate::anyKey);
+  mXSectionUpdate = when;
+  mPdfInfoUpdate = when;
+  mHeavyIonUpdate = when;
 
   mTimer.Reset();
+
+  if (mStreamerFlags) {
+    mStreamer = std::make_unique("AO2DStreamer.root", "RECREATE");
+  }
+}
+
+namespace
+{
+void add_additional_meta_info(std::vector& keys, std::vector& values)
+{
+  // see if we should put additional meta info (e.g. from MC)
+  auto aod_external_meta_info_file = getenv("AOD_ADDITIONAL_METADATA_FILE");
+  if (aod_external_meta_info_file != nullptr) {
+    LOG(info) << "Trying to inject additional AOD meta-data from " << aod_external_meta_info_file;
+    if (std::filesystem::exists(aod_external_meta_info_file)) {
+      std::ifstream input_file(aod_external_meta_info_file);
+      if (input_file) {
+        nlohmann::json json_data;
+        try {
+          input_file >> json_data;
+        } catch (nlohmann::json::parse_error& e) {
+          std::cerr << "JSON Parse Error: " << e.what() << "\n";
+          std::cerr << "Exception ID: " << e.id << "\n";
+          std::cerr << "Byte position: " << e.byte << "\n";
+          return;
+        }
+        // If parsing succeeds, iterate over key-value pairs
+        for (const auto& [key, value] : json_data.items()) {
+          LOG(info) << "Adding AOD MetaData" << key << " : " << value;
+          keys.push_back(key.c_str());
+          values.push_back(value.get());
+        }
+      }
+    }
+  }
 }
+} // namespace
 
 void AODProducerWorkflowDPL::run(ProcessingContext& pc)
 {
@@ -1259,89 +1877,99 @@ void AODProducerWorkflowDPL::run(ProcessingContext& pc)
   auto zdcBCRecData = recoData.getZDCBCRecData();
   auto zdcTDCData = recoData.getZDCTDCData();
 
-  // get calo information
-  auto caloEMCCells = recoData.getEMCALCells();
-  auto caloEMCCellsTRGR = recoData.getEMCALTriggers();
+  auto cpvClusters = recoData.getCPVClusters();
+  auto cpvTrigRecs = recoData.getCPVTriggers();
 
-  auto caloPHOSCells = recoData.getPHOSCells();
-  auto caloPHOSCellsTRGR = recoData.getPHOSTriggers();
   auto ctpDigits = recoData.getCTPDigits();
   const auto& tinfo = pc.services().get();
   std::vector ctpDigitsCreated;
   if (mCTPReadout == 1) {
+    LOG(info) << "CTP : creating ctpreadout in AOD producer";
     createCTPReadout(recoData, ctpDigitsCreated, pc);
+    LOG(info) << "CTP : ctpreadout created from AOD";
     ctpDigits = gsl::span(ctpDigitsCreated);
   }
   LOG(debug) << "FOUND " << primVertices.size() << " primary vertices";
   LOG(debug) << "FOUND " << ft0RecPoints.size() << " FT0 rec. points";
   LOG(debug) << "FOUND " << fv0RecPoints.size() << " FV0 rec. points";
   LOG(debug) << "FOUND " << fddRecPoints.size() << " FDD rec. points";
-  LOG(debug) << "FOUND " << caloEMCCells.size() << " EMC cells";
-  LOG(debug) << "FOUND " << caloEMCCellsTRGR.size() << " EMC Trigger Records";
-
-  auto& bcBuilder = pc.outputs().make(Output{"AOD", "BC"});
-  auto& cascadesBuilder = pc.outputs().make(Output{"AOD", "CASCADE_001"});
-  auto& collisionsBuilder = pc.outputs().make(Output{"AOD", "COLLISION"});
-  auto& fddBuilder = pc.outputs().make(Output{"AOD", "FDD_001"});
-  auto& ft0Builder = pc.outputs().make(Output{"AOD", "FT0"});
-  auto& fv0aBuilder = pc.outputs().make(Output{"AOD", "FV0A"});
-  auto& fwdTracksBuilder = pc.outputs().make(Output{"AOD", "FWDTRACK"});
-  auto& fwdTracksCovBuilder = pc.outputs().make(Output{"AOD", "FWDTRACKCOV"});
-  auto& mcColLabelsBuilder = pc.outputs().make(Output{"AOD", "MCCOLLISIONLABEL"});
-  auto& mcCollisionsBuilder = pc.outputs().make(Output{"AOD", "MCCOLLISION"});
-  auto& mcMFTTrackLabelBuilder = pc.outputs().make(Output{"AOD", "MCMFTTRACKLABEL"});
-  auto& mcFwdTrackLabelBuilder = pc.outputs().make(Output{"AOD", "MCFWDTRACKLABEL"});
-  auto& mcParticlesBuilder = pc.outputs().make(Output{"AOD", "MCPARTICLE_001"});
-  auto& mcTrackLabelBuilder = pc.outputs().make(Output{"AOD", "MCTRACKLABEL"});
-  auto& mftTracksBuilder = pc.outputs().make(Output{"AOD", "MFTTRACK"});
-  auto& tracksBuilder = pc.outputs().make(Output{"AOD", "TRACK_IU"});
-  auto& tracksCovBuilder = pc.outputs().make(Output{"AOD", "TRACKCOV_IU"});
-  auto& tracksExtraBuilder = pc.outputs().make(Output{"AOD", "TRACKEXTRA"});
-  auto& ambigTracksBuilder = pc.outputs().make(Output{"AOD", "AMBIGUOUSTRACK"});
-  auto& ambigMFTTracksBuilder = pc.outputs().make(Output{"AOD", "AMBIGUOUSMFTTR"});
-  auto& ambigFwdTracksBuilder = pc.outputs().make(Output{"AOD", "AMBIGUOUSFWDTR"});
-  auto& v0sBuilder = pc.outputs().make(Output{"AOD", "V0_001"});
-  auto& zdcBuilder = pc.outputs().make(Output{"AOD", "ZDC"});
-  auto& caloCellsBuilder = pc.outputs().make(Output{"AOD", "CALO"});
-  auto& caloCellsTRGTableBuilder = pc.outputs().make(Output{"AOD", "CALOTRIGGER"});
-  auto& originTableBuilder = pc.outputs().make(Output{"AOD", "ORIGIN"});
-
-  auto bcCursor = bcBuilder.cursor();
-  auto cascadesCursor = cascadesBuilder.cursor();
-  auto collisionsCursor = collisionsBuilder.cursor();
-  auto fddCursor = fddBuilder.cursor();
-  auto ft0Cursor = ft0Builder.cursor();
-  auto fv0aCursor = fv0aBuilder.cursor();
-  auto fwdTracksCursor = fwdTracksBuilder.cursor();
-  auto fwdTracksCovCursor = fwdTracksCovBuilder.cursor();
-  auto mcColLabelsCursor = mcColLabelsBuilder.cursor();
-  auto mcCollisionsCursor = mcCollisionsBuilder.cursor();
-  auto mcMFTTrackLabelCursor = mcMFTTrackLabelBuilder.cursor();
-  auto mcFwdTrackLabelCursor = mcFwdTrackLabelBuilder.cursor();
-  auto mcParticlesCursor = mcParticlesBuilder.cursor();
-  auto mcTrackLabelCursor = mcTrackLabelBuilder.cursor();
-  auto mftTracksCursor = mftTracksBuilder.cursor();
-  auto tracksCovCursor = tracksCovBuilder.cursor();
-  auto tracksCursor = tracksBuilder.cursor();
-  auto tracksExtraCursor = tracksExtraBuilder.cursor();
-  auto ambigTracksCursor = ambigTracksBuilder.cursor();
-  auto ambigMFTTracksCursor = ambigMFTTracksBuilder.cursor();
-  auto ambigFwdTracksCursor = ambigFwdTracksBuilder.cursor();
-  auto v0sCursor = v0sBuilder.cursor();
-  auto zdcCursor = zdcBuilder.cursor();
-  auto caloCellsCursor = caloCellsBuilder.cursor();
-  auto caloCellsTRGTableCursor = caloCellsTRGTableBuilder.cursor();
-  auto originCursor = originTableBuilder.cursor();
+  LOG(debug) << "FOUND " << cpvClusters.size() << " CPV clusters";
+  LOG(debug) << "FOUND " << cpvTrigRecs.size() << " CPV trigger records";
+
+  LOG(info) << "FOUND " << primVertices.size() << " primary vertices";
+
+  using namespace o2::aodhelpers;
+
+  auto bcCursor = createTableCursor(pc);
+  auto bcFlagsCursor = createTableCursor(pc);
+  auto cascadesCursor = createTableCursor(pc);
+  auto collisionsCursor = createTableCursor(pc);
+  auto decay3BodyCursor = createTableCursor(pc);
+  auto trackedCascadeCursor = createTableCursor(pc);
+  auto trackedV0Cursor = createTableCursor(pc);
+  auto tracked3BodyCurs = createTableCursor(pc);
+  auto fddCursor = createTableCursor(pc);
+  auto fddExtraCursor = createTableCursor(pc);
+  auto ft0Cursor = createTableCursor(pc);
+  auto ft0ExtraCursor = createTableCursor(pc);
+  auto fv0aCursor = createTableCursor(pc);
+  auto fv0aExtraCursor = createTableCursor(pc);
+  auto fwdTracksCursor = createTableCursor(pc);
+  auto fwdTracksCovCursor = createTableCursor(pc);
+  auto fwdTrkClsCursor = createTableCursor(pc);
+  auto mftTracksCursor = createTableCursor(pc);
+  auto mftTracksCovCursor = createTableCursor(pc);
+  auto tracksCursor = createTableCursor(pc);
+  auto tracksCovCursor = createTableCursor(pc);
+  auto tracksExtraCursor = createTableCursor(pc);
+  auto tracksQACursor = createTableCursor(pc);
+  auto ambigTracksCursor = createTableCursor(pc);
+  auto ambigMFTTracksCursor = createTableCursor(pc);
+  auto ambigFwdTracksCursor = createTableCursor(pc);
+  auto v0sCursor = createTableCursor(pc);
+  auto zdcCursor = createTableCursor(pc);
+  auto hmpCursor = createTableCursor(pc);
+  auto caloCellsCursor = createTableCursor(pc);
+  auto caloCellsTRGTableCursor = createTableCursor(pc);
+  auto cpvClustersCursor = createTableCursor(pc);
+  auto originCursor = createTableCursor(pc);
+
+  // Declare MC cursors type without adding the output for a table
+  o2::framework::Produces mcColLabelsCursor;
+  o2::framework::Produces mcCollisionsCursor;
+  o2::framework::Produces hepmcXSectionsCursor;
+  o2::framework::Produces hepmcPdfInfosCursor;
+  o2::framework::Produces hepmcHeavyIonsCursor;
+  o2::framework::Produces mcMFTTrackLabelCursor;
+  o2::framework::Produces mcFwdTrackLabelCursor;
+  o2::framework::Produces mcParticlesCursor;
+  o2::framework::Produces mcTrackLabelCursor;
+  o2::framework::Produces mcCaloLabelsCursor;
+  if (mUseMC) { // This creates the actual writercursor
+    mcColLabelsCursor = createTableCursor(pc);
+    mcCollisionsCursor = createTableCursor(pc);
+    hepmcXSectionsCursor = createTableCursor(pc);
+    hepmcPdfInfosCursor = createTableCursor(pc);
+    hepmcHeavyIonsCursor = createTableCursor(pc);
+    mcMFTTrackLabelCursor = createTableCursor(pc);
+    mcFwdTrackLabelCursor = createTableCursor(pc);
+    mcParticlesCursor = createTableCursor(pc);
+    mcTrackLabelCursor = createTableCursor(pc);
+    mcCaloLabelsCursor = createTableCursor(pc);
+  }
 
   std::unique_ptr mcReader;
   if (mUseMC) {
     mcReader = std::make_unique("collisioncontext.root");
   }
+  mMCKineReader = mcReader.get(); // for use in different functions
   std::map bcsMap;
   collectBCs(recoData, mUseMC ? mcReader->getDigitizationContext()->getEventRecords() : std::vector{}, bcsMap);
   if (!primVer2TRefs.empty()) { // if the vertexing was done, the last slot refers to orphan tracks
     addRefGlobalBCsForTOF(primVer2TRefs.back(), primVerGIs, recoData, bcsMap);
   }
+  // initialize the bunch crossing container for further use below
+  mBCLookup.init(bcsMap);
 
   uint64_t tfNumber;
   const int runNumber = (mRunNumber == -1) ? int(tinfo.runNumber) : mRunNumber;
@@ -1352,15 +1980,18 @@ void AODProducerWorkflowDPL::run(ProcessingContext& pc)
     tfNumber = mTFNumber;
   }
 
-  std::vector aAmplitudes;
+  std::vector aAmplitudes, aTimes;
   std::vector aChannels;
+  fv0aCursor.reserve(fv0RecPoints.size());
   for (auto& fv0RecPoint : fv0RecPoints) {
     aAmplitudes.clear();
     aChannels.clear();
+    aTimes.clear();
     const auto channelData = fv0RecPoint.getBunchChannelData(fv0ChData);
     for (auto& channel : channelData) {
       if (channel.charge > 0) {
         aAmplitudes.push_back(truncateFloatFraction(channel.charge, mV0Amplitude));
+        aTimes.push_back(truncateFloatFraction(channel.time * 1.E-3, mV0ChannelTime));
         aChannels.push_back(channel.channel);
       }
     }
@@ -1372,14 +2003,21 @@ void AODProducerWorkflowDPL::run(ProcessingContext& pc)
     } else {
       LOG(fatal) << "Error: could not find a corresponding BC ID for a FV0 rec. point; BC = " << bc;
     }
-    fv0aCursor(0,
-               bcID,
+    fv0aCursor(bcID,
                aAmplitudes,
                aChannels,
                truncateFloatFraction(fv0RecPoint.getCollisionGlobalMeanTime() * 1E-3, mV0Time), // ps to ns
                fv0RecPoint.getTrigger().getTriggersignals());
+
+    if (mEnableFITextra) {
+      fv0aExtraCursor(bcID,
+                      aTimes);
+    }
   }
 
+  std::vector zdcEnergy, zdcAmplitudes, zdcTime;
+  std::vector zdcChannelsE, zdcChannelsT;
+  zdcCursor.reserve(zdcBCRecData.size());
   for (auto zdcRecData : zdcBCRecData) {
     uint64_t bc = zdcRecData.ir.toLong();
     auto item = bcsMap.find(bc);
@@ -1389,134 +2027,128 @@ void AODProducerWorkflowDPL::run(ProcessingContext& pc)
     } else {
       LOG(fatal) << "Error: could not find a corresponding BC ID for a ZDC rec. point; BC = " << bc;
     }
-    float energyZEM1 = 0;
-    float energyZEM2 = 0;
-    float energyCommonZNA = 0;
-    float energyCommonZNC = 0;
-    float energyCommonZPA = 0;
-    float energyCommonZPC = 0;
-    float energySectorZNA[4] = {0.};
-    float energySectorZNC[4] = {0.};
-    float energySectorZPA[4] = {0.};
-    float energySectorZPC[4] = {0.};
     int fe, ne, ft, nt, fi, ni;
     zdcRecData.getRef(fe, ne, ft, nt, fi, ni);
+    zdcEnergy.clear();
+    zdcChannelsE.clear();
+    zdcAmplitudes.clear();
+    zdcTime.clear();
+    zdcChannelsT.clear();
     for (int ie = 0; ie < ne; ie++) {
       auto& zdcEnergyData = zdcEnergies[fe + ie];
-      float energy = zdcEnergyData.energy();
-      string chName = o2::zdc::channelName(zdcEnergyData.ch());
-      mZDCEnergyMap.at(chName) = energy;
+      zdcEnergy.emplace_back(zdcEnergyData.energy());
+      zdcChannelsE.emplace_back(zdcEnergyData.ch());
     }
     for (int it = 0; it < nt; it++) {
       auto& tdc = zdcTDCData[ft + it];
-      float tdcValue = tdc.value();
-      int channelID = o2::zdc::TDCSignal[tdc.ch()];
-      auto channelName = o2::zdc::ChannelNames[channelID];
-      mZDCTDCMap.at((string)channelName) = tdcValue;
-    }
-    energySectorZNA[0] = mZDCEnergyMap.at("ZNA1");
-    energySectorZNA[1] = mZDCEnergyMap.at("ZNA2");
-    energySectorZNA[2] = mZDCEnergyMap.at("ZNA3");
-    energySectorZNA[3] = mZDCEnergyMap.at("ZNA4");
-    energySectorZNC[0] = mZDCEnergyMap.at("ZNC1");
-    energySectorZNC[1] = mZDCEnergyMap.at("ZNC2");
-    energySectorZNC[2] = mZDCEnergyMap.at("ZNC3");
-    energySectorZNC[3] = mZDCEnergyMap.at("ZNC4");
-    energySectorZPA[0] = mZDCEnergyMap.at("ZPA1");
-    energySectorZPA[1] = mZDCEnergyMap.at("ZPA2");
-    energySectorZPA[2] = mZDCEnergyMap.at("ZPA3");
-    energySectorZPA[3] = mZDCEnergyMap.at("ZPA4");
-    energySectorZPC[0] = mZDCEnergyMap.at("ZPC1");
-    energySectorZPC[1] = mZDCEnergyMap.at("ZPC2");
-    energySectorZPC[2] = mZDCEnergyMap.at("ZPC3");
-    energySectorZPC[3] = mZDCEnergyMap.at("ZPC4");
-    zdcCursor(0,
-              bcID,
-              mZDCEnergyMap.at("ZEM1"),
-              mZDCEnergyMap.at("ZEM2"),
-              mZDCEnergyMap.at("ZNAC"),
-              mZDCEnergyMap.at("ZNCC"),
-              mZDCEnergyMap.at("ZPAC"),
-              mZDCEnergyMap.at("ZPCC"),
-              energySectorZNA,
-              energySectorZNC,
-              energySectorZPA,
-              energySectorZPC,
-              mZDCTDCMap.at("ZEM1"),
-              mZDCTDCMap.at("ZEM2"),
-              mZDCTDCMap.at("ZNAC"),
-              mZDCTDCMap.at("ZNCC"),
-              mZDCTDCMap.at("ZPAC"),
-              mZDCTDCMap.at("ZPCC"));
+      zdcAmplitudes.emplace_back(tdc.amplitude());
+      zdcTime.emplace_back(tdc.value());
+      zdcChannelsT.emplace_back(o2::zdc::TDCSignal[tdc.ch()]);
+    }
+    zdcCursor(bcID,
+              zdcEnergy,
+              zdcChannelsE,
+              zdcAmplitudes,
+              zdcTime,
+              zdcChannelsT);
   }
 
   // keep track event/source id for each mc-collision
   // using map and not unordered_map to ensure
   // correct ordering when iterating over container elements
-  std::map, int> mcColToEvSrc;
+  std::vector> mcColToEvSrc;
 
   if (mUseMC) {
-    // TODO: figure out collision weight
-    float mcColWeight = 1.;
+    using namespace o2::aodmchelpers;
+
     // filling mcCollision table
     int nMCCollisions = mcReader->getDigitizationContext()->getNCollisions();
     const auto& mcRecords = mcReader->getDigitizationContext()->getEventRecords();
     const auto& mcParts = mcReader->getDigitizationContext()->getEventParts();
+
+    // if signal filtering enabled, let's check if there are more than one source; otherwise fatalise
+    if (mUseSigFiltMC) {
+      std::vector sourceIDs{};
+      for (int iCol = 0; iCol < nMCCollisions; iCol++) {
+        for (auto const& colPart : mcParts[iCol]) {
+          int sourceID = colPart.sourceID;
+          if (std::find(sourceIDs.begin(), sourceIDs.end(), sourceID) == sourceIDs.end()) {
+            sourceIDs.push_back(sourceID);
+          }
+          if (sourceIDs.size() > 1) { // we found more than one, exit
+            break;
+          }
+        }
+        if (sourceIDs.size() > 1) { // we found more than one, exit
+          break;
+        }
+      }
+      if (sourceIDs.size() <= 1) {
+        LOGP(fatal, "Signal filtering cannot be enabled without embedding. Please fix the configuration either enabling the embedding, or turning off the signal filtering.");
+      }
+    }
+
+    // count all parts
+    int totalNParts = 0;
+    for (int iCol = 0; iCol < nMCCollisions; iCol++) {
+      totalNParts += mcParts[iCol].size();
+    }
+    mcCollisionsCursor.reserve(totalNParts);
+
     for (int iCol = 0; iCol < nMCCollisions; iCol++) {
-      auto time = mcRecords[iCol].getTimeNS();
+      const auto time = mcRecords[iCol].getTimeOffsetWrtBC();
       auto globalBC = mcRecords[iCol].toLong();
       auto item = bcsMap.find(globalBC);
       int bcID = -1;
       if (item != bcsMap.end()) {
         bcID = item->second;
       } else {
-        LOG(fatal) << "Error: could not find a corresponding BC ID for MC collision; BC = " << globalBC << ", mc collision = " << iCol;
+        LOG(fatal) << "Error: could not find a corresponding BC ID "
+                   << "for MC collision; BC = " << globalBC
+                   << ", mc collision = " << iCol;
       }
       auto& colParts = mcParts[iCol];
       auto nParts = colParts.size();
       for (auto colPart : colParts) {
         auto eventID = colPart.entryID;
         auto sourceID = colPart.sourceID;
-        // enable embedding: if several colParts exist, then they are saved as one collision
+        // enable embedding: if several colParts exist, then they are
+        // saved as one collision
         if (nParts == 1 || sourceID == 0) {
           // FIXME:
           // use generators' names for generatorIDs (?)
-          short generatorID = sourceID;
           auto& header = mcReader->getMCEventHeader(sourceID, eventID);
-          mcCollisionsCursor(0,
-                             bcID,
-                             generatorID,
-                             truncateFloatFraction(header.GetX(), mCollisionPosition),
-                             truncateFloatFraction(header.GetY(), mCollisionPosition),
-                             truncateFloatFraction(header.GetZ(), mCollisionPosition),
-                             truncateFloatFraction(time, mCollisionPosition),
-                             truncateFloatFraction(mcColWeight, mCollisionPosition),
-                             header.GetB());
+          updateMCHeader(mcCollisionsCursor.cursor,
+                         hepmcXSectionsCursor.cursor,
+                         hepmcPdfInfosCursor.cursor,
+                         hepmcHeavyIonsCursor.cursor,
+                         header,
+                         iCol,
+                         bcID,
+                         time,
+                         0,
+                         sourceID);
         }
-        mcColToEvSrc.emplace(std::pair(eventID, sourceID), iCol); // point background and injected signal events to one collision
+        mcColToEvSrc.emplace_back(std::vector{iCol, sourceID, eventID}); // point background and injected signal events to one collision
       }
     }
   }
 
+  std::sort(mcColToEvSrc.begin(), mcColToEvSrc.end(),
+            [](const std::vector& left, const std::vector& right) { return (left[0] < right[0]); });
+
   // vector of FDD amplitudes
-  int16_t aFDDAmplitudesA[8] = {0u};
-  int16_t aFDDAmplitudesC[8] = {0u};
+  int16_t aFDDAmplitudesA[8] = {0u}, aFDDAmplitudesC[8] = {0u};
+  float aFDDTimesA[8] = {0.f}, aFDDTimesC[8] = {0.f};
   // filling FDD table
+  fddCursor.reserve(fddRecPoints.size());
   for (const auto& fddRecPoint : fddRecPoints) {
     for (int i = 0; i < 8; i++) {
       aFDDAmplitudesA[i] = 0;
       aFDDAmplitudesC[i] = 0;
+      aFDDTimesA[i] = 0.f;
+      aFDDTimesC[i] = 0.f;
     }
-
-    const auto channelData = fddRecPoint.getBunchChannelData(fddChData);
-    for (const auto& channel : channelData) {
-      if (channel.mPMNumber < 8) {
-        aFDDAmplitudesC[channel.mPMNumber] = channel.mChargeADC; // amplitude
-      } else {
-        aFDDAmplitudesA[channel.mPMNumber - 8] = channel.mChargeADC; // amplitude
-      }
-    }
-
     uint64_t globalBC = fddRecPoint.getInteractionRecord().toLong();
     uint64_t bc = globalBC;
     auto item = bcsMap.find(bc);
@@ -1526,21 +2158,39 @@ void AODProducerWorkflowDPL::run(ProcessingContext& pc)
     } else {
       LOG(fatal) << "Error: could not find a corresponding BC ID for a FDD rec. point; BC = " << bc;
     }
-    fddCursor(0,
-              bcID,
+    const auto channelData = fddRecPoint.getBunchChannelData(fddChData);
+    for (const auto& channel : channelData) {
+      if (channel.mPMNumber < 8) {
+        aFDDAmplitudesC[channel.mPMNumber] = channel.mChargeADC;                                      // amplitude
+        aFDDTimesC[channel.mPMNumber] = truncateFloatFraction(channel.mTime * 1E-3, mFDDChannelTime); // time
+      } else {
+        aFDDAmplitudesA[channel.mPMNumber - 8] = channel.mChargeADC;                                      // amplitude
+        aFDDTimesA[channel.mPMNumber - 8] = truncateFloatFraction(channel.mTime * 1E-3, mFDDChannelTime); // time
+      }
+    }
+
+    fddCursor(bcID,
               aFDDAmplitudesA,
               aFDDAmplitudesC,
               truncateFloatFraction(fddRecPoint.getCollisionTimeA() * 1E-3, mFDDTime), // ps to ns
               truncateFloatFraction(fddRecPoint.getCollisionTimeC() * 1E-3, mFDDTime), // ps to ns
               fddRecPoint.getTrigger().getTriggersignals());
+    if (mEnableFITextra) {
+      fddExtraCursor(bcID,
+                     aFDDTimesA,
+                     aFDDTimesC);
+    }
   }
 
   // filling FT0 table
-  std::vector aAmplitudesA, aAmplitudesC;
+  std::vector aAmplitudesA, aAmplitudesC, aTimesA, aTimesC;
   std::vector aChannelsA, aChannelsC;
+  ft0Cursor.reserve(ft0RecPoints.size());
   for (auto& ft0RecPoint : ft0RecPoints) {
     aAmplitudesA.clear();
     aAmplitudesC.clear();
+    aTimesA.clear();
+    aTimesC.clear();
     aChannelsA.clear();
     aChannelsC.clear();
     const auto channelData = ft0RecPoint.getBunchChannelData(ft0ChData);
@@ -1551,9 +2201,11 @@ void AODProducerWorkflowDPL::run(ProcessingContext& pc)
         if (channel.ChId < nFT0ChannelsAside) {
           aChannelsA.push_back(channel.ChId);
           aAmplitudesA.push_back(truncateFloatFraction(channel.QTCAmpl, mT0Amplitude));
+          aTimesA.push_back(truncateFloatFraction(channel.CFDTime * 1E-3, mT0ChannelTime));
         } else {
           aChannelsC.push_back(channel.ChId - nFT0ChannelsAside);
           aAmplitudesC.push_back(truncateFloatFraction(channel.QTCAmpl, mT0Amplitude));
+          aTimesC.push_back(truncateFloatFraction(channel.CFDTime * 1E-3, mT0ChannelTime));
         }
       }
     }
@@ -1566,8 +2218,7 @@ void AODProducerWorkflowDPL::run(ProcessingContext& pc)
     } else {
       LOG(fatal) << "Error: could not find a corresponding BC ID for a FT0 rec. point; BC = " << bc;
     }
-    ft0Cursor(0,
-              bcID,
+    ft0Cursor(bcID,
               aAmplitudesA,
               aChannelsA,
               aAmplitudesC,
@@ -1575,19 +2226,30 @@ void AODProducerWorkflowDPL::run(ProcessingContext& pc)
               truncateFloatFraction(ft0RecPoint.getCollisionTimeA() * 1E-3, mT0Time), // ps to ns
               truncateFloatFraction(ft0RecPoint.getCollisionTimeC() * 1E-3, mT0Time), // ps to ns
               ft0RecPoint.getTrigger().getTriggersignals());
+    if (mEnableFITextra) {
+      ft0ExtraCursor(bcID,
+                     aTimesA,
+                     aTimesC);
+    }
   }
 
   if (mUseMC) {
     // filling MC collision labels
+    mcColLabelsCursor.reserve(primVerLabels.size());
     for (auto& label : primVerLabels) {
-      auto it = mcColToEvSrc.find(std::pair(label.getEventID(), label.getSourceID()));
-      int32_t mcCollisionID = it != mcColToEvSrc.end() ? it->second : -1;
+      auto it = std::find_if(mcColToEvSrc.begin(), mcColToEvSrc.end(),
+                             [&label](const auto& mcColInfo) { return mcColInfo[1] == label.getSourceID() && mcColInfo[2] == label.getEventID(); });
+      int32_t mcCollisionID = -1;
+      if (it != mcColToEvSrc.end()) {
+        mcCollisionID = it->at(0);
+      }
       uint16_t mcMask = 0; // todo: set mask using normalized weights?
-      mcColLabelsCursor(0, mcCollisionID, mcMask);
+      mcColLabelsCursor(mcCollisionID, mcMask);
     }
   }
 
   cacheTriggers(recoData);
+  countTPCClusters(recoData);
 
   int collisionID = 0;
   mIndexTableMFT.resize(recoData.getMFTTracks().size());
@@ -1602,19 +2264,47 @@ void AODProducerWorkflowDPL::run(ProcessingContext& pc)
     collisionID++;
   }
 
+  /// Strangeness tracking requires its index LUTs to be filled before the tracks are filled
+  prepareStrangenessTracking(recoData);
+
   mGIDToTableFwdID.clear(); // reset the tables to be used by 'fillTrackTablesPerCollision'
   mGIDToTableMFTID.clear();
 
+  if (mPropTracks || mThinTracks) {
+    auto v0s = recoData.getV0sIdx();
+    auto cascades = recoData.getCascadesIdx();
+    auto decays3Body = recoData.getDecays3BodyIdx();
+    mGIDUsedBySVtx.reserve(v0s.size() * 2 + cascades.size() + decays3Body.size() * 3);
+    for (const auto& v0 : v0s) {
+      mGIDUsedBySVtx.insert(v0.getProngID(0));
+      mGIDUsedBySVtx.insert(v0.getProngID(1));
+    }
+    for (const auto& cascade : cascades) {
+      mGIDUsedBySVtx.insert(cascade.getBachelorID());
+    }
+    for (const auto& id3Body : decays3Body) {
+      mGIDUsedBySVtx.insert(id3Body.getProngID(0));
+      mGIDUsedBySVtx.insert(id3Body.getProngID(1));
+      mGIDUsedBySVtx.insert(id3Body.getProngID(2));
+    }
+
+    mGIDUsedByStr.reserve(recoData.getStrangeTracks().size());
+    for (const auto& sTrk : recoData.getStrangeTracks()) {
+      mGIDUsedByStr.emplace(sTrk.mITSRef, GIndex::ITS);
+    }
+  }
+
   // filling unassigned tracks first
   // so that all unassigned tracks are stored in the beginning of the table together
   auto& trackRef = primVer2TRefs.back(); // references to unassigned tracks are at the end
   // fixme: interaction time is undefined for unassigned tracks (?)
-  fillTrackTablesPerCollision(-1, std::uint64_t(-1), trackRef, primVerGIs, recoData, tracksCursor, tracksCovCursor, tracksExtraCursor,
-                              ambigTracksCursor, mftTracksCursor, ambigMFTTracksCursor,
-                              fwdTracksCursor, fwdTracksCovCursor, ambigFwdTracksCursor, bcsMap);
+  fillTrackTablesPerCollision(-1, std::uint64_t(-1), trackRef, primVerGIs, recoData, tracksCursor, tracksCovCursor, tracksExtraCursor, tracksQACursor,
+                              ambigTracksCursor, mftTracksCursor, mftTracksCovCursor, ambigMFTTracksCursor,
+                              fwdTracksCursor, fwdTracksCovCursor, ambigFwdTracksCursor, fwdTrkClsCursor, bcsMap);
 
   // filling collisions and tracks into tables
   collisionID = 0;
+  collisionsCursor.reserve(primVertices.size());
   for (auto& vertex : primVertices) {
     auto& cov = vertex.getCov();
     auto& timeStamp = vertex.getTimeStamp();                       // this is a relative time
@@ -1631,8 +2321,7 @@ void AODProducerWorkflowDPL::run(ProcessingContext& pc)
     } else {
       LOG(fatal) << "Error: could not find a corresponding BC ID for a collision; BC = " << globalBC << ", collisionID = " << collisionID;
     }
-    collisionsCursor(0,
-                     bcID,
+    collisionsCursor(bcID,
                      truncateFloatFraction(vertex.getX(), mCollisionPosition),
                      truncateFloatFraction(vertex.getY(), mCollisionPosition),
                      truncateFloatFraction(vertex.getZ(), mCollisionPosition),
@@ -1651,67 +2340,99 @@ void AODProducerWorkflowDPL::run(ProcessingContext& pc)
 
     auto& trackRef = primVer2TRefs[collisionID];
     // passing interaction time in [ps]
-    fillTrackTablesPerCollision(collisionID, globalBC, trackRef, primVerGIs, recoData, tracksCursor, tracksCovCursor, tracksExtraCursor, ambigTracksCursor,
-                                mftTracksCursor, ambigMFTTracksCursor,
-                                fwdTracksCursor, fwdTracksCovCursor, ambigFwdTracksCursor, bcsMap);
+    fillTrackTablesPerCollision(collisionID, globalBC, trackRef, primVerGIs, recoData, tracksCursor, tracksCovCursor, tracksExtraCursor, tracksQACursor, ambigTracksCursor,
+                                mftTracksCursor, mftTracksCovCursor, ambigMFTTracksCursor,
+                                fwdTracksCursor, fwdTracksCovCursor, ambigFwdTracksCursor, fwdTrkClsCursor, bcsMap);
     collisionID++;
   }
 
-  fillSecondaryVertices(recoData, v0sCursor, cascadesCursor);
+  fillSecondaryVertices(recoData, v0sCursor, cascadesCursor, decay3BodyCursor);
+  fillHMPID(recoData, hmpCursor);
+  fillStrangenessTrackingTables(recoData, trackedV0Cursor, trackedCascadeCursor, tracked3BodyCurs);
 
   // helper map for fast search of a corresponding class mask for a bc
-  std::unordered_map bcToClassMask;
+  auto emcalIncomplete = filterEMCALIncomplete(recoData.getEMCALTriggers());
+  std::unordered_map> bcToClassMask;
   if (mInputSources[GID::CTP]) {
+    LOG(debug) << "CTP input available";
     for (auto& ctpDigit : ctpDigits) {
       uint64_t bc = ctpDigit.intRecord.toLong();
       uint64_t classMask = ctpDigit.CTPClassMask.to_ulong();
-      bcToClassMask[bc] = classMask;
+      uint64_t inputMask = ctpDigit.CTPInputMask.to_ulong();
+      if (emcalIncomplete.find(bc) != emcalIncomplete.end()) {
+        // reject EMCAL triggers as BC was rejected as incomplete at readout level
+        auto classMaskOrig = classMask;
+        classMask = classMask & ~mEMCALTrgClassMask;
+        LOG(debug) << "Found EMCAL incomplete event, mask before " << std::bitset<64>(classMaskOrig) << ", after " << std::bitset<64>(classMask);
+      }
+      bcToClassMask[bc] = {classMask, inputMask};
+      // LOG(debug) << Form("classmask:0x%llx", classMask);
     }
   }
 
   // filling BC table
-  uint64_t triggerMask = 0;
+  bcCursor.reserve(bcsMap.size());
   for (auto& item : bcsMap) {
     uint64_t bc = item.first;
+    std::pair masks{0, 0};
     if (mInputSources[GID::CTP]) {
       auto bcClassPair = bcToClassMask.find(bc);
       if (bcClassPair != bcToClassMask.end()) {
-        triggerMask = bcClassPair->second;
-      } else {
-        triggerMask = 0;
+        masks = bcClassPair->second;
       }
     }
-    bcCursor(0,
-             runNumber,
+    bcCursor(runNumber,
              bc,
-             triggerMask);
+             masks.first,
+             masks.second);
   }
 
   bcToClassMask.clear();
 
-  if (mInputSources[GIndex::EMC]) {
-    // fill EMC cells to tables
-    // TODO handle MC info
-    o2::emcal::EventHandler caloEventHandler;
-    fillCaloTable(&caloEventHandler, caloEMCCells, caloEMCCellsTRGR, caloCellsCursor, caloCellsTRGTableCursor, bcsMap, 1);
+  // filling BC flags table:
+  auto bcFlags = fillBCFlags(recoData, bcsMap);
+  bcFlagsCursor.reserve(bcFlags.size());
+  for (auto f : bcFlags) {
+    bcFlagsCursor(f);
   }
 
-  if (mInputSources[GIndex::PHS]) {
-    o2::phos::EventHandler caloEventHandler;
-    fillCaloTable(&caloEventHandler, caloPHOSCells, caloPHOSCellsTRGR, caloCellsCursor, caloCellsTRGTableCursor, bcsMap, 0);
+  // fill cpvcluster table
+  if (mInputSources[GIndex::CPV]) {
+    float posX, posZ;
+    cpvClustersCursor.reserve(cpvTrigRecs.size());
+    for (auto& cpvEvent : cpvTrigRecs) {
+      uint64_t bc = cpvEvent.getBCData().toLong();
+      auto item = bcsMap.find(bc);
+      int bcID = -1;
+      if (item != bcsMap.end()) {
+        bcID = item->second;
+      } else {
+        LOG(fatal) << "Error: could not find a corresponding BC ID for a CPV Trigger Record; BC = " << bc;
+      }
+      for (int iClu = cpvEvent.getFirstEntry(); iClu < cpvEvent.getFirstEntry() + cpvEvent.getNumberOfObjects(); iClu++) {
+        auto& clu = cpvClusters[iClu];
+        clu.getLocalPosition(posX, posZ);
+        cpvClustersCursor(bcID,
+                          truncateFloatFraction(posX, mCPVPos),
+                          truncateFloatFraction(posZ, mCPVPos),
+                          truncateFloatFraction(clu.getEnergy(), mCPVAmpl),
+                          clu.getPackedClusterStatus());
+      }
+    }
   }
 
-  bcsMap.clear();
-
   if (mUseMC) {
+    TStopwatch timer;
+    timer.Start();
     // filling mc particles table
     fillMCParticlesTable(*mcReader,
-                         mcParticlesCursor,
+                         mcParticlesCursor.cursor,
                          primVer2TRefs,
                          primVerGIs,
                          recoData,
                          mcColToEvSrc);
-
+    timer.Stop();
+    LOG(info) << "FILL MC took " << timer.RealTime() << " s";
     mcColToEvSrc.clear();
 
     // ------------------------------------------------------
@@ -1719,12 +2440,19 @@ void AODProducerWorkflowDPL::run(ProcessingContext& pc)
 
     // need to go through labels in the same order as for tracks
     fillMCTrackLabelsTable(mcTrackLabelCursor, mcMFTTrackLabelCursor, mcFwdTrackLabelCursor, primVer2TRefs.back(), primVerGIs, recoData);
-    for (int iref = 0; iref < primVer2TRefs.size() - 1; iref++) {
+    for (auto iref = 0U; iref < primVer2TRefs.size() - 1; iref++) {
       auto& trackRef = primVer2TRefs[iref];
-      fillMCTrackLabelsTable(mcTrackLabelCursor, mcMFTTrackLabelCursor, mcFwdTrackLabelCursor, trackRef, primVerGIs, recoData);
+      fillMCTrackLabelsTable(mcTrackLabelCursor, mcMFTTrackLabelCursor, mcFwdTrackLabelCursor, trackRef, primVerGIs, recoData, iref);
     }
   }
-  mToStore.clear();
+
+  // Fill calo tables and if MC also the MCCaloTable, therefore, has to be after fillMCParticlesTable call!
+  if (mInputSources[GIndex::PHS] || mInputSources[GIndex::EMC]) {
+    fillCaloTable(caloCellsCursor, caloCellsTRGTableCursor, mcCaloLabelsCursor, bcsMap, recoData);
+  }
+
+  bcsMap.clear();
+  clearMCKeepStore(mToStore);
   mGIDToTableID.clear();
   mTableTrID = 0;
   mGIDToTableFwdID.clear();
@@ -1741,9 +2469,26 @@ void AODProducerWorkflowDPL::run(ProcessingContext& pc)
   mIndexTableMFT.clear();
   mIndexMFTID = 0;
 
-  originCursor(0, tfNumber);
+  mBCLookup.clear();
 
-  pc.outputs().snapshot(Output{"TFN", "TFNumber", 0, Lifetime::Timeframe}, tfNumber);
+  mGIDUsedBySVtx.clear();
+  mGIDUsedByStr.clear();
+
+  originCursor(tfNumber);
+
+  // sending metadata to writer
+  TString dataType = mUseMC ? "MC" : "RAW";
+  TString O2Version = o2::fullVersion();
+  TString ROOTVersion = ROOT_RELEASE;
+  mMetaDataKeys = {"DataType", "Run", "O2Version", "ROOTVersion", "RecoPassName", "AnchorProduction", "AnchorPassName", "LPMProductionTag", "CreatedBy"};
+  mMetaDataVals = {dataType, "3", O2Version, ROOTVersion, mRecoPass, mAnchorProd, mAnchorPass, mLPMProdTag, mUser};
+  add_additional_meta_info(mMetaDataKeys, mMetaDataVals);
+
+  pc.outputs().snapshot(Output{"AMD", "AODMetadataKeys", 0}, mMetaDataKeys);
+  pc.outputs().snapshot(Output{"AMD", "AODMetadataVals", 0}, mMetaDataVals);
+
+  pc.outputs().snapshot(Output{"TFN", "TFNumber", 0}, tfNumber);
+  pc.outputs().snapshot(Output{"TFF", "TFFilename", 0}, "");
 
   mTimer.Stop();
 }
@@ -1824,8 +2569,8 @@ AODProducerWorkflowDPL::TrackExtraInfo AODProducerWorkflowDPL::processBarrelTrac
   if (collisionID < 0) {
     extraInfoHolder.flags |= o2::aod::track::OrphanTrack;
   }
-  bool needBCSlice = collisionID < 0 || trackIndex.isAmbiguous(); // track is associated to multiple vertices
-  uint64_t bcOfTimeRef = collisionBC - mStartIR.toLong();         // by default (unambiguous) track time is wrt collision BC
+  bool needBCSlice = collisionID < 0;                     // track is associated to multiple vertices
+  uint64_t bcOfTimeRef = collisionBC - mStartIR.toLong(); // by default track time is wrt collision BC (unless no collision assigned)
 
   auto setTrackTime = [&](double t, double terr, bool gaussian) {
     // set track time and error, for ambiguous tracks define the bcSlice as it was used in vertex-track association
@@ -1839,6 +2584,7 @@ AODProducerWorkflowDPL::TrackExtraInfo AODProducerWorkflowDPL::processBarrelTrac
       bcOfTimeRef = fillBCSlice(extraInfoHolder.bcSlice, t - error, t + error, bcsMap);
     }
     extraInfoHolder.trackTime = float(t - bcOfTimeRef * o2::constants::lhc::LHCBunchSpacingNS);
+    extraInfoHolder.diffBCRef = int(bcOfTimeRef);
     LOGP(debug, "time : {}/{} -> {}/{} -> trunc: {}/{} CollID: {} Amb: {}", t, terr, t - bcOfTimeRef * o2::constants::lhc::LHCBunchSpacingNS, terr,
          truncateFloatFraction(extraInfoHolder.trackTime, mTrackTime), truncateFloatFraction(extraInfoHolder.trackTimeRes, mTrackTimeError),
          collisionID, trackIndex.isAmbiguous());
@@ -1855,7 +2601,10 @@ AODProducerWorkflowDPL::TrackExtraInfo AODProducerWorkflowDPL::processBarrelTrac
     extraInfoHolder.length = intLen;
     const float mass = o2::constants::physics::MassPionCharged; // default pid = pion
     if (tofInt.getTOF(o2::track::PID::Pion) > 0.f) {
-      const float expBeta = (intLen / (tofInt.getTOF(o2::track::PID::Pion) * cSpeed));
+      float expBeta = (intLen / (tofInt.getTOF(o2::track::PID::Pion) * cSpeed));
+      if (expBeta > o2::constants::math::Almost1) {
+        expBeta = o2::constants::math::Almost1;
+      }
       extraInfoHolder.tofExpMom = mass * expBeta / std::sqrt(1.f - expBeta * expBeta);
     }
     // correct the time of the track
@@ -1868,6 +2617,7 @@ AODProducerWorkflowDPL::TrackExtraInfo AODProducerWorkflowDPL::processBarrelTrac
   if (contributorsGID[GIndex::Source::TRD].isIndexSet()) {                                        // ITS-TPC-TRD-TOF, TPC-TRD-TOF, TPC-TRD, ITS-TPC-TRD
     const auto& trdOrig = data.getTrack(contributorsGID[GIndex::Source::TRD]); // refitted TRD trac
     extraInfoHolder.trdChi2 = trdOrig.getChi2();
+    extraInfoHolder.trdSignal = trdOrig.getSignal();
     extraInfoHolder.trdPattern = getTRDPattern(trdOrig);
     if (extraInfoHolder.trackTimeRes < 0.) { // time is not set yet, this is possible only for TPC-TRD and ITS-TPC-TRD tracks, since those with TOF are set upstream
       // TRD is triggered: time uncertainty is within a BC
@@ -1881,31 +2631,52 @@ AODProducerWorkflowDPL::TrackExtraInfo AODProducerWorkflowDPL::processBarrelTrac
     int nClusters = itsTrack.getNClusters();
     float chi2 = itsTrack.getChi2();
     extraInfoHolder.itsChi2NCl = nClusters != 0 ? chi2 / (float)nClusters : 0;
-    extraInfoHolder.itsClusterMap = itsTrack.getPattern();
+    extraInfoHolder.itsClusterSizes = itsTrack.getClusterSizes();
     if (src == GIndex::ITS) { // standalone ITS track should set its time from the ROF
       const auto& rof = data.getITSTracksROFRecords()[mITSROFs[trackIndex.getIndex()]];
-      double t = rof.getBCData().differenceInBC(mStartIR) * o2::constants::lhc::LHCBunchSpacingNS + mITSROFrameHalfLengthNS;
+      double t = rof.getBCData().differenceInBC(mStartIR) * o2::constants::lhc::LHCBunchSpacingNS + mITSROFrameHalfLengthNS + mITSROFBiasNS;
       setTrackTime(t, mITSROFrameHalfLengthNS, false);
     }
   } else if (contributorsGID[GIndex::Source::ITSAB].isIndexSet()) { // this is an ITS-TPC afterburner contributor
-    extraInfoHolder.itsClusterMap = data.getITSABRefs()[contributorsGID[GIndex::Source::ITSAB].getIndex()].pattern;
+    extraInfoHolder.itsClusterSizes = data.getITSABRefs()[contributorsGID[GIndex::Source::ITSAB].getIndex()].getClusterSizes();
   }
   if (contributorsGID[GIndex::Source::TPC].isIndexSet()) {
     const auto& tpcOrig = data.getTPCTrack(contributorsGID[GIndex::TPC]);
-    extraInfoHolder.tpcInnerParam = tpcOrig.getP();
+    const auto& tpcClData = mTPCCounters[contributorsGID[GIndex::TPC]];
+    const auto& dEdx = tpcOrig.getdEdx().dEdxTotTPC > 0 ? tpcOrig.getdEdx() : tpcOrig.getdEdxAlt();
+    if (tpcOrig.getdEdx().dEdxTotTPC == 0) {
+      extraInfoHolder.flags |= o2::aod::track::TPCdEdxAlt;
+    }
+    if (tpcOrig.hasASideClusters()) {
+      extraInfoHolder.flags |= o2::aod::track::TPCSideA;
+    }
+    if (tpcOrig.hasCSideClusters()) {
+      extraInfoHolder.flags |= o2::aod::track::TPCSideC;
+    }
+    extraInfoHolder.tpcInnerParam = tpcOrig.getP() / tpcOrig.getAbsCharge();
     extraInfoHolder.tpcChi2NCl = tpcOrig.getNClusters() ? tpcOrig.getChi2() / tpcOrig.getNClusters() : 0;
-    extraInfoHolder.tpcSignal = tpcOrig.getdEdx().dEdxTotTPC;
-    uint8_t shared, found, crossed; // fixme: need to switch from these placeholders to something more reasonable
-    countTPCClusters(tpcOrig, data.getTPCTracksClusterRefs(), data.clusterShMapTPC, data.getTPCClusters(), shared, found, crossed);
+    extraInfoHolder.tpcSignal = dEdx.dEdxTotTPC;
     extraInfoHolder.tpcNClsFindable = tpcOrig.getNClusters();
-    extraInfoHolder.tpcNClsFindableMinusFound = tpcOrig.getNClusters() - found;
-    extraInfoHolder.tpcNClsFindableMinusCrossedRows = tpcOrig.getNClusters() - crossed;
-    extraInfoHolder.tpcNClsShared = shared;
-    if (src == GIndex::TPC) {                                                                                // standalone TPC track should set its time from their timebins range
-      double terr = 0.5 * (tpcOrig.getDeltaTFwd() + tpcOrig.getDeltaTBwd()) * mTPCBinNS;                     // half-span of the interval
-      double t = (tpcOrig.getTime0() + 0.5 * (tpcOrig.getDeltaTFwd() - tpcOrig.getDeltaTBwd())) * mTPCBinNS; // central value
-      LOG(debug) << "TPC tracks t0:" << tpcOrig.getTime0() << " tbwd: " << tpcOrig.getDeltaTBwd() << " tfwd: " << tpcOrig.getDeltaTFwd() << " t: " << t << " te: " << terr;
-      setTrackTime(t, terr, false);
+    extraInfoHolder.tpcNClsFindableMinusFound = tpcOrig.getNClusters() - tpcClData.found;
+    extraInfoHolder.tpcNClsFindableMinusCrossedRows = tpcOrig.getNClusters() - tpcClData.crossed;
+    extraInfoHolder.tpcNClsShared = tpcClData.shared;
+    uint32_t clsUsedForPID = dEdx.NHitsIROC + dEdx.NHitsOROC1 + dEdx.NHitsOROC2 + dEdx.NHitsOROC3;
+    extraInfoHolder.tpcNClsFindableMinusPID = tpcOrig.getNClusters() - clsUsedForPID;
+    if (src == GIndex::TPC) { // standalone TPC track should set its time from their timebins range
+      if (needBCSlice) {
+        double t = (tpcOrig.getTime0() + 0.5 * (tpcOrig.getDeltaTFwd() - tpcOrig.getDeltaTBwd())) * mTPCBinNS; // central value
+        double terr = 0.5 * (tpcOrig.getDeltaTFwd() + tpcOrig.getDeltaTBwd()) * mTPCBinNS;
+        double err = mTimeMarginTrackTime + terr;
+        bcOfTimeRef = fillBCSlice(extraInfoHolder.bcSlice, t - err, t + err, bcsMap);
+      }
+      aod::track::extensions::TPCTimeErrEncoding p;
+      p.setDeltaTFwd(tpcOrig.getDeltaTFwd());
+      p.setDeltaTBwd(tpcOrig.getDeltaTBwd());
+      extraInfoHolder.trackTimeRes = p.getTimeErr();
+      extraInfoHolder.trackTime = float(tpcOrig.getTime0() * mTPCBinNS - bcOfTimeRef * o2::constants::lhc::LHCBunchSpacingNS);
+      extraInfoHolder.diffBCRef = int(bcOfTimeRef);
+      extraInfoHolder.isTPConly = true; // no truncation
+      extraInfoHolder.flags |= o2::aod::track::TrackTimeAsym;
     } else if (src == GIndex::ITSTPC) { // its-tpc matched tracks have gaussian time error and the time was not set above
       const auto& trITSTPC = data.getTPCITSTrack(trackIndex);
       auto ts = trITSTPC.getTimeMUS();
@@ -1913,6 +2684,7 @@ AODProducerWorkflowDPL::TrackExtraInfo AODProducerWorkflowDPL::processBarrelTrac
     }
   }
 
+  extrapolateToCalorimeters(extraInfoHolder, data.getTrackParamOut(trackIndex));
   // set bit encoding for PVContributor property as part of the flag field
   if (trackIndex.isPVContributor()) {
     extraInfoHolder.flags |= o2::aod::track::PVContributor;
@@ -1920,6 +2692,290 @@ AODProducerWorkflowDPL::TrackExtraInfo AODProducerWorkflowDPL::processBarrelTrac
   return extraInfoHolder;
 }
 
+AODProducerWorkflowDPL::TrackQA AODProducerWorkflowDPL::processBarrelTrackQA(int collisionID, std::uint64_t collisionBC, GIndex trackIndex,
+                                                                             const o2::globaltracking::RecoContainer& data, const std::map& bcsMap)
+{
+  TrackQA trackQAHolder;
+  auto contributorsGID = data.getTPCContributorGID(trackIndex);
+  const auto& trackPar = data.getTrackParam(trackIndex);
+  if (contributorsGID.isIndexSet()) {
+    auto prop = o2::base::Propagator::Instance();
+    const auto& tpcOrig = data.getTPCTrack(contributorsGID);
+    /// getDCA - should be done  with the copy of TPC only track
+    o2::track::TrackParametrization tpcTMP = tpcOrig;                                             /// get backup of the track
+    const o2::base::Propagator::MatCorrType mMatType = o2::base::Propagator::MatCorrType::USEMatCorrLUT; /// should be parameterized
+    const o2::dataformats::VertexBase v = mVtx.getMeanVertex(collisionID < 0 ? 0.f : data.getPrimaryVertex(collisionID).getZ());
+    std::array dcaInfo{-999., -999.};
+    if (prop->propagateToDCABxByBz({v.getX(), v.getY(), v.getZ()}, tpcTMP, 2.f, mMatType, &dcaInfo)) {
+      trackQAHolder.tpcdcaR = 100. * dcaInfo[0] / sqrt(1. + trackPar.getQ2Pt() * trackPar.getQ2Pt());
+      trackQAHolder.tpcdcaZ = 100. * dcaInfo[1] / sqrt(1. + trackPar.getQ2Pt() * trackPar.getQ2Pt());
+    }
+    // This allows to safely clamp any float to one byte, using the
+    // minmal/maximum values as under-/overflow borders and rounding to the nearest integer
+    auto safeInt8Clamp = [](auto value) -> int8_t {
+      using ValType = decltype(value);
+      return static_cast(TMath::Nint(std::clamp(value, static_cast(std::numeric_limits::min()), static_cast(std::numeric_limits::max()))));
+    };
+    auto safeUInt8Clamp = [](auto value) -> uint8_t {
+      using ValType = decltype(value);
+      return static_cast(TMath::Nint(std::clamp(value, static_cast(std::numeric_limits::min()), static_cast(std::numeric_limits::max()))));
+    };
+
+    /// get tracklet byteMask
+    uint8_t clusterCounters[8] = {0};
+    {
+      uint8_t sectorIndex, rowIndex;
+      uint32_t clusterIndex;
+      const auto& tpcClusRefs = data.getTPCTracksClusterRefs();
+      for (int i = 0; i < tpcOrig.getNClusterReferences(); i++) {
+        o2::tpc::TrackTPC::getClusterReference(tpcClusRefs, i, sectorIndex, rowIndex, clusterIndex, tpcOrig.getClusterRef());
+        char indexTracklet = (rowIndex % 152) / 19;
+        clusterCounters[indexTracklet]++;
+      }
+    }
+    uint8_t byteMask = 0;
+    for (int i = 0; i < 8; i++) {
+      if (clusterCounters[i] > 5) {
+        byteMask |= (1 << i);
+      }
+    }
+    trackQAHolder.tpcTime0 = tpcOrig.getTime0();
+    trackQAHolder.tpcClusterByteMask = byteMask;
+    const auto& dEdxInfoAlt = tpcOrig.getdEdxAlt(); // tpcOrig.getdEdx()
+    const float dEdxNorm = (dEdxInfoAlt.dEdxTotTPC > 0) ? 100. / dEdxInfoAlt.dEdxTotTPC : 0;
+    trackQAHolder.tpcdEdxNorm = dEdxInfoAlt.dEdxTotTPC;
+    trackQAHolder.tpcdEdxMax0R = safeUInt8Clamp(dEdxInfoAlt.dEdxMaxIROC * dEdxNorm);
+    trackQAHolder.tpcdEdxMax1R = safeUInt8Clamp(dEdxInfoAlt.dEdxMaxOROC1 * dEdxNorm);
+    trackQAHolder.tpcdEdxMax2R = safeUInt8Clamp(dEdxInfoAlt.dEdxMaxOROC2 * dEdxNorm);
+    trackQAHolder.tpcdEdxMax3R = safeUInt8Clamp(dEdxInfoAlt.dEdxMaxOROC3 * dEdxNorm);
+    //
+    trackQAHolder.tpcdEdxTot0R = safeUInt8Clamp(dEdxInfoAlt.dEdxTotIROC * dEdxNorm);
+    trackQAHolder.tpcdEdxTot1R = safeUInt8Clamp(dEdxInfoAlt.dEdxTotOROC1 * dEdxNorm);
+    trackQAHolder.tpcdEdxTot2R = safeUInt8Clamp(dEdxInfoAlt.dEdxTotOROC2 * dEdxNorm);
+    trackQAHolder.tpcdEdxTot3R = safeUInt8Clamp(dEdxInfoAlt.dEdxTotOROC3 * dEdxNorm);
+    ///
+    float scaleTOF{0};
+    auto contributorsGIDA = data.getSingleDetectorRefs(trackIndex);
+    if (contributorsGIDA[GIndex::Source::TOF].isIndexSet()) { // ITS-TPC-TRD-TOF, ITS-TPC-TOF, TPC-TRD-TOF, TPC-TOF
+      const auto& tofMatch = data.getTOFMatch(trackIndex);
+      const float qpt = trackPar.getQ2Pt();
+      scaleTOF = std::sqrt(o2::aod::track::trackQAScaledTOF[0] * o2::aod::track::trackQAScaledTOF[0] + qpt * qpt * o2::aod::track::trackQAScaledTOF[1] * o2::aod::track::trackQAScaledTOF[1]) / (2. * o2::aod::track::trackQAScaleBins);
+      trackQAHolder.dTofdX = safeInt8Clamp(tofMatch.getDXatTOF() / scaleTOF);
+      trackQAHolder.dTofdZ = safeInt8Clamp(tofMatch.getDZatTOF() / scaleTOF);
+    }
+
+    // Add matching information at a reference point (defined by
+    // o2::aod::track::trackQARefRadius) in the same frame as the global track
+    // without material corrections and error propagation
+    if (auto itsContGID = data.getITSContributorGID(trackIndex); itsContGID.isIndexSet() && itsContGID.getSource() != GIndex::ITSAB) {
+      const auto& itsOrig = data.getITSTrack(itsContGID);
+      o2::track::TrackPar gloCopy = trackPar;
+      o2::track::TrackPar itsCopy = itsOrig.getParamOut();
+      o2::track::TrackPar tpcCopy = tpcOrig;
+      if (prop->propagateToX(gloCopy, o2::aod::track::trackQARefRadius, prop->getNominalBz(), o2::base::Propagator::MAX_SIN_PHI, o2::base::Propagator::MAX_STEP, mMatCorr) &&
+          prop->propagateToAlphaX(tpcCopy, gloCopy.getAlpha(), o2::aod::track::trackQARefRadius, false, o2::base::Propagator::MAX_SIN_PHI, o2::base::Propagator::MAX_STEP, 1, mMatCorr) &&
+          prop->propagateToAlphaX(itsCopy, gloCopy.getAlpha(), o2::aod::track::trackQARefRadius, false, o2::base::Propagator::MAX_SIN_PHI, o2::base::Propagator::MAX_STEP, 1, mMatCorr)) {
+        // All tracks are now at the same radius and in the same frame and we can calculate the deltas wrt. to the global track
+        // The scale is defined by the global track scaling depending on beta0
+        const float beta0 = std::sqrt(std::min(50.f / tpcOrig.getdEdx().dEdxMaxTPC, 1.f));
+        const float qpt = gloCopy.getQ2Pt();
+        const float x = qpt / beta0;
+        // scaling is defined as sigmaBins/sqrt(p0^2 + (p1 * q/pt / beta)^2)
+        auto scaleCont = [&x](int i) -> float {
+          return o2::aod::track::trackQAScaleBins / std::sqrt(o2::aod::track::trackQAScaleContP0[i] * o2::aod::track::trackQAScaleContP0[i] + (o2::aod::track::trackQAScaleContP1[i] * x) * (o2::aod::track::trackQAScaleContP1[i] * x));
+        };
+        auto scaleGlo = [&x](int i) -> float {
+          return o2::aod::track::trackQAScaleBins / std::sqrt(o2::aod::track::trackQAScaleGloP0[i] * o2::aod::track::trackQAScaleGloP0[i] + (o2::aod::track::trackQAScaleGloP1[i] * x) * (o2::aod::track::trackQAScaleGloP1[i] * x));
+        };
+
+        // Calculate deltas for contributors
+        trackQAHolder.dRefContY = safeInt8Clamp((itsCopy.getY() - tpcCopy.getY()) * scaleCont(0));
+        trackQAHolder.dRefContZ = safeInt8Clamp((itsCopy.getZ() - tpcCopy.getZ()) * scaleCont(1));
+        trackQAHolder.dRefContSnp = safeInt8Clamp((itsCopy.getSnp() - tpcCopy.getSnp()) * scaleCont(2));
+        trackQAHolder.dRefContTgl = safeInt8Clamp((itsCopy.getTgl() - tpcCopy.getTgl()) * scaleCont(3));
+        trackQAHolder.dRefContQ2Pt = safeInt8Clamp((itsCopy.getQ2Pt() - tpcCopy.getQ2Pt()) * scaleCont(4));
+        // Calculate deltas for global track against averaged contributors
+        trackQAHolder.dRefGloY = safeInt8Clamp(((itsCopy.getY() + tpcCopy.getY()) * 0.5f - gloCopy.getY()) * scaleGlo(0));
+        trackQAHolder.dRefGloZ = safeInt8Clamp(((itsCopy.getZ() + tpcCopy.getZ()) * 0.5f - gloCopy.getZ()) * scaleGlo(1));
+        trackQAHolder.dRefGloSnp = safeInt8Clamp(((itsCopy.getSnp() + tpcCopy.getSnp()) * 0.5f - gloCopy.getSnp()) * scaleGlo(2));
+        trackQAHolder.dRefGloTgl = safeInt8Clamp(((itsCopy.getTgl() + tpcCopy.getTgl()) * 0.5f - gloCopy.getTgl()) * scaleGlo(3));
+        trackQAHolder.dRefGloQ2Pt = safeInt8Clamp(((itsCopy.getQ2Pt() + tpcCopy.getQ2Pt()) * 0.5f - gloCopy.getQ2Pt()) * scaleGlo(4));
+        //
+
+        if (mStreamerFlags[AODProducerStreamerFlags::TrackQA]) {
+          (*mStreamer) << "trackQA"
+                       << "trackITSOrig=" << itsOrig
+                       << "trackTPCOrig=" << tpcOrig
+                       << "trackITSTPCOrig=" << trackPar
+                       << "trackITSProp=" << itsCopy
+                       << "trackTPCProp=" << tpcCopy
+                       << "trackITSTPCProp=" << gloCopy
+                       << "refRadius=" << o2::aod::track::trackQARefRadius
+                       << "scaleBins=" << o2::aod::track::trackQAScaleBins
+                       << "scaleCont0=" << scaleCont(0)
+                       << "scaleCont1=" << scaleCont(1)
+                       << "scaleCont2=" << scaleCont(2)
+                       << "scaleCont3=" << scaleCont(3)
+                       << "scaleCont4=" << scaleCont(4)
+                       << "scaleGlo0=" << scaleGlo(0)
+                       << "scaleGlo1=" << scaleGlo(1)
+                       << "scaleGlo2=" << scaleGlo(2)
+                       << "scaleGlo3=" << scaleGlo(3)
+                       << "scaleGlo4=" << scaleGlo(4)
+                       << "trackQAHolder.tpcTime0=" << trackQAHolder.tpcTime0
+                       << "trackQAHolder.tpcdEdxNorm=" << trackQAHolder.tpcdEdxNorm
+                       << "trackQAHolder.tpcdcaR=" << trackQAHolder.tpcdcaR
+                       << "trackQAHolder.tpcdcaZ=" << trackQAHolder.tpcdcaZ
+                       << "trackQAHolder.tpcdcaClusterByteMask=" << trackQAHolder.tpcClusterByteMask
+                       << "trackQAHolder.tpcdEdxMax0R=" << trackQAHolder.tpcdEdxMax0R
+                       << "trackQAHolder.tpcdEdxMax1R=" << trackQAHolder.tpcdEdxMax1R
+                       << "trackQAHolder.tpcdEdxMax2R=" << trackQAHolder.tpcdEdxMax2R
+                       << "trackQAHolder.tpcdEdxMax3R=" << trackQAHolder.tpcdEdxMax3R
+                       << "trackQAHolder.tpcdEdxTot0R=" << trackQAHolder.tpcdEdxTot0R
+                       << "trackQAHolder.tpcdEdxTot1R=" << trackQAHolder.tpcdEdxTot1R
+                       << "trackQAHolder.tpcdEdxTot2R=" << trackQAHolder.tpcdEdxTot2R
+                       << "trackQAHolder.tpcdEdxTot3R=" << trackQAHolder.tpcdEdxTot3R
+                       << "trackQAHolder.dRefContY=" << trackQAHolder.dRefContY
+                       << "trackQAHolder.dRefContZ=" << trackQAHolder.dRefContZ
+                       << "trackQAHolder.dRefContSnp=" << trackQAHolder.dRefContSnp
+                       << "trackQAHolder.dRefContTgl=" << trackQAHolder.dRefContTgl
+                       << "trackQAHolder.dRefContQ2Pt=" << trackQAHolder.dRefContQ2Pt
+                       << "trackQAHolder.dRefGloY=" << trackQAHolder.dRefGloY
+                       << "trackQAHolder.dRefGloZ=" << trackQAHolder.dRefGloZ
+                       << "trackQAHolder.dRefGloSnp=" << trackQAHolder.dRefGloSnp
+                       << "trackQAHolder.dRefGloTgl=" << trackQAHolder.dRefGloTgl
+                       << "trackQAHolder.dRefGloQ2Pt=" << trackQAHolder.dRefGloQ2Pt
+                       << "trackQAHolder.dTofdX=" << trackQAHolder.dTofdX
+                       << "trackQAHolder.dTofdZ=" << trackQAHolder.dTofdZ
+                       << "scaleTOF=" << scaleTOF
+                       << "\n";
+        }
+      }
+    }
+  }
+
+  return trackQAHolder;
+}
+
+bool AODProducerWorkflowDPL::propagateTrackToPV(o2::track::TrackParametrizationWithError& trackPar,
+                                                const o2::globaltracking::RecoContainer& data,
+                                                int colID)
+{
+  o2::dataformats::DCA dcaInfo;
+  dcaInfo.set(999.f, 999.f, 999.f, 999.f, 999.f);
+  o2::dataformats::VertexBase v = mVtx.getMeanVertex(colID < 0 ? 0.f : data.getPrimaryVertex(colID).getZ());
+  return o2::base::Propagator::Instance()->propagateToDCABxByBz(v, trackPar, 2.f, mMatCorr, &dcaInfo);
+}
+
+void AODProducerWorkflowDPL::extrapolateToCalorimeters(TrackExtraInfo& extraInfoHolder, const o2::track::TrackPar& track)
+{
+  constexpr float XEMCAL = 440.f, XPHOS = 460.f, XEMCAL2 = XEMCAL * XEMCAL;
+  constexpr float ETAEMCAL = 0.75;                                  // eta of EMCAL/DCAL with margin
+  constexpr float ZEMCALFastCheck = 460.;                           // Max Z (with margin to check with straightline extrapolarion)
+  constexpr float ETADCALINNER = 0.22;                              // eta of the DCAL PHOS Hole (at XEMCAL)
+  constexpr float ETAPHOS = 0.13653194;                             // nominal eta of the PHOS acceptance (at XPHOS): -log(tan((TMath::Pi()/2 - atan2(63, 460))/2))
+  constexpr float ETAPHOSMARGIN = 0.17946979;                       // etat of the PHOS acceptance with 20 cm margin (at XPHOS): -log(tan((TMath::Pi()/2 + atan2(63+20., 460))/2)), not used, for the ref only
+  constexpr float ETADCALPHOSSWITCH = (ETADCALINNER + ETAPHOS) / 2; // switch to DCAL to PHOS check if eta < this value
+  constexpr short SNONE = 0, SEMCAL = 0x1, SPHOS = 0x2;
+  constexpr short SECTORTYPE[18] = {
+    SNONE, SNONE, SNONE, SNONE,                     // 0:3
+    SEMCAL, SEMCAL, SEMCAL, SEMCAL, SEMCAL, SEMCAL, // 3:9 EMCAL only
+    SNONE, SNONE,                                   // 10:11
+    SPHOS,                                          // 12 PHOS only
+    SPHOS | SEMCAL, SPHOS | SEMCAL, SPHOS | SEMCAL, // 13:15 PHOS & DCAL
+    SEMCAL,                                         // 16 DCAL only
+    SNONE                                           // 17
+  };
+
+  o2::track::TrackPar outTr{track};
+  auto prop = o2::base::Propagator::Instance();
+  // 1st propagate to EMCAL nominal radius
+  float xtrg = 0;
+  // quick check with straight line propagtion
+  if (!outTr.getXatLabR(XEMCAL, xtrg, prop->getNominalBz(), o2::track::DirType::DirOutward) ||
+      (std::abs(outTr.getZAt(xtrg, 0)) > ZEMCALFastCheck) ||
+      !prop->PropagateToXBxByBz(outTr, xtrg, 0.95, 10, o2::base::Propagator::MatCorrType::USEMatCorrLUT)) {
+    LOGP(debug, "preliminary step: does not reach R={} {}", XEMCAL, outTr.asString());
+    return;
+  }
+  // we do not necessarilly reach wanted radius in a single propagation
+  if ((outTr.getX() * outTr.getX() + outTr.getY() * outTr.getY() < XEMCAL2) &&
+      (!outTr.rotateParam(outTr.getPhi()) ||
+       !outTr.getXatLabR(XEMCAL, xtrg, prop->getNominalBz(), o2::track::DirType::DirOutward) ||
+       !prop->PropagateToXBxByBz(outTr, xtrg, 0.95, 10, o2::base::Propagator::MatCorrType::USEMatCorrLUT))) {
+    LOGP(debug, "does not reach R={} {}", XEMCAL, outTr.asString());
+    return;
+  }
+  // rotate to proper sector
+  int sector = o2::math_utils::angle2Sector(outTr.getPhiPos());
+
+  auto propExactSector = [&outTr, §or, prop](float xprop) -> bool { // propagate exactly to xprop in the proper sector frame
+    int ntri = 0;
+    while (ntri < 2) {
+      auto outTrTmp = outTr;
+      float alpha = o2::math_utils::sector2Angle(sector);
+      if ((std::abs(outTr.getZ()) > ZEMCALFastCheck) || !outTrTmp.rotateParam(alpha) ||
+          !prop->PropagateToXBxByBz(outTrTmp, xprop, 0.95, 10, o2::base::Propagator::MatCorrType::USEMatCorrLUT)) {
+        LOGP(debug, "failed on rotation to {} (sector {}) or propagation to X={} {}", alpha, sector, xprop, outTrTmp.asString());
+        return false;
+      }
+      // make sure we are still in the target sector
+      int sectorTmp = o2::math_utils::angle2Sector(outTrTmp.getPhiPos());
+      if (sectorTmp == sector) {
+        outTr = outTrTmp;
+        break;
+      }
+      sector = sectorTmp;
+      ntri++;
+    }
+    if (ntri == 2) {
+      LOGP(debug, "failed to rotate to sector, {}", outTr.asString());
+      return false;
+    }
+    return true;
+  };
+
+  // we are at the EMCAL X, check if we are in the good sector
+  if (!propExactSector(XEMCAL) || SECTORTYPE[sector] == SNONE) { // propagation failed or neither EMCAL not DCAL/PHOS
+    return;
+  }
+
+  // check if we are in a good eta range
+  float r = std::sqrt(outTr.getX() * outTr.getX() + outTr.getY() * outTr.getY()), tg = std::atan2(r, outTr.getZ());
+  float eta = -std::log(std::tan(0.5f * tg)), etaAbs = std::abs(eta);
+  if (etaAbs > ETAEMCAL) {
+    LOGP(debug, "eta = {} is off at EMCAL radius", eta, outTr.asString());
+    return;
+  }
+  // are we in the PHOS hole (with margin)?
+  if ((SECTORTYPE[sector] & SPHOS) && etaAbs < ETADCALPHOSSWITCH) { // propagate to PHOS radius
+    if (!propExactSector(XPHOS)) {
+      return;
+    }
+    r = std::sqrt(outTr.getX() * outTr.getX() + outTr.getY() * outTr.getY());
+    tg = std::atan2(r, outTr.getZ());
+    eta = -std::log(std::tan(0.5f * tg));
+  } else if (!(SECTORTYPE[sector] & SEMCAL)) { // are in the sector with PHOS only
+    return;
+  }
+  extraInfoHolder.trackPhiEMCAL = outTr.getPhiPos();
+  extraInfoHolder.trackEtaEMCAL = eta;
+  LOGP(debug, "eta = {} phi = {} sector {} for {}", extraInfoHolder.trackEtaEMCAL, extraInfoHolder.trackPhiEMCAL, sector, outTr.asString());
+  //
+}
+
+std::set AODProducerWorkflowDPL::filterEMCALIncomplete(const gsl::span triggers)
+{
+  std::set emcalIncompletes;
+  for (const auto& trg : triggers) {
+    if (trg.getTriggerBits() & o2::emcal::triggerbits::Inc) {
+      // trigger record masked at incomplete at readout level
+      emcalIncompletes.insert(trg.getBCData().toLong());
+    }
+  }
+  return emcalIncompletes;
+}
+
 void AODProducerWorkflowDPL::updateTimeDependentParams(ProcessingContext& pc)
 {
   o2::base::GRPGeomHelper::instance().checkUpdates(pc);
@@ -1927,12 +2983,12 @@ void AODProducerWorkflowDPL::updateTimeDependentParams(ProcessingContext& pc)
   if (!initOnceDone) { // this params need to be queried only once
     initOnceDone = true;
     // Note: DPLAlpideParam for ITS and MFT will be loaded by the RecoContainer
-
+    mSqrtS = o2::base::GRPGeomHelper::instance().getGRPLHCIF()->getSqrtS();
     // apply settings
     auto grpECS = o2::base::GRPGeomHelper::instance().getGRPECS();
     o2::BunchFilling bcf = o2::base::GRPGeomHelper::instance().getGRPLHCIF()->getBunchFilling();
     std::bitset<3564> bs = bcf.getBCPattern();
-    for (int i = 0; i < bs.size(); i++) {
+    for (auto i = 0U; i < bs.size(); i++) {
       if (bs.test(i)) {
         o2::tof::Utils::addInteractionBC(i);
       }
@@ -1940,9 +2996,10 @@ void AODProducerWorkflowDPL::updateTimeDependentParams(ProcessingContext& pc)
 
     const auto& alpParamsITS = o2::itsmft::DPLAlpideParam::Instance();
     mITSROFrameHalfLengthNS = 0.5 * (grpECS->isDetContinuousReadOut(o2::detectors::DetID::ITS) ? alpParamsITS.roFrameLengthInBC * o2::constants::lhc::LHCBunchSpacingNS : alpParamsITS.roFrameLengthTrig);
-
+    mITSROFBiasNS = o2::itsmft::DPLAlpideParam::Instance().roFrameBiasInBC * o2::constants::lhc::LHCBunchSpacingNS;
     const auto& alpParamsMFT = o2::itsmft::DPLAlpideParam::Instance();
     mMFTROFrameHalfLengthNS = 0.5 * (grpECS->isDetContinuousReadOut(o2::detectors::DetID::MFT) ? alpParamsMFT.roFrameLengthInBC * o2::constants::lhc::LHCBunchSpacingNS : alpParamsMFT.roFrameLengthTrig);
+    mMFTROFBiasNS = o2::itsmft::DPLAlpideParam::Instance().roFrameBiasInBC * o2::constants::lhc::LHCBunchSpacingNS;
 
     // RS FIXME: this is not yet fetched from the CCDB
     auto& elParam = o2::tpc::ParameterElectronics::Instance();
@@ -1951,6 +3008,12 @@ void AODProducerWorkflowDPL::updateTimeDependentParams(ProcessingContext& pc)
     const auto& pvParams = o2::vertexing::PVertexerParams::Instance();
     mNSigmaTimeTrack = pvParams.nSigmaTimeTrack;
     mTimeMarginTrackTime = pvParams.timeMarginTrackTime * 1.e3;
+    mFieldON = std::abs(o2::base::Propagator::Instance()->getNominalBz()) > 0.01;
+
+    pc.inputs().get("ctpconfig");
+  }
+  if (mPropTracks) {
+    pc.inputs().get("meanvtx");
   }
 }
 
@@ -1959,6 +3022,9 @@ void AODProducerWorkflowDPL::finaliseCCDB(ConcreteDataMatcher& matcher, void* ob
 {
   // Note: strictly speaking, for Configurable params we don't need finaliseCCDB check, the singletons are updated at the CCDB fetcher level
   if (o2::base::GRPGeomHelper::instance().finaliseCCDB(matcher, obj)) {
+    if (matcher == ConcreteDataMatcher("GLO", "GRPMAGFIELD", 0)) {
+      o2::mch::TrackExtrap::setField();
+    }
     return;
   }
   if (matcher == ConcreteDataMatcher("ITS", "ALPIDEPARAM", 0)) {
@@ -1973,6 +3039,22 @@ void AODProducerWorkflowDPL::finaliseCCDB(ConcreteDataMatcher& matcher, void* ob
     par.printKeyValues();
     return;
   }
+  if (matcher == ConcreteDataMatcher("GLO", "MEANVERTEX", 0)) {
+    LOG(info) << "Imposing new MeanVertex: " << ((const o2::dataformats::MeanVertexObject*)obj)->asString();
+    mVtx = *(const o2::dataformats::MeanVertexObject*)obj;
+    return;
+  }
+  if (matcher == ConcreteDataMatcher("CTP", "CTPCONFIG", 0)) {
+    // construct mask with EMCAL trigger classes for rejection of incomplete triggers
+    auto ctpconfig = *(const o2::ctp::CTPConfiguration*)obj;
+    mEMCALTrgClassMask = 0;
+    for (const auto& trgclass : ctpconfig.getCTPClasses()) {
+      if (trgclass.cluster->maskCluster[o2::detectors::DetID::EMC]) {
+        mEMCALTrgClassMask |= trgclass.classMask;
+      }
+    }
+    LOG(info) << "Loaded EMCAL trigger class mask: " << std::bitset<64>(mEMCALTrgClassMask);
+  }
 }
 
 void AODProducerWorkflowDPL::addRefGlobalBCsForTOF(const o2::dataformats::VtxTrackRef& trackRef, const gsl::span& GIndices,
@@ -2015,6 +3097,9 @@ void AODProducerWorkflowDPL::addRefGlobalBCsForTOF(const o2::dataformats::VtxTra
       float tofExpMom = 0.;
       if (tofInt.getTOF(o2::track::PID::Pion) > 0.f) {
         float expBeta = (intLen / (tofInt.getTOF(o2::track::PID::Pion) * cSpeed));
+        if (expBeta > o2::constants::math::Almost1) {
+          expBeta = o2::constants::math::Almost1;
+        }
         tofExpMom = o2::constants::physics::MassPionCharged * expBeta / std::sqrt(1.f - expBeta * expBeta);
       } else {
         continue;
@@ -2056,55 +3141,126 @@ std::uint64_t AODProducerWorkflowDPL::fillBCSlice(int (&slice)[2], double tmin,
   // The track time in the TrackExtraInfo is stored in ns wrt the collision BC for unambigous tracks and wrt bcSlice[0] for ambiguous ones,
   // with convention for errors: trackSigma in case (1) and half of the time interval for case (2) above.
 
-  // find indices of widest slice of global BCs in the map compatible with provided BC range. bcsMap is guaranteed to be non-empty
+  // find indices of widest slice of global BCs in the map compatible with provided BC range. bcsMap is guaranteed to be non-empty.
+  // We also assume that tmax >= tmin.
+
   uint64_t bcMin = relativeTime_to_GlobalBC(tmin), bcMax = relativeTime_to_GlobalBC(tmax);
-  auto lower = bcsMap.lower_bound(bcMin), upper = bcsMap.upper_bound(bcMax);
-  if (lower == bcsMap.end()) {
-    --lower;
-  }
-  if (upper != lower) {
-    --upper;
-  }
-  slice[0] = std::distance(bcsMap.begin(), lower);
-  slice[1] = std::distance(bcsMap.begin(), upper);
-  auto bcOfTimeRef = lower->first - this->mStartIR.toLong();
-  LOG(debug) << "BC slice t:" << tmin << " " << slice[0] << "(" << lower->first << "/" << lower->second << ")"
-             << " t: " << tmax << " " << slice[1] << "(" << upper->first << "/" << upper->second << ")"
+
+  /*
+    // brute force way of searching bcs via direct binary search in the map
+    auto lower = bcsMap.lower_bound(bcMin), upper = bcsMap.upper_bound(bcMax);
+
+    if (lower == bcsMap.end()) {
+      --lower;
+    }
+    if (upper != lower) {
+      --upper;
+    }
+    slice[0] = std::distance(bcsMap.begin(), lower);
+    slice[1] = std::distance(bcsMap.begin(), upper);
+  */
+
+  // faster way to search in bunch crossing via the accelerated bunch crossing lookup structure
+  auto p = mBCLookup.lower_bound(bcMin);
+  // assuming that bcMax will be >= bcMin and close to bcMin; we can find
+  // the upper bound quickly by lineary iterating from p.first to the point where
+  // the time becomes larger than bcMax.
+  // (if this is not the case we could determine it with a similar call to mBCLookup)
+  auto& bcvector = mBCLookup.getBCTimeVector();
+  auto upperindex = p.first;
+  while (upperindex < bcvector.size() && bcvector[upperindex] <= bcMax) {
+    upperindex++;
+  }
+  if (upperindex != p.first) {
+    upperindex--;
+  }
+  slice[0] = p.first;
+  slice[1] = upperindex;
+
+  auto bcOfTimeRef = p.second - this->mStartIR.toLong();
+  LOG(debug) << "BC slice t:" << tmin << " " << slice[0]
+             << " t: " << tmax << " " << slice[1]
              << " bcref: " << bcOfTimeRef;
   return bcOfTimeRef;
 }
 
-void AODProducerWorkflowDPL::endOfStream(EndOfStreamContext& ec)
+std::vector AODProducerWorkflowDPL::fillBCFlags(const o2::globaltracking::RecoContainer& data, std::map& bcsMap) const
+{
+  std::vector flags(bcsMap.size());
+
+  // flag BCs belonging to UPC mode ITS ROFs
+  auto bcIt = bcsMap.begin();
+  auto itsrofs = data.getITSTracksROFRecords();
+  auto lROF = o2::itsmft::DPLAlpideParam::Instance().roFrameLengthInBC;
+  auto bROF = o2::itsmft::DPLAlpideParam::Instance().roFrameBiasInBC;
+  for (auto& rof : itsrofs) {
+    if (!rof.getFlag(o2::itsmft::ROFRecord::VtxUPCMode)) {
+      continue;
+    }
+    uint64_t globalBC0 = rof.getBCData().toLong() + bROF, globalBC1 = globalBC0 + lROF - 1;
+    // BCs are sorted, iterate until the start of ROF
+    while (bcIt != bcsMap.end()) {
+      if (bcIt->first < globalBC0) {
+        ++bcIt;
+        continue;
+      }
+      if (bcIt->first > globalBC1) {
+        break;
+      }
+      flags[bcIt->second] |= o2::aod::bc::ITSUPCMode;
+      ++bcIt;
+    }
+  }
+  return flags;
+}
+
+void AODProducerWorkflowDPL::endOfStream(EndOfStreamContext& /*ec*/)
 {
   LOGF(info, "aod producer dpl total timing: Cpu: %.3e Real: %.3e s in %d slots",
        mTimer.CpuTime(), mTimer.RealTime(), mTimer.Counter() - 1);
+
+  mStreamer.reset();
 }
 
-DataProcessorSpec getAODProducerWorkflowSpec(GID::mask_t src, bool enableSV, bool useMC, std::string resFile)
+DataProcessorSpec getAODProducerWorkflowSpec(GID::mask_t src, bool enableSV, bool enableStrangenessTracking, bool useMC, bool CTPConfigPerRun, bool enableFITextra)
 {
-  std::vector outputs;
   auto dataRequest = std::make_shared();
-
-  dataRequest->inputs.emplace_back("ctpconfig", "CTP", "CTPCONFIG", 0, Lifetime::Condition, ccdbParamSpec("CTP/Config/Config"));
+  dataRequest->inputs.emplace_back("ctpconfig", "CTP", "CTPCONFIG", 0, Lifetime::Condition, ccdbParamSpec("CTP/Config/Config", CTPConfigPerRun));
 
   dataRequest->requestTracks(src, useMC);
-  dataRequest->requestPrimaryVertertices(useMC);
+  dataRequest->requestPrimaryVertices(useMC);
   if (src[GID::CTP]) {
-    LOGF(info, "Requesting CTP digits");
     dataRequest->requestCTPDigits(useMC);
   }
   if (enableSV) {
-    dataRequest->requestSecondaryVertertices(useMC);
+    dataRequest->requestSecondaryVertices(useMC);
+  }
+  if (enableStrangenessTracking) {
+    dataRequest->requestStrangeTracks(useMC);
+    LOGF(info, "requestStrangeTracks Finish");
+  }
+  if (src[GID::ITS]) {
+    dataRequest->requestClusters(GIndex::getSourcesMask("ITS"), false);
   }
   if (src[GID::TPC]) {
     dataRequest->requestClusters(GIndex::getSourcesMask("TPC"), false); // no need to ask for TOF clusters as they are requested with TOF tracks
   }
+  if (src[GID::TOF]) {
+    dataRequest->requestTOFClusters(useMC);
+  }
   if (src[GID::PHS]) {
     dataRequest->requestPHOSCells(useMC);
   }
+  if (src[GID::TRD]) {
+    dataRequest->requestTRDTracklets(false);
+  }
   if (src[GID::EMC]) {
     dataRequest->requestEMCALCells(useMC);
   }
+  if (src[GID::CPV]) {
+    dataRequest->requestCPVClusters(useMC);
+  }
+
   auto ggRequest = std::make_shared(true,                              // orbitResetTime
                                                               true,                              // GRPECS=true
                                                               true,                              // GRPLHCIF
@@ -2114,39 +3270,73 @@ DataProcessorSpec getAODProducerWorkflowSpec(GID::mask_t src, bool enableSV, boo
                                                               dataRequest->inputs,
                                                               true); // query only once all objects except mag.field
 
-  outputs.emplace_back(OutputLabel{"O2bc"}, "AOD", "BC", 0, Lifetime::Timeframe);
-  outputs.emplace_back(OutputLabel{"O2cascade_001"}, "AOD", "CASCADE_001", 0, Lifetime::Timeframe);
-  outputs.emplace_back(OutputLabel{"O2collision"}, "AOD", "COLLISION", 0, Lifetime::Timeframe);
-  outputs.emplace_back(OutputLabel{"O2fdd_001"}, "AOD", "FDD_001", 0, Lifetime::Timeframe);
-  outputs.emplace_back(OutputLabel{"O2ft0"}, "AOD", "FT0", 0, Lifetime::Timeframe);
-  outputs.emplace_back(OutputLabel{"O2fv0a"}, "AOD", "FV0A", 0, Lifetime::Timeframe);
-  outputs.emplace_back(OutputLabel{"O2fwdtrack"}, "AOD", "FWDTRACK", 0, Lifetime::Timeframe);
-  outputs.emplace_back(OutputLabel{"O2fwdtrackcov"}, "AOD", "FWDTRACKCOV", 0, Lifetime::Timeframe);
-  outputs.emplace_back(OutputLabel{"O2mccollision"}, "AOD", "MCCOLLISION", 0, Lifetime::Timeframe);
-  outputs.emplace_back(OutputLabel{"O2mccollisionlabel"}, "AOD", "MCCOLLISIONLABEL", 0, Lifetime::Timeframe);
-  outputs.emplace_back(OutputLabel{"O2mcmfttracklabel"}, "AOD", "MCMFTTRACKLABEL", 0, Lifetime::Timeframe);
-  outputs.emplace_back(OutputLabel{"O2mcfwdtracklabel"}, "AOD", "MCFWDTRACKLABEL", 0, Lifetime::Timeframe);
-  outputs.emplace_back(OutputLabel{"O2mcparticle_001"}, "AOD", "MCPARTICLE_001", 0, Lifetime::Timeframe);
-  outputs.emplace_back(OutputLabel{"O2mctracklabel"}, "AOD", "MCTRACKLABEL", 0, Lifetime::Timeframe);
-  outputs.emplace_back(OutputLabel{"O2mfttrack"}, "AOD", "MFTTRACK", 0, Lifetime::Timeframe);
-  outputs.emplace_back(OutputLabel{"O2track_iu"}, "AOD", "TRACK_IU", 0, Lifetime::Timeframe);
-  outputs.emplace_back(OutputLabel{"O2trackcov_iu"}, "AOD", "TRACKCOV_IU", 0, Lifetime::Timeframe);
-  outputs.emplace_back(OutputLabel{"O2trackextra"}, "AOD", "TRACKEXTRA", 0, Lifetime::Timeframe);
-  outputs.emplace_back(OutputLabel{"O2ambiguoustrack"}, "AOD", "AMBIGUOUSTRACK", 0, Lifetime::Timeframe);
-  outputs.emplace_back(OutputLabel{"O2ambiguousMFTtrack"}, "AOD", "AMBIGUOUSMFTTR", 0, Lifetime::Timeframe);
-  outputs.emplace_back(OutputLabel{"O2ambiguousFwdtrack"}, "AOD", "AMBIGUOUSFWDTR", 0, Lifetime::Timeframe);
-  outputs.emplace_back(OutputLabel{"O2v0_001"}, "AOD", "V0_001", 0, Lifetime::Timeframe);
-  outputs.emplace_back(OutputLabel{"O2zdc"}, "AOD", "ZDC", 0, Lifetime::Timeframe);
-  outputs.emplace_back(OutputLabel{"O2caloCell"}, "AOD", "CALO", 0, Lifetime::Timeframe);
-  outputs.emplace_back(OutputLabel{"O2caloCellTRGR"}, "AOD", "CALOTRIGGER", 0, Lifetime::Timeframe);
-  outputs.emplace_back(OutputLabel{"O2origin"}, "AOD", "ORIGIN", 0, Lifetime::Timeframe);
-  outputs.emplace_back(OutputSpec{"TFN", "TFNumber"});
+  dataRequest->inputs.emplace_back("meanvtx", "GLO", "MEANVERTEX", 0, Lifetime::Condition, ccdbParamSpec("GLO/Calib/MeanVertex", {}, 1));
+
+  using namespace o2::aod;
+  using namespace o2::aodproducer;
+
+  std::vector outputs{
+    OutputForTable::spec(),
+    OutputForTable::spec(),
+    OutputForTable::spec(),
+    OutputForTable::spec(),
+    OutputForTable::spec(),
+    OutputForTable::spec(),
+    OutputForTable::spec(),
+    OutputForTable::spec(),
+    OutputForTable::spec(),
+    OutputForTable::spec(),
+    OutputForTable::spec(),
+    OutputForTable::spec(),
+    OutputForTable::spec(),
+    OutputForTable::spec(),
+    OutputForTable::spec(),
+    OutputForTable::spec(),
+    OutputForTable::spec(),
+    OutputForTable::spec(),
+    OutputForTable::spec(),
+    OutputForTable::spec(),
+    OutputForTable::spec(),
+    OutputForTable::spec(),
+    OutputForTable::spec(),
+    OutputForTable::spec(),
+    OutputForTable::spec(),
+    OutputForTable::spec(),
+    OutputForTable::spec(),
+    OutputForTable::spec(),
+    OutputForTable::spec(),
+    OutputForTable::spec(),
+    OutputForTable::spec(),
+    OutputForTable::spec(),
+    OutputForTable::spec(),
+    OutputSpec{"TFN", "TFNumber"},
+    OutputSpec{"TFF", "TFFilename"},
+    OutputSpec{"AMD", "AODMetadataKeys"},
+    OutputSpec{"AMD", "AODMetadataVals"}};
+
+  if (useMC) {
+    outputs.insert(outputs.end(),
+                   {OutputForTable::spec(),
+                    OutputForTable::spec(),
+                    OutputForTable::spec(),
+                    OutputForTable::spec(),
+                    OutputForTable::spec(),
+                    OutputForTable::spec(),
+                    OutputForTable::spec(),
+                    OutputForTable::spec(),
+                    OutputForTable::spec(),
+                    // todo: use addTableToOuput helper?
+                    //  currently the description is MCCOLLISLABEL, so
+                    //  the name in AO2D would be O2mccollislabel
+                    // addTableToOutput(outputs);
+                    {OutputLabel{"McCollisionLabels"}, "AOD", "MCCOLLISIONLABEL", 0, Lifetime::Timeframe}});
+  }
 
   return DataProcessorSpec{
     "aod-producer-workflow",
     dataRequest->inputs,
     outputs,
-    AlgorithmSpec{adaptFromTask(src, dataRequest, ggRequest, enableSV, resFile, useMC)},
+    AlgorithmSpec{adaptFromTask(src, dataRequest, ggRequest, enableSV, useMC, enableFITextra)},
     Options{
       ConfigParamSpec{"run-number", VariantType::Int64, -1L, {"The run-number. If left default we try to get it from DPL header."}},
       ConfigParamSpec{"aod-timeframe-id", VariantType::Int64, -1L, {"Set timeframe number"}},
@@ -2156,8 +3346,26 @@ DataProcessorSpec getAODProducerWorkflowSpec(GID::mask_t src, bool enableSV, boo
       ConfigParamSpec{"anchor-pass", VariantType::String, "", {"AnchorPassName"}},
       ConfigParamSpec{"anchor-prod", VariantType::String, "", {"AnchorProduction"}},
       ConfigParamSpec{"reco-pass", VariantType::String, "", {"RecoPassName"}},
+      ConfigParamSpec{"created-by", VariantType::String, "", {"Who created this AO2D"}},
+      ConfigParamSpec{"nthreads", VariantType::Int, std::max(1, int(std::thread::hardware_concurrency() / 2)), {"Number of threads"}},
       ConfigParamSpec{"reco-mctracks-only", VariantType::Int, 0, {"Store only reconstructed MC tracks and their mothers/daughters. 0 -- off, != 0 -- on"}},
-      ConfigParamSpec{"ctpreadout-create", VariantType::Int, 0, {"Create CTP digits from detector readout and CTP inputs. !=1 -- off, 1 -- on"}}}};
+      ConfigParamSpec{"ctpreadout-create", VariantType::Int, 0, {"Create CTP digits from detector readout and CTP inputs. !=1 -- off, 1 -- on"}},
+      ConfigParamSpec{"emc-select-leading", VariantType::Bool, false, {"Flag to select if only the leading contributing particle for an EMCal cell should be stored"}},
+      ConfigParamSpec{"propagate-tracks", VariantType::Bool, false, {"Propagate tracks (not used for secondary vertices) to IP"}},
+      ConfigParamSpec{"propagate-tracks-max-xiu", VariantType::Float, 5.0f, {"Propagate tracks to IP if X_IU smaller than this value (and if propagate tracks enabled)"}},
+      ConfigParamSpec{"hepmc-update", VariantType::String, "always", {"When to update HepMC Aux tables: always - force update, never - never update, all - if all keys are present, any - when any key is present (not valid yet)"}},
+      ConfigParamSpec{"propagate-muons", VariantType::Bool, false, {"Propagate muons to IP"}},
+      ConfigParamSpec{"thin-tracks", VariantType::Bool, false, {"Produce thinned track tables"}},
+      ConfigParamSpec{"trackqc-keepglobaltracks", VariantType::Bool, false, {"Always keep TrackQA for global tracks"}},
+      ConfigParamSpec{"trackqc-retainonlydedx", VariantType::Bool, false, {"Keep only dEdx information, zero out everything else"}},
+      ConfigParamSpec{"trackqc-fraction", VariantType::Float, float(0.1), {"Fraction of tracks to QC"}},
+      ConfigParamSpec{"trackqc-NTrCut", VariantType::Int64, 4L, {"Minimal length of the track - in amount of tracklets"}},
+      ConfigParamSpec{"trackqc-tpc-dca", VariantType::Float, 3.f, {"Keep TPC standalone track with this DCAxy to the PV"}},
+      ConfigParamSpec{"trackqc-tpc-cls", VariantType::Int, 80, {"Keep TPC standalone track with this #clusters"}},
+      ConfigParamSpec{"trackqc-tpc-pt", VariantType::Float, 0.2f, {"Keep TPC standalone track with this pt"}},
+      ConfigParamSpec{"with-streamers", VariantType::String, "", {"Bit-mask to steer writing of intermediate streamer files"}},
+      ConfigParamSpec{"seed", VariantType::Int, 0, {"Set seed for random generator used for sampling (0 (default) means using a random_device)"}},
+      ConfigParamSpec{"mc-signal-filt", VariantType::Bool, false, {"Enable usage of signal filtering (only for MC with embedding)"}}}};
 }
 
 } // namespace o2::aodproducer
diff --git a/Detectors/AOD/src/StandaloneAODProducer.cxx b/Detectors/AOD/src/StandaloneAODProducer.cxx
index 151038465c1d1..7ac59a556ac08 100644
--- a/Detectors/AOD/src/StandaloneAODProducer.cxx
+++ b/Detectors/AOD/src/StandaloneAODProducer.cxx
@@ -86,7 +86,7 @@ void fillMCollisionTable(o2::steer::MCKinematicsReader const& mcreader)
     //                  mccollision::PosX, mccollision::PosY, mccollision::PosZ, mccollision::T, mccollision::Weight,
     //                 mccollision::ImpactParameter);
 
-    mcCollCursor(0, 0 /*bcID*/, 0 /*genID*/, header.GetX(), header.GetY(), header.GetZ(), time, 1. /*weight*/, header.GetB());
+    mcCollCursor(0, 0 /*bcID*/, 0 /*genID*/, header.GetX(), header.GetY(), header.GetZ(), time, 1. /*weight*/, header.GetB(), 0.0);
 
     index++;
   }
@@ -94,7 +94,7 @@ void fillMCollisionTable(o2::steer::MCKinematicsReader const& mcreader)
 
   TFile outfile("aod.root", "UPDATE");
   {
-    TableToTree t2t(mccoltable, &outfile, aod::MetadataTrait::metadata::tableLabel());
+    TableToTree t2t(mccoltable, &outfile, aod::description_str(aod::signature()).data());
     t2t.addAllBranches();
     t2t.process();
   }
@@ -200,12 +200,12 @@ void fillCollisionAndTrackTable()
       f.Close();
       TFile outfile("aod.root", "RECREATE");
       {
-        TableToTree t2t(colltable, &outfile, aod::MetadataTrait::metadata::tableLabel());
+        TableToTree t2t(colltable, &outfile, aod::description_str(aod::signature()).data());
         t2t.addAllBranches();
         t2t.process();
       }
       {
-        TableToTree t2t(tracktable, &outfile, "Tracks" /* aod::MetadataTrait::metadata::tableLabel() */);
+        TableToTree t2t(tracktable, &outfile, aod::description_str(aod::signature()).data());
         t2t.addAllBranches();
         t2t.process();
       }
diff --git a/Detectors/AOD/src/aod-mc-producer-workflow.cxx b/Detectors/AOD/src/aod-mc-producer-workflow.cxx
new file mode 100644
index 0000000000000..4f32bacac6fef
--- /dev/null
+++ b/Detectors/AOD/src/aod-mc-producer-workflow.cxx
@@ -0,0 +1,46 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "AODProducerWorkflow/AODMcProducerWorkflowSpec.h"
+#include "Framework/CallbacksPolicy.h"
+#include "Framework/CompletionPolicy.h"
+#include "DetectorsRaw/HBFUtilsInitializer.h"
+#include "DetectorsBase/DPLWorkflowUtils.h"
+
+using namespace o2::framework;
+
+void customize(std::vector& policies)
+{
+  o2::raw::HBFUtilsInitializer::addNewTimeSliceCallback(policies);
+}
+
+void customize(std::vector& workflowOptions)
+{
+  // option allowing to set parameters
+  std::vector options{
+    {"disable-root-input", o2::framework::VariantType::Bool, false, {"disable root-files input reader"}},
+    {"disable-root-output", o2::framework::VariantType::Bool, false, {"disable root-files output writer"}},
+    {"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings ..."}}};
+  std::swap(workflowOptions, options);
+}
+
+#include "Framework/runDataProcessing.h"
+
+WorkflowSpec defineDataProcessing(ConfigContext const& configcontext)
+{
+  o2::conf::ConfigurableParam::updateFromString(configcontext.options().get("configKeyValues"));
+  WorkflowSpec wf;
+  wf.emplace_back(o2::aodmcproducer::getAODMcProducerWorkflowSpec());
+  return std::move(wf);
+}
+//
+// EOF
+//
diff --git a/Detectors/AOD/src/aod-producer-workflow.cxx b/Detectors/AOD/src/aod-producer-workflow.cxx
index 6e54644aa2c4a..81e178642e403 100644
--- a/Detectors/AOD/src/aod-producer-workflow.cxx
+++ b/Detectors/AOD/src/aod-producer-workflow.cxx
@@ -36,9 +36,12 @@ void customize(std::vector& workflowOptions)
     {"disable-root-output", o2::framework::VariantType::Bool, false, {"disable root-files output writer"}},
     {"disable-mc", o2::framework::VariantType::Bool, false, {"disable MC propagation"}},
     {"disable-secondary-vertices", o2::framework::VariantType::Bool, false, {"disable filling secondary vertices"}},
+    {"disable-strangeness-tracker", o2::framework::VariantType::Bool, false, {"disable filling strangeness tracking"}},
+    {"enable-FIT-extra", o2::framework::VariantType::Bool, false, {"enable FIT extra output"}},
     {"info-sources", VariantType::String, std::string{GID::ALL}, {"comma-separated list of sources to use"}},
     {"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings ..."}},
-    {"combine-source-devices", o2::framework::VariantType::Bool, false, {"merge DPL source devices"}}};
+    {"combine-source-devices", o2::framework::VariantType::Bool, false, {"merge DPL source devices"}},
+    {"ctpconfig-run-independent", o2::framework::VariantType::Bool, false, {"Use CTP config w/o runNumber tag"}}};
   o2::raw::HBFUtilsInitializer::addConfigOption(options);
   std::swap(workflowOptions, options);
 }
@@ -49,14 +52,21 @@ WorkflowSpec defineDataProcessing(ConfigContext const& configcontext)
 {
   o2::conf::ConfigurableParam::updateFromString(configcontext.options().get("configKeyValues"));
   auto useMC = !configcontext.options().get("disable-mc");
-  auto resFile = configcontext.options().get("aod-writer-resfile");
   bool enableSV = !configcontext.options().get("disable-secondary-vertices");
+  bool enableST = !configcontext.options().get("disable-strangeness-tracker");
+  bool ctpcfgperrun = !configcontext.options().get("ctpconfig-run-independent");
+  bool enableFITextra = configcontext.options().get("enable-FIT-extra");
 
-  GID::mask_t allowedSrc = GID::getSourcesMask("ITS,MFT,MCH,MID,MCH-MID,TPC,ITS-TPC,TPC-TOF,TPC-TRD,ITS-TPC-TOF,ITS-TPC-TRD,TPC-TRD-TOF,ITS-TPC-TRD-TOF,MFT-MCH,FT0,FV0,FDD,ZDC,EMC,CTP,PHS");
+  GID::mask_t allowedSrc = GID::getSourcesMask("ITS,MFT,MCH,MID,MCH-MID,TPC,TRD,ITS-TPC,TPC-TOF,TPC-TRD,ITS-TPC-TOF,ITS-TPC-TRD,TPC-TRD-TOF,ITS-TPC-TRD-TOF,MFT-MCH,FT0,FV0,FDD,ZDC,EMC,CTP,PHS,CPV,HMP");
   GID::mask_t src = allowedSrc & GID::getSourcesMask(configcontext.options().get("info-sources"));
 
+  // manually add TOF to MC mask for addInputSpecs()
+  if (src[GID::TPCTOF] || src[GID::ITSTPCTRDTOF] || src[GID::ITSTPCTOF] || src[GID::TPCTRDTOF]) {
+    src.set(o2::detectors::DetID::TOF);
+  }
+
   WorkflowSpec specs;
-  specs.emplace_back(o2::aodproducer::getAODProducerWorkflowSpec(src, enableSV, useMC, resFile));
+  specs.emplace_back(o2::aodproducer::getAODProducerWorkflowSpec(src, enableSV, enableST, useMC, ctpcfgperrun, enableFITextra));
 
   auto srcCls = src & ~(GID::getSourceMask(GID::MCH) | GID::getSourceMask(GID::MID)); // Don't read global MID and MCH clusters (those attached to tracks are always read)
   auto srcMtc = src;
@@ -67,6 +77,9 @@ WorkflowSpec defineDataProcessing(ConfigContext const& configcontext)
   if (enableSV) {
     o2::globaltracking::InputHelper::addInputSpecsSVertex(configcontext, inputspecs);
   }
+  if (enableST) {
+    o2::globaltracking::InputHelper::addInputSpecsStrangeTrack(configcontext, inputspecs, useMC);
+  }
   if (configcontext.options().get("combine-source-devices")) {
     std::vector unmerged;
     specs.push_back(specCombiner("AOD-input-reader", inputspecs, unmerged));
diff --git a/Detectors/Align/CMakeLists.txt b/Detectors/Align/CMakeLists.txt
index b13d377bbb9c5..d99e1a803612f 100644
--- a/Detectors/Align/CMakeLists.txt
+++ b/Detectors/Align/CMakeLists.txt
@@ -9,16 +9,17 @@
 # granted to it by virtue of its status as an Intergovernmental Organization
 # or submit itself to any jurisdiction.
 
+#add_compile_options(-O0 -g -fPIC)
+
 o2_add_library(Align
                SOURCES  src/GeometricalConstraint.cxx
                         src/DOFSet.cxx
                         src/AlignableDetector.cxx
                         src/AlignableDetectorITS.cxx
                         #src/AlignableDetectorHMPID.cxx
-                        #src/AlignableDetectorTOF.cxx
-                        #src/AlignableDetectorTPC.cxx
-                        #src/AlignableDetectorTRD.cxx
-                        src/DOFStatistics.cxx
+                        src/AlignableDetectorTOF.cxx
+                        src/AlignableDetectorTPC.cxx
+                        src/AlignableDetectorTRD.cxx
                         src/Millepede2Record.cxx
                         src/AlignmentPoint.cxx
                         src/ResidualsController.cxx
@@ -26,26 +27,31 @@ o2_add_library(Align
                         src/AlignableSensor.cxx
                         #src/AlignableSensorHMPID.cxx
                         src/AlignableSensorITS.cxx
-                        #src/AlignableSensorTOF.cxx
-                        #src/AlignableSensorTPC.cxx
-                        #src/AlignableSensorTRD.cxx
+                        src/AlignableSensorTOF.cxx
+                        src/AlignableSensorTPC.cxx
+                        src/AlignableSensorTRD.cxx
                         src/Controller.cxx
                         src/AlignmentTrack.cxx
                         src/AlignableVolume.cxx
                         src/EventVertex.cxx
                         src/AlignConfig.cxx
                         src/Mille.cxx
+                        src/AlgPntDbg.cxx
+                        src/AlgTrcDbg.cxx
                PUBLIC_LINK_LIBRARIES O2::FrameworkLogger
+                                     O2::Steer
                                      O2::ReconstructionDataFormats
                                      O2::DetectorsCommonDataFormats
                                      O2::DetectorsBase
                                      O2::ITSBase
                                      O2::ITStracking
+                                     O2::TRDBase
+                                     O2::TOFBase
                                      O2::DataFormatsGlobalTracking
+                                     O2::TPCCalibration
+                                     O2::GPUTracking
                                      ROOT::Core
                                      ROOT::Geom
-                                     ROOT::Gpad
-                                     ROOT::Graf3d
                                      ROOT::MathCore
                                      ROOT::Matrix
                                      ROOT::Hist
@@ -57,17 +63,17 @@ o2_target_root_dictionary(
   HEADERS include/Align/DOFSet.h
           include/Align/AlignableDetector.h
           include/Align/AlignableDetectorITS.h
-          #include/Align/AlignableDetectorTOF.h
-          #include/Align/AlignableDetectorTPC.h
-          #include/Align/AlignableDetectorTRD.h
+          include/Align/AlignableDetectorTOF.h
+          include/Align/AlignableDetectorTPC.h
+          include/Align/AlignableDetectorTRD.h
           #include/Align/AlignableDetectorHMPID.h
           include/Align/Millepede2Record.h
           include/Align/AlignmentPoint.h
           include/Align/AlignableSensor.h
           include/Align/AlignableSensorITS.h
-          #include/Align/AlignableSensorTOF.h
-          #include/Align/AlignableSensorTPC.h
-          #include/Align/AlignableSensorTRD.h
+          include/Align/AlignableSensorTOF.h
+          include/Align/AlignableSensorTPC.h
+          include/Align/AlignableSensorTRD.h
           #include/Align/AlignableSensorHMPID.h
           include/Align/Controller.h
           include/Align/AlignmentTrack.h
@@ -76,9 +82,11 @@ o2_target_root_dictionary(
           include/Align/ResidualsController.h
           include/Align/ResidualsControllerFast.h
           include/Align/GeometricalConstraint.h
-          include/Align/DOFStatistics.h
           include/Align/utils.h
           include/Align/AlignConfig.h
+          include/Align/AlgPntDbg.h
+          include/Align/AlgTrcDbg.h
           )
 
 add_subdirectory(Workflow)
+add_subdirectory(macro)
diff --git a/Detectors/Align/Workflow/CMakeLists.txt b/Detectors/Align/Workflow/CMakeLists.txt
index c6f2b9916de16..8ceb50f71ca29 100644
--- a/Detectors/Align/Workflow/CMakeLists.txt
+++ b/Detectors/Align/Workflow/CMakeLists.txt
@@ -1,17 +1,16 @@
-# Copyright CERN and copyright holders of ALICE O2. This software is distributed
-# under the terms of the GNU General Public License v3 (GPL Version 3), copied
-# verbatim in the file "COPYING".
+# Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+# See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+# All rights not expressly granted are reserved.
 #
-# See http://alice-o2.web.cern.ch/license for full licensing information.
+# This software is distributed under the terms of the GNU General Public
+# License v3 (GPL Version 3), copied verbatim in the file "COPYING".
 #
 # In applying this license CERN does not waive the privileges and immunities
-# granted to it by virtue of its status as an Intergovernmental Organization or
-# submit itself to any jurisdiction.
-
-# FIXME: do we actually need a library here, or is the executable enough ?
+# granted to it by virtue of its status as an Intergovernmental Organization
+# or submit itself to any jurisdiction.
 
 o2_add_library(AlignmentWorkflow
-               SOURCES  src/BarrelAlignmentSpec.cxx
+               SOURCES src/BarrelAlignmentSpec.cxx
                PUBLIC_LINK_LIBRARIES O2::Align
                                      O2::FrameworkLogger
                                      O2::DataFormatsGlobalTracking
@@ -23,6 +22,8 @@ o2_add_library(AlignmentWorkflow
                                      O2::DataFormatsTPC
                                      O2::DataFormatsTRD
                                      O2::DataFormatsTOF
+                                     O2::GPUTracking
+                                     O2::TPCCalibration
                                      ROOT::Core
                                      ROOT::Geom
                                      ROOT::MathCore
diff --git a/Detectors/Align/Workflow/include/AlignmentWorkflow/BarrelAlignmentSpec.h b/Detectors/Align/Workflow/include/AlignmentWorkflow/BarrelAlignmentSpec.h
index 4172af26796ce..197ace2bd9d20 100644
--- a/Detectors/Align/Workflow/include/AlignmentWorkflow/BarrelAlignmentSpec.h
+++ b/Detectors/Align/Workflow/include/AlignmentWorkflow/BarrelAlignmentSpec.h
@@ -21,11 +21,17 @@ using namespace o2::framework;
 
 namespace o2
 {
+namespace tpc
+{
+struct CorrectionMapsLoaderGloOpts;
+}
+
 namespace align
 {
 
 /// create a processor spec
-framework::DataProcessorSpec getBarrelAlignmentSpec(o2::dataformats::GlobalTrackID::mask_t src, o2::detectors::DetID::mask_t dets);
+framework::DataProcessorSpec getBarrelAlignmentSpec(o2::dataformats::GlobalTrackID::mask_t srcMP, o2::dataformats::GlobalTrackID::mask_t src,
+                                                    o2::detectors::DetID::mask_t dets, o2::detectors::DetID::mask_t skipDetClusters, bool enableCosmic, int postproc, bool useMC, const o2::tpc::CorrectionMapsLoaderGloOpts& sclOpts);
 
 } // namespace align
 } // namespace o2
diff --git a/Detectors/Align/Workflow/src/BarrelAlignmentSpec.cxx b/Detectors/Align/Workflow/src/BarrelAlignmentSpec.cxx
index 3d08dd4adfe1b..d4ab53c8181ce 100644
--- a/Detectors/Align/Workflow/src/BarrelAlignmentSpec.cxx
+++ b/Detectors/Align/Workflow/src/BarrelAlignmentSpec.cxx
@@ -13,10 +13,16 @@
 
 #include 
 #include 
-#include "TStopwatch.h"
+#include 
+#include 
+#include 
+#include 
+#include 
+#include "TMethodCall.h"
 #include "AlignmentWorkflow/BarrelAlignmentSpec.h"
 #include "Align/AlignableDetectorITS.h"
 #include "Align/Controller.h"
+#include "Align/AlignConfig.h"
 #include "DetectorsBase/GRPGeomHelper.h"
 #include "DataFormatsGlobalTracking/RecoContainerCreateTracksVariadic.h"
 #include "DataFormatsGlobalTracking/RecoContainer.h"
@@ -26,12 +32,22 @@
 #include "CommonUtils/NameConf.h"
 #include "DetectorsBase/Propagator.h"
 #include "DetectorsBase/GeometryManager.h"
-
 #include "DataFormatsITSMFT/TopologyDictionary.h"
-
+#include "TRDBase/TrackletTransformer.h"
+#include "CommonUtils/TreeStreamRedirector.h"
+#include "TPCCalibration/VDriftHelper.h"
+#include "TPCCalibration/CorrectionMapsLoader.h"
+#include "GPUO2ExternalUser.h"
+#include "GPUO2InterfaceUtils.h"
+#include "GPUParam.h"
 #include "Headers/DataHeader.h"
 #include "Framework/ConfigParamRegistry.h"
 #include "Framework/Task.h"
+#include "Framework/CCDBParamSpec.h"
+#include "Framework/ControlService.h"
+#include "Framework/DeviceSpec.h"
+
+#include 
 
 /*
 #include "DataFormatsITSMFT/TopologyDictionary.h"
@@ -66,8 +82,18 @@ namespace align
 class BarrelAlignmentSpec : public Task
 {
  public:
-  BarrelAlignmentSpec(std::shared_ptr dr, std::shared_ptr ggrec, DetID::mask_t m)
-    : mDataRequest(dr), mGRPGeomRequest(ggrec), mDetMask{m} {}
+  enum PostProc { WriteResults = 0x1 << 0,
+                  CheckConstaints = 0x1 << 1,
+                  GenPedeFiles = 0x1 << 2,
+                  LabelPedeResults = 0x1 << 3 };
+  BarrelAlignmentSpec(GTrackID::mask_t srcMP, std::shared_ptr dr, std::shared_ptr ggrec, const o2::tpc::CorrectionMapsLoaderGloOpts& tpcOpt,
+                      DetID::mask_t detmask, bool cosmic, int postprocess, bool useMC, bool loadTPCCalib)
+    : mDataRequest(dr), mGRPGeomRequest(ggrec), mMPsrc{srcMP}, mDetMask{detmask}, mCosmic(cosmic), mPostProcessing(postprocess), mUseMC(useMC), mLoadTPCCalib(loadTPCCalib)
+  {
+    mTPCCorrMapsLoader.setLumiScaleType(tpcOpt.lumiType);
+    mTPCCorrMapsLoader.setLumiScaleMode(tpcOpt.lumiMode);
+    mTPCCorrMapsLoader.setCheckCTPIDCConsistency(tpcOpt.checkCTPIDCconsistency);
+  }
   ~BarrelAlignmentSpec() override = default;
   void init(InitContext& ic) final;
   void run(ProcessingContext& pc) final;
@@ -76,11 +102,29 @@ class BarrelAlignmentSpec : public Task
 
  private:
   void updateTimeDependentParams(ProcessingContext& pc);
-
+  std::string mIniParFile{};
+  bool mUseIniParErrors = true;
+  bool mUseMC = false;
+  bool mIgnoreCCDBAlignment = false;
+  bool mCosmic = false;
+  bool mLoadTPCCalib = false;
+  int mLane = 0;
+  int mPostProcessing = 0; // special mode of extracting alignment or constraints check
+  GTrackID::mask_t mMPsrc{};
   DetID::mask_t mDetMask{};
   std::unique_ptr mController;
   std::shared_ptr mDataRequest;
   std::shared_ptr mGRPGeomRequest;
+  std::string mConfMacro{};
+  std::unique_ptr mUsrConfMethod;
+  std::unique_ptr mTRDTransformer;
+  std::unique_ptr mDBGOut;
+  std::unique_ptr mTPCParam;
+
+  o2::tpc::VDriftHelper mTPCVDriftHelper{};
+  o2::tpc::CorrectionMapsLoader mTPCCorrMapsLoader{};
+
+  //
   TStopwatch mTimer;
 };
 
@@ -89,19 +133,164 @@ void BarrelAlignmentSpec::init(InitContext& ic)
   mTimer.Stop();
   mTimer.Reset();
   o2::base::GRPGeomHelper::instance().setRequest(mGRPGeomRequest);
-  mController = std::make_unique(mDetMask);
+
+  int dbg = ic.options().get("debug-output");
+  mLane = ic.services().get().inputTimesliceId;
+  mController = std::make_unique(mDetMask, mMPsrc, mCosmic, mUseMC, mLane);
+  if (dbg) {
+    mController->setDebugOutputLevel(dbg);
+  }
+
+  mConfMacro = ic.options().get("config-macro");
+  if (!mConfMacro.empty()) {
+    if (!std::filesystem::exists(mConfMacro)) {
+      LOG(fatal) << "Requested user macro " << mConfMacro << " does not exist";
+    }
+    std::string tmpmacro = mConfMacro + "+";
+    TString cmd = gSystem->GetMakeSharedLib();
+    cmd += " -O0 -g -ggdb";
+    // protect macro compilation by semaphore (to avoid problems in the pipelined code)
+    {
+      boost::interprocess::named_semaphore* sem = nullptr;
+      std::string semhashedstring{};
+      std::hash hasher;
+      semhashedstring = "align_macro_" + std::to_string(hasher(mConfMacro)).substr(0, 16);
+      try {
+        sem = new boost::interprocess::named_semaphore(boost::interprocess::open_or_create_t{}, semhashedstring.c_str(), 1);
+      } catch (std::exception e) {
+        LOGP(error, "Exception occurred during {} compilation semaphore setup", tmpmacro);
+        sem = nullptr;
+      }
+      if (sem) {
+        sem->wait(); // wait until we can enter (no one else there)
+      }
+      gSystem->SetMakeSharedLib(cmd.Data());
+      auto res = gROOT->LoadMacro(tmpmacro.c_str());
+      if (sem) {
+        sem->post();
+        if (sem->try_wait()) { // if nobody else is waiting remove the semaphore resource
+          sem->post();
+          boost::interprocess::named_semaphore::remove(semhashedstring.c_str());
+        }
+      }
+      if (res) {
+        LOG(fatal) << "Failed to load user macro " << tmpmacro;
+      }
+    }
+    std::filesystem::path mpth(mConfMacro);
+    mConfMacro = mpth.stem();
+    mUsrConfMethod = std::make_unique();
+    mUsrConfMethod->InitWithPrototype(mConfMacro.c_str(), "o2::align::Controller*, int");
+  }
+  mIgnoreCCDBAlignment = ic.options().get("ignore-ccdb-alignment");
+  if (!mPostProcessing) {
+    if (mLoadTPCCalib) {
+      mTPCCorrMapsLoader.init(ic);
+    }
+    if (GTrackID::includesDet(DetID::TRD, mMPsrc)) {
+      mTRDTransformer.reset(new o2::trd::TrackletTransformer);
+      if (ic.options().get("apply-xor")) {
+        mTRDTransformer->setApplyXOR();
+      }
+      auto prevShift = mTRDTransformer->isShiftApplied();
+      if (getenv("ALIEN_JDL_LPMPRODUCTIONTYPE") && std::strcmp(getenv("ALIEN_JDL_LPMPRODUCTIONTYPE"), "MC") == 0) {
+        // apply artificial pad shift in case non-ideal alignment is used to compensate for shift in current alignment from real data
+        mTRDTransformer->setApplyShift(false);
+      }
+      LOGP(info, "Old TRD shift : {} new : {}", prevShift, mTRDTransformer->isShiftApplied());
+      mController->setTRDTransformer(mTRDTransformer.get());
+    }
+    mController->setAllowAfterburnerTracks(ic.options().get("allow-afterburner-tracks"));
+  }
+
+  mIniParFile = ic.options().get("initial-params-file");
+  mUseIniParErrors = !ic.options().get("ignore-initial-params-errors");
+  if (mPostProcessing && !(mPostProcessing != GenPedeFiles) && (mIniParFile.empty() || mIniParFile == "none")) {
+    LOGP(warn, "Postprocessing {} is requested but the initial-params-file is not provided", mPostProcessing);
+  }
 }
 
 void BarrelAlignmentSpec::updateTimeDependentParams(ProcessingContext& pc)
 {
-  bool initOnceDone = false;
-  if (!initOnceDone) {
-    initOnceDone = true;
-    o2::base::GRPGeomHelper::instance().checkUpdates(pc);
+  o2::base::GRPGeomHelper::instance().checkUpdates(pc);
+  auto tinfo = pc.services().get();
+  if (pc.services().get().globalRunNumberChanged) {
+    if (mController->getDebugOutputLevel()) {
+      mDBGOut = std::make_unique(fmt::format("mpDebug_{}_{:08d}_{:010d}.root", mLane, tinfo.runNumber, tinfo.tfCounter).c_str(), "recreate");
+      mController->setDebugStream(mDBGOut.get());
+    }
+    if (!mIgnoreCCDBAlignment) {
+      for (auto id = DetID::First; id <= DetID::Last; id++) {
+        const auto* alg = o2::base::GRPGeomHelper::instance().getAlignment(id);
+        if (alg && !alg->empty()) {
+          o2::base::GeometryManager::applyAlignment(*alg);
+        }
+        gGeoManager->RefreshPhysicalNodes(false);
+      }
+    } else {
+      LOG(warn) << "CCDB alignment is NOT applied to ideal geometry";
+    }
     if (!mController->getInitGeomDone()) {
       mController->initDetectors();
     }
-    o2::base::PropagatorD::initFieldFromGRP(); // RS FIXME, do this via GRPGeomHelper once we switch to GRPECS
+    if (mTRDTransformer) { // need geometry loaded
+      mTRDTransformer->init();
+    }
+
+    if (!(mIniParFile.empty() || mIniParFile == "none")) {
+      mController->readParameters(mIniParFile, mUseIniParErrors);
+      mController->applyAlignmentFromMPSol();
+    }
+
+    // call this in the very end
+    if (mUsrConfMethod) {
+      int dummyPar = 0, ret = -1;
+      Controller* tmpPtr = mController.get();
+      const void* args[2] = {&tmpPtr, &dummyPar};
+      mUsrConfMethod->Execute(nullptr, args, 2, &ret);
+      if (ret != 0) {
+        LOG(fatal) << "Execution of user method config method " << mConfMacro << " failed with " << ret;
+      }
+    }
+    AlignConfig::Instance().printKeyValues(true);
+    o2::base::PropagatorD::Instance()->setTGeoFallBackAllowed(false);
+  }
+  if (GTrackID::includesDet(DetID::TRD, mMPsrc) && mTRDTransformer) {
+    pc.inputs().get("calvdexb"); // just to trigger the finaliseCCDB
+  }
+  if (mLoadTPCCalib) {
+
+    static float prevField = 1e-6;
+    float newField = o2::base::Propagator::Instance()->getNominalBz();
+    if (prevField != newField) {
+      prevField = newField;
+      if (mDetMask[DetID::TPC]) {
+        mTPCParam.reset(new o2::gpu::GPUParam);
+        mTPCParam->SetDefaults(o2::base::Propagator::Instance()->getNominalBz(), false);
+        mController->setTPCParam(mTPCParam.get());
+      }
+    }
+
+    mTPCVDriftHelper.extractCCDBInputs(pc);
+    mTPCCorrMapsLoader.extractCCDBInputs(pc);
+    bool updateMaps = false;
+    if (mTPCCorrMapsLoader.isUpdated()) {
+      mTPCCorrMapsLoader.acknowledgeUpdate();
+      updateMaps = true;
+    }
+    mController->setTPCCorrMaps(&mTPCCorrMapsLoader);
+    if (mTPCVDriftHelper.isUpdated()) {
+      LOGP(info, "Updating TPC fast transform map with new VDrift factor of {} wrt reference {} and DriftTimeOffset correction {} wrt {} from source {}",
+           mTPCVDriftHelper.getVDriftObject().corrFact, mTPCVDriftHelper.getVDriftObject().refVDrift,
+           mTPCVDriftHelper.getVDriftObject().timeOffsetCorr, mTPCVDriftHelper.getVDriftObject().refTimeOffset,
+           mTPCVDriftHelper.getSourceName());
+      mController->setTPCVDrift(mTPCVDriftHelper.getVDriftObject());
+      mTPCVDriftHelper.acknowledgeUpdate();
+      updateMaps = true;
+    }
+    if (updateMaps) {
+      mTPCCorrMapsLoader.updateVDrift(mTPCVDriftHelper.getVDriftObject().corrFact, mTPCVDriftHelper.getVDriftObject().refVDrift, mTPCVDriftHelper.getVDriftObject().getTimeOffset());
+    }
   }
 }
 
@@ -116,50 +305,120 @@ void BarrelAlignmentSpec::finaliseCCDB(o2::framework::ConcreteDataMatcher& match
       return;
     }
   }
+  if (matcher == ConcreteDataMatcher("TRD", "CALVDRIFTEXB", 0)) {
+    LOG(info) << "CalVdriftExB object has been updated";
+    mTRDTransformer->setCalVdriftExB((const o2::trd::CalVdriftExB*)obj);
+    return;
+  }
+  if (mTPCVDriftHelper.accountCCDBInputs(matcher, obj)) {
+
+    return;
+  }
+  if (mTPCCorrMapsLoader.accountCCDBInputs(matcher, obj)) {
+    return;
+  }
 }
 
 void BarrelAlignmentSpec::run(ProcessingContext& pc)
 {
   mTimer.Start(false);
-  updateTimeDependentParams(pc);
-  RecoContainer recoData;
-  recoData.collectData(pc, *mDataRequest.get());
-  mController->setRecoContainer(&recoData);
-  mController->process();
-
+  if (mPostProcessing) { // special mode, no data processing
+    updateTimeDependentParams(pc);
+    if (mController->getInstanceID() == 0) {
+      if (mPostProcessing & PostProc::CheckConstaints) {
+        mController->addAutoConstraints();
+        mController->checkConstraints();
+      }
+      if (mPostProcessing & PostProc::WriteResults) {
+        mController->writeCalibrationResults();
+      }
+    }
+    pc.services().get().readyToQuit(framework::QuitRequest::Me);
+  } else {
+    RecoContainer recoData;
+    recoData.collectData(pc, *mDataRequest.get());
+    updateTimeDependentParams(pc); // call after collectData !!!
+    mController->setRecoContainer(&recoData);
+    mController->setTimingInfo(pc.services().get());
+    if (mCosmic) {
+      mController->processCosmic();
+    } else {
+      mController->process();
+    }
+  }
   mTimer.Stop();
 }
 
 void BarrelAlignmentSpec::endOfStream(EndOfStreamContext& ec)
 {
-  //mBarrelAlign.end();
-  LOGF(info, "Barrel alignment data pereparation total timing: Cpu: %.3e Real: %.3e s in %d slots",
-       mTimer.CpuTime(), mTimer.RealTime(), mTimer.Counter() - 1);
+  auto inst = ec.services().get().inputTimesliceId;
+  if (!mPostProcessing) {
+    LOGP(info, "Barrel alignment data pereparation total timing: Cpu: {:.3e} Real: {:.3e}  s in {} slots, instance {}", mTimer.CpuTime(), mTimer.RealTime(), mTimer.Counter() - 1, inst);
+    mController->closeMPRecOutput();
+    mController->closeMilleOutput();
+    mController->closeResidOutput();
+  }
+  if (inst == 0) {
+    if (!mPostProcessing || (mPostProcessing & PostProc::GenPedeFiles)) {
+      LOG(info) << "Writing millepede control files";
+      if (!mPostProcessing) {
+        mController->terminate(); // finalize data stat
+      }
+      mController->addAutoConstraints();
+      mController->genPedeSteerFile();
+      mController->getStat().print();
+    } else if (mPostProcessing & PostProc::LabelPedeResults) {
+      mController->writeLabeledPedeResults();
+    }
+  }
+  mDBGOut.reset();
 }
 
-DataProcessorSpec getBarrelAlignmentSpec(GTrackID::mask_t src, DetID::mask_t dets)
+DataProcessorSpec getBarrelAlignmentSpec(GTrackID::mask_t srcMP, GTrackID::mask_t src, DetID::mask_t dets, DetID::mask_t skipDetClusters, bool enableCosmic, int postprocess, bool useMC, const o2::tpc::CorrectionMapsLoaderGloOpts& sclOpts)
 {
   std::vector outputs;
   auto dataRequest = std::make_shared();
-
-  dataRequest->requestTracks(src, false);
-  dataRequest->requestClusters(src, false);
-  dataRequest->requestPrimaryVertertices(false);
-
+  bool loadTPCCalib = false;
+  Options opts{
+    ConfigParamSpec{"apply-xor", o2::framework::VariantType::Bool, false, {"flip the 8-th bit of slope and position (for processing TRD CTFs from 2021 pilot beam)"}},
+    ConfigParamSpec{"allow-afterburner-tracks", VariantType::Bool, false, {"allow using ITS-TPC afterburner tracks"}},
+    ConfigParamSpec{"ignore-ccdb-alignment", VariantType::Bool, false, {"do not aplly CCDB alignment to ideal geometry"}},
+    ConfigParamSpec{"initial-params-file", VariantType::String, "", {"initial parameters file"}},
+    ConfigParamSpec{"config-macro", VariantType::String, "", {"configuration macro with signature (o2::align::Controller*, int) to execute from init"}},
+    ConfigParamSpec{"ignore-initial-params-errors", VariantType::Bool, false, {"ignore initial params (if any) errors for precondition"}},
+    ConfigParamSpec{"debug-output", VariantType::Int, 0, {"produce debugging output root files"}}};
+  if (!postprocess) {
+    dataRequest->requestTracks(src, useMC);
+    dataRequest->requestClusters(src, false, skipDetClusters);
+    dataRequest->requestPrimaryVertices(useMC);
+    if (GTrackID::includesDet(DetID::TRD, srcMP)) {
+      dataRequest->inputs.emplace_back("calvdexb", "TRD", "CALVDRIFTEXB", 0, Lifetime::Condition, ccdbParamSpec("TRD/Calib/CalVdriftExB"));
+    }
+    if (enableCosmic) {
+      dataRequest->requestCoscmicTracks(useMC);
+    }
+    if (src[DetID::TPC] && !skipDetClusters[DetID::TPC]) {
+      o2::tpc::VDriftHelper::requestCCDBInputs(dataRequest->inputs);
+      o2::tpc::CorrectionMapsLoader::requestCCDBInputs(dataRequest->inputs, opts, sclOpts);
+      loadTPCCalib = true;
+    }
+  }
   auto ccdbRequest = std::make_shared(true,                                 // orbitResetTime
                                                                 true,                                 // GRPECS=true
                                                                 false,                                // GRPLHCIF
-                                                                false,                                // GRPMagField
-                                                                false,                                // askMatLUT
+                                                                true,                                 // GRPMagField
+                                                                true,                                 // askMatLUT
                                                                 o2::base::GRPGeomRequest::Alignments, // geometry
-                                                                dataRequest->inputs);
-
+                                                                dataRequest->inputs,
+                                                                false,              // ask update once (except field)
+                                                                true,               // init PropagatorD
+                                                                "ITS,TPC,TRD,TOF"); // alignment objects to apply
   return DataProcessorSpec{
     "barrel-alignment",
     dataRequest->inputs,
     outputs,
-    AlgorithmSpec{adaptFromTask(dataRequest, ccdbRequest, dets)},
-    Options{}};
+    AlgorithmSpec{adaptFromTask(srcMP, dataRequest, ccdbRequest, sclOpts, dets, enableCosmic, postprocess, useMC, loadTPCCalib)},
+    opts};
 }
 
 } // namespace align
diff --git a/Detectors/Align/Workflow/src/barrel-alignment-workflow.cxx b/Detectors/Align/Workflow/src/barrel-alignment-workflow.cxx
index dbf57519825b3..8df479ba39260 100644
--- a/Detectors/Align/Workflow/src/barrel-alignment-workflow.cxx
+++ b/Detectors/Align/Workflow/src/barrel-alignment-workflow.cxx
@@ -19,6 +19,8 @@
 #include "TPCReaderWorkflow/TrackReaderSpec.h"
 #include "TPCReaderWorkflow/ClusterReaderSpec.h"
 #include "TPCWorkflow/ClusterSharingMapSpec.h"
+#include "TPCWorkflow/TPCScalerSpec.h"
+#include "TPCCalibration/CorrectionMapsLoader.h"
 #include "TOFWorkflowIO/ClusterReaderSpec.h"
 #include "TOFWorkflowIO/TOFMatchedReaderSpec.h"
 #include "TOFWorkflowIO/ClusterReaderSpec.h"
@@ -29,6 +31,7 @@
 #include "Algorithm/RangeTokenizer.h"
 #include "DetectorsRaw/HBFUtilsInitializer.h"
 #include "Framework/CallbacksPolicy.h"
+#include "GlobalTrackingWorkflowHelpers/NoInpDummyOutSpec.h"
 #include "GlobalTrackingWorkflowHelpers/InputHelper.h"
 
 using namespace o2::framework;
@@ -48,9 +51,15 @@ void customize(std::vector& workflowOptions)
   std::vector options{
     {"disable-root-input", o2::framework::VariantType::Bool, false, {"disable root-files input reader"}},
     {"disable-root-output", o2::framework::VariantType::Bool, false, {"disable root-files output writer"}},
+    {"enable-mc", o2::framework::VariantType::Bool, false, {"enable MC-info checks"}},
     {"track-sources", VariantType::String, std::string{GID::ALL}, {"comma-separated list of sources to use"}},
     {"detectors", VariantType::String, std::string{"ITS,TPC,TRD,TOF"}, {"comma-separated list of detectors"}},
+    {"enable-tpc-tracks", VariantType::Bool, false, {"allow reading TPC tracks"}},
+    {"enable-tpc-clusters", VariantType::Bool, false, {"allow reading TPC clusters (will trigger TPC tracks reading)"}},
+    {"enable-cosmic", VariantType::Bool, false, {"enable cosmic tracks)"}},
+    {"postprocessing", VariantType::Int, 0, {"postprocessing bits: 1 - extract alignment objects, 2 - check constraints, 4 - print mpParams/Constraints, 8 - relabel pede results"}},
     {"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings ..."}}};
+  o2::tpc::CorrectionMapsLoader::addGlobalOptions(options);
   o2::raw::HBFUtilsInitializer::addConfigOption(options);
   std::swap(workflowOptions, options);
 }
@@ -72,39 +81,91 @@ void customize(std::vector& policies)
 WorkflowSpec defineDataProcessing(ConfigContext const& configcontext)
 {
   WorkflowSpec specs;
-  GID::mask_t alowedSources = GID::getSourcesMask("ITS,MFT,TPC,TRD,ITS-TPC,TPC-TOF,TPC-TRD,ITS-TPC-TRD,TPC-TRD-TOF,ITS-TPC-TOF,ITS-TPC-TRD-TOF");
-  DetID::mask_t allowedDets = DetID::getMask("ITS,TPC,TRD,TOF,CPV,PHS,EMC,HMP");
+  GID::mask_t alowedSources = GID::getSourcesMask("ITS,TPC,TRD,ITS-TPC,TPC-TOF,TPC-TRD,ITS-TPC-TRD,TPC-TRD-TOF,ITS-TPC-TOF,ITS-TPC-TRD-TOF");
+  DetID::mask_t allowedDets = DetID::getMask("ITS,TPC,TRD,TOF");
 
   // Update the (declared) parameters if changed from the command line
   o2::conf::ConfigurableParam::updateFromString(configcontext.options().get("configKeyValues"));
-  // write the configuration used for the workflow
-  o2::conf::ConfigurableParam::writeINI("o2_barrel_alignment_configuration.ini");
+  int postprocess = configcontext.options().get("postprocessing");
 
   auto disableRootOut = configcontext.options().get("disable-root-output");
+  bool loadTPCClusters = configcontext.options().get("enable-tpc-clusters");
+  bool loadTPCTracks = configcontext.options().get("enable-tpc-tracks");
+  bool enableCosmic = configcontext.options().get("enable-cosmic");
+  bool useMC = configcontext.options().get("enable-mc");
 
   DetID::mask_t dets = allowedDets & DetID::getMask(configcontext.options().get("detectors"));
-
+  DetID::mask_t skipDetClusters; // optionally skip automatically loaded clusters
   GID::mask_t src = alowedSources & GID::getSourcesMask(configcontext.options().get("track-sources"));
-  if (GID::includesDet(DetID::TPC, src)) {
-    src |= GID::getSourceMask(GID::TPC);
-    LOG(info) << "adding TPC request";
+  GID::mask_t srcCl{}, srcMP = src; // we may need to load more track types than requested to satisfy dependencies, but only those will be fed to millipede
+
+  if (dets[DetID::TPC]) {
+    loadTPCClusters = loadTPCTracks = true;
   }
-  if (GID::includesDet(DetID::TPC, src)) {
-    src |= GID::getSourceMask(GID::TPC);
-    LOG(info) << "adding TPC request";
+  auto sclOpt = o2::tpc::CorrectionMapsLoader::parseGlobalOptions(configcontext.options());
+  if (!postprocess) { // this part is needed only if the data should be read
+    if (GID::includesDet(DetID::ITS, src)) {
+      src |= GID::getSourceMask(GID::ITS);
+      srcCl |= GID::getSourceMask(GID::ITS);
+      LOG(info) << "adding ITS request";
+    }
+
+    if (GID::includesDet(DetID::TPC, src)) {
+      if (loadTPCTracks || loadTPCClusters) {
+        src |= GID::getSourceMask(GID::TPC);
+        LOG(info) << "adding TPC request";
+      }
+      if (loadTPCClusters) {
+        srcCl |= GID::getSourceMask(GID::TPC);
+      } else {
+        skipDetClusters |= DetID::getMask(DetID::TPC);
+        LOG(info) << "Skipping TPC clusters";
+      }
+    }
+    if (GID::includesDet(DetID::TRD, src)) {
+      src |= GID::getSourceMask(GID::TRD);
+      srcCl |= GID::getSourceMask(GID::TRD);
+      if (GID::includesDet(DetID::ITS, src)) {
+        src |= GID::getSourceMask(GID::ITSTPC);
+      }
+      LOG(info) << "adding TRD request";
+    }
+    if (GID::includesDet(DetID::TOF, src)) {
+      src |= GID::getSourceMask(GID::TOF);
+      srcCl |= GID::getSourceMask(GID::TOF);
+      if (GID::includesDet(DetID::ITS, src)) {
+        src |= GID::getSourceMask(GID::ITSTPC);
+      }
+      if (GID::includesDet(DetID::TRD, src)) {
+        src |= GID::getSourceMask(GID::ITSTPCTRD);
+      }
+      LOG(info) << "adding TOF request";
+    }
+    if (sclOpt.requestCTPLumi) {
+      src = src | GID::getSourcesMask("CTP");
+    }
+    // write the configuration used for the workflow
+    o2::conf::ConfigurableParam::writeINI("o2_barrel_alignment_configuration.ini");
   }
 
-  GID::mask_t dummy;
-  specs.emplace_back(o2::align::getBarrelAlignmentSpec(src, dets));
-  // RS FIXME: check which clusters are really needed
-  o2::globaltracking::InputHelper::addInputSpecs(configcontext, specs, src, src, src, false, dummy); // clusters MC is not needed
-  o2::globaltracking::InputHelper::addInputSpecsPVertex(configcontext, specs, false);
+  if (sclOpt.needTPCScalersWorkflow() && !configcontext.options().get("disable-root-input")) {
+    specs.emplace_back(o2::tpc::getTPCScalerSpec(sclOpt.lumiType == 2, sclOpt.enableMShapeCorrection));
+  }
 
-  if (!disableRootOut) {
+  specs.emplace_back(o2::align::getBarrelAlignmentSpec(srcMP, src, dets, skipDetClusters, enableCosmic, postprocess, useMC, sclOpt));
+  // RS FIXME: check which clusters are really needed
+  if (!postprocess) {
+    GID::mask_t dummy;
+    o2::globaltracking::InputHelper::addInputSpecs(configcontext, specs, srcCl, src, src, useMC, dummy); // clusters MC is not needed
+    o2::globaltracking::InputHelper::addInputSpecsPVertex(configcontext, specs, useMC);
+    if (enableCosmic) {
+      o2::globaltracking::InputHelper::addInputSpecsCosmics(configcontext, specs, useMC);
+    }
+  } else { // add dummy driver
+    specs.emplace_back(o2::globaltracking::getNoInpDummyOutSpec(0));
   }
 
   // configure dpl timer to inject correct firstTForbit: start from the 1st orbit of TF containing 1st sampled orbit
   o2::raw::HBFUtilsInitializer hbfIni(configcontext, specs);
-
   return std::move(specs);
 }
diff --git a/Detectors/Align/include/Align/AlgPntDbg.h b/Detectors/Align/include/Align/AlgPntDbg.h
new file mode 100644
index 0000000000000..12e8205f6b614
--- /dev/null
+++ b/Detectors/Align/include/Align/AlgPntDbg.h
@@ -0,0 +1,91 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// @file   AlignmentPoint.h
+/// @author ruben.shahoyan@cern.ch, michael.lettrich@cern.ch
+/// @since  2021-02-01
+/// @brief  Meausered point in the sensor.
+
+/**
+ * Compact alignment point info for debugging
+ */
+
+#ifndef ALGPNTDBG_H
+#define ALGPNTDBG_H
+
+#include "Align/AlignmentPoint.h"
+
+namespace o2
+{
+namespace align
+{
+
+struct AlgPntDbg {
+ public:
+  using DetID = o2::detectors::DetID;
+  //
+  enum {
+    UpperLeg = 0
+  };
+  //
+  AlgPntDbg() = default;
+  AlgPntDbg(const AlgPntDbg&) = default;
+  ~AlgPntDbg() = default;
+  AlgPntDbg& operator=(const AlgPntDbg& other) = default;
+  AlgPntDbg(const AlignmentPoint* point) : mDetID(point->getDetID()), mSID(point->getSID()), mAlpha(point->getAlphaSens()), mX(point->getXTracking()), mY(point->getYTracking()), mZ(point->getZTracking()), mErrYY(point->getYZErrTracking()[0]), mErrZZ(point->getYZErrTracking()[2]), mErrYZ(point->getYZErrTracking()[1])
+  {
+    mSinAlp = std::sin(mAlpha);
+    mCosAlp = std::cos(mAlpha);
+    mSnp = point->getTrParamWSA()[2]; // track Snp at the sensor
+    if (point->isInvDir()) {
+      setUpperLeg();
+    }
+  }
+
+  float getR() const { return std::sqrt(mX * mX + mY * mY); }
+  float getYTrack() const { return mY + mYRes; }
+  float getZTrack() const { return mZ + mZRes; }
+  float getXTrack() const { return mX; }
+  float getXLab() const { return mX * mCosAlp - mY * mSinAlp; }
+  float getYLab() const { return mX * mSinAlp + mY * mCosAlp; }
+  float getZLap() const { return mZ; }
+  float getXTrackLab() const { return mX * mCosAlp - getYTrack() * mSinAlp; }
+  float getYTrackLab() const { return mX * mSinAlp + getYTrack() * mCosAlp; }
+  float getZTrackLab() const { return getZTrack(); }
+  float getPhi() const { return std::atan2(getYLab(), getXLab()); }
+  void setFlag(int i) { mFlags |= 0x1 << i; }
+  bool getFlag(int i) const { return (mFlags & (0x1 << i)) != 0; }
+
+  void setUpperLeg() { setFlag(int(UpperLeg)); }
+  bool isUpperLeg() const { return getFlag(int(UpperLeg)); }
+
+  int mDetID{};        // DetectorID
+  int16_t mSID = -1;   // sensor ID in the detector
+  uint16_t mFlags = 0; // flags
+  float mAlpha = 0.f;  // Alpha of tracking frame
+  float mSinAlp = 0.f;
+  float mCosAlp = 0.f;
+  float mX = 0.f;    // tracking X
+  float mY = 0.f;    // tracking Y
+  float mZ = 0.f;    // Z
+  float mYRes = 0.f; // tracking Y residual (track - point)
+  float mZRes = 0.f; // Z residual
+  float mErrYY = 0.f;
+  float mErrZZ = 0.f;
+  float mErrYZ = 0.f;
+  float mSnp = 0.f;
+
+  ClassDefNV(AlgPntDbg, 1);
+};
+
+} // namespace align
+} // namespace o2
+#endif
diff --git a/Detectors/Align/include/Align/AlgTrcDbg.h b/Detectors/Align/include/Align/AlgTrcDbg.h
new file mode 100644
index 0000000000000..cb179dd8b9b2f
--- /dev/null
+++ b/Detectors/Align/include/Align/AlgTrcDbg.h
@@ -0,0 +1,90 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// @file   AlgTrcDbg.h
+/// @author ruben.shahoyan@cern.ch
+
+#ifndef ALGTRCDBG_H
+#define ALGTRCDBG_H
+
+#include "Align/AlignmentTrack.h"
+#include "Align/AlgPntDbg.h"
+#include "ReconstructionDataFormats/GlobalTrackID.h"
+#include "ReconstructionDataFormats/Track.h"
+
+namespace o2
+{
+namespace align
+{
+
+struct AlgTrcDbg : public o2::track::TrackParCov {
+  AlgTrcDbg(const AlignmentTrack* trc) { setTrackParam(trc); }
+  AlgTrcDbg() = default;
+  ~AlgTrcDbg() = default;
+  AlgTrcDbg(const AlgTrcDbg&) = default;
+  AlgTrcDbg& operator=(const AlgTrcDbg&) = default;
+
+  bool setTrackParam(const AlignmentTrack* trc)
+  {
+    if (!trc) {
+      return false;
+    }
+    setX(trc->getX());
+    setY(trc->getAlpha());
+    for (int i = 0; i < 5; i++) {
+      setParam(trc->getParam(i), i);
+    }
+    for (int i = 0; i < 15; i++) {
+      setCov(trc->getCov()[i], i);
+    }
+    mPoints.clear();
+    for (int i = 0; i < trc->getNPoints(); i++) {
+      const auto* tpoint = trc->getPoint(i);
+      if (tpoint->containsMeasurement()) {
+        auto& pnt = mPoints.emplace_back(tpoint);
+        pnt.mYRes = trc->getResidual(0, i);
+        pnt.mZRes = trc->getResidual(1, i);
+      }
+    }
+    setX(trc->getX());
+    setY(trc->getAlpha());
+    for (int i = 0; i < 5; i++) {
+      setParam(trc->getParam(i), i);
+    }
+    for (int i = 0; i < 15; i++) {
+      setCov(trc->getCov()[i], i);
+    }
+    for (int i = 0; i < trc->getNPoints(); i++) {
+      const auto* tpoint = trc->getPoint(i);
+      if (tpoint->containsMeasurement()) {
+        auto& pnt = mPoints.emplace_back(tpoint);
+        pnt.mYRes = trc->getResidual(0, i);
+        pnt.mZRes = trc->getResidual(1, i);
+      }
+    }
+    mGID.clear();
+    mGIDCosmUp.clear();
+    return true;
+  }
+
+  auto getNPoints() const { return mPoints.size(); }
+  bool isCosmic() const { return mGIDCosmUp.isSourceSet(); }
+
+  std::vector mPoints;
+  o2::dataformats::GlobalTrackID mGID{};
+  o2::dataformats::GlobalTrackID mGIDCosmUp{}; // GID of upper leg in case of cosmic
+  //
+  ClassDefNV(AlgTrcDbg, 1);
+};
+
+} // namespace align
+} // namespace o2
+#endif
diff --git a/Detectors/Align/include/Align/AlignConfig.h b/Detectors/Align/include/Align/AlignConfig.h
index 4b4a37b3e81ae..e72d436a14e3b 100644
--- a/Detectors/Align/include/Align/AlignConfig.h
+++ b/Detectors/Align/include/Align/AlignConfig.h
@@ -29,25 +29,78 @@ struct AlignConfig : public o2::conf::ConfigurableParamHelper {
                    Cosmic,
                    NTrackTypes };
 
-  float maxStep = 2.; // max step for propagation
-  float maxSnp = 0.9; // max snp for propagation
-  o2::base::PropagatorD::MatCorrType matCorType = o2::base::PropagatorD::MatCorrType::USEMatCorrTGeo;
+  float maxStep = 3.;  // max step for propagation
+  float maxSnp = 0.95; // max snp for propagation
+  int matCorType = (int)o2::base::PropagatorD::MatCorrType::USEMatCorrLUT;
   float q2PtMin[NTrackTypes] = {0.01, 0.01};
   float q2PtMax[NTrackTypes] = {10., 10.};
   float tglMax[NTrackTypes] = {3., 10.};
+  float defPTB0Coll = 0.6;
+  float defPTB0Cosm = 3.0;
   int minPoints[NTrackTypes] = {4, 10};
   int minDetAcc[NTrackTypes] = {1, 1};
 
-  float minX2X0Pt2Account = 0.5e-3;
+  float minScatteringAngleToAccount = 0.0003;
+
+  int verbose = 0;
 
   int vtxMinCont = 2;     // require min number of contributors in Vtx
   int vtxMaxCont = 99999; // require max number of contributors in Vtx
   int vtxMinContVC = 20;  // min number of contributors to use as constraint
 
   int minPointTotal = 4; // total min number of alignment point to account track
+  int minDetectors = 1;  // min number of detectors per track
+  int minITSClusters = 4;  // min ITS clusters to accept the track
+  int minTRDTracklets = 3; // min TRD tracklets to accept the track
+  int minTPCClusters = 10; // discard tracks with less clusters
+  int minTOFClusters = 1;  // min TOF clusters to accept track
+  int maxTPCRowsCombined = 1;        // allow combining clusters on so many rows to a single cluster
+  int discardEdgePadrows = 3;        // discard padrow if its distance to stack edge padrow < this
+  float discardSectorEdgeDepth = 2.5; // discard clusters too close to the sector edge
+  float ITSOverlapMargin = 0.15;     // consider for overlaps only clusters within this marging from the chip edge (in cm)
+  float ITSOverlapMaxChi2 = 16;      // max chi2 between track and overlapping cluster
+  int ITSOverlapEdgeRows = 1;        // require clusters to not have pixels closer than this distance from the edge
+  float ITSOverlapMaxDZ = 0.3;       // max difference in Z for clusters on overlapping ITS chips to consider as candidate for a double hit
+
+  int minPointTotalCosm = 4;      // total min number of alignment point to account cosmic track
+  int minDetectorsCosm = 1;       // min number of detectors per cosmic track
+  int minITSClustersCosm = 0;     // min ITS clusters to accept the cosmic track
+  int minITSClustersCosmLeg = 2;  // min ITS clusters per leg to accept the cosmic track
+  int minTRDTrackletsCosm = 0;    // min TRD tracklets to accept the cosmic track
+  int minTRDTrackletsCosmLeg = 2; // min TRD tracklets per leg to accept the cosmic track
+  int minTPCClustersCosm = 0;     // discard cosmic tracks with less clusters
+  int minTPCClustersCosmLeg = 10; // discard cosmic tracks with less clusters per leg
+  int minTOFClustersCosm = 0;     // min TOF clusters to accept track
+  int minTOFClustersCosmLeg = 1;  // min TOF clusters per leg to accept track
+
+  int minTPCPadRow = 6;   // min TPC pad-row to account
+  int maxTPCPadRow = 146; // max TPC pad-row to account
+
+  float cosmMaxDSnp = 0.025; // reject cosmic tracks with larger than this snp difference
+  float cosmMaxDTgl = 0.1;   // reject cosmic tracks with larger than this tgl difference
+
+  float maxDCAforVC[2] = {-1, -1}; // DCA cut in R,Z to allow track be subjected to vertex constraint
+  float maxChi2forVC = -1;         // track-vertex chi2 cut to allow the track be subjected to vertex constraint
+  float alignParamZero = 1e-13;    // assign 0 to final alignment parameter if its abs val is below this threshold
+  float controlFraction = -1.;     // fraction for which control output is requested, if negative - only 1st instance of device will write them
+  float MPRecOutFraction = -1.;    // compact Millepede2Record fraction, if negative - only 1st instance of device will write them
+
+  bool useLinRef = true;      // use initial track for lienarization reference point
+  bool MilleOut = true;       // Mille output
+  bool KalmanResid = true;    // Kalman residuals
+  bool MilleOutBin = true;    // text vs binary output for mille data
+  bool GZipMilleOut = false;  // compress binary records
 
-  float maxDCAforVC[2]; // DCA cut in R,Z to allow track be subjected to vertex constraint
-  float maxChi2forVC;   // track-vertex chi2 cut to allow the track be subjected to vertex constraint
+  std::string mpDatFileName{"mpData"};            //  file name for records mille data output
+  std::string mpParFileName{"mpParams.txt"};      //  file name for MP params
+  std::string mpConFileName{"mpConstraints.txt"}; //  file name for MP constraints
+  std::string mpSteerFileName{"mpSteer.txt"};     //  file name for MP steering
+  std::string residFileName{"mpContolRes"};       //  file name for optional control residuals
+  std::string mpLabFileName{"mpResultsLabeled.txt"}; //  file name for relabeled MP params
+  //
+  std::string outCDBPath{};        // output OCDB path
+  std::string outCDBComment{};     // optional comment to add to output cdb objects
+  std::string outCDBResponsible{}; // optional responsible for output metadata
 
   O2ParamDef(AlignConfig, "alignConf");
 };
diff --git a/Detectors/Align/include/Align/AlignableDetector.h b/Detectors/Align/include/Align/AlignableDetector.h
index e6ada27017bc9..97963c006aa46 100644
--- a/Detectors/Align/include/Align/AlignableDetector.h
+++ b/Detectors/Align/include/Align/AlignableDetector.h
@@ -24,7 +24,6 @@
 #include "Align/DOFSet.h"
 #include "Align/utils.h"
 #include "Align/AlignmentTrack.h"
-#include "Align/DOFStatistics.h"
 #include "Align/AlignmentPoint.h"
 #include "Align/AlignableSensor.h"
 #include "Align/AlignableVolume.h"
@@ -110,14 +109,13 @@ class AlignableDetector : public DOFSet
   virtual int assignDOFs();
   virtual void initDOFs();
   virtual void terminate();
-  void fillDOFStat(DOFStatistics& dofst) const;
   virtual void addVolume(AlignableVolume* vol);
   virtual void defineVolumes();
   virtual void defineMatrices();
   void Print(const Option_t* opt = "") const override;
 
   virtual void reset();
-  virtual int processPoints(GIndex gid, bool inv = false);
+  virtual int processPoints(GIndex gid, int npntCut = 0, bool inv = false);
   virtual bool prepareDetectorData() { return true; }
 
   virtual void updatePointByTrackInfo(AlignmentPoint* pnt, const trackParam_t* t) const;
@@ -135,9 +133,9 @@ class AlignableDetector : public DOFSet
   void setInitDOFsDone() { SetBit(kInitDOFsDone); }
   bool getInitDOFsDone() const { return TestBit(kInitDOFsDone); }
   void fixNonSensors();
-  void setFreeDOFPattern(uint32_t pat = 0xffffffff, int lev = -1, const char* match = nullptr);
-  void setDOFCondition(int dof, float condErr, int lev = -1, const char* match = nullptr);
-  int selectVolumes(TObjArray* arr, int lev = -1, const char* match = nullptr);
+  void setFreeDOFPattern(uint32_t pat = 0xffffffff, int lev = -1, const std::string& regexStr = "");
+  void setDOFCondition(int dof, float condErr, int lev = -1, const std::string& regexStr = "");
+  int selectVolumes(std::vector cont, int lev = -1, const std::string& regexStr = "");
   //
   void setDisabled(int tp, bool v)
   {
@@ -156,13 +154,6 @@ class AlignableDetector : public DOFSet
   bool IsDisabledColl() const { return isDisabled(utils::Coll); }
   bool IsDisabledCosm() const { return isDisabled(utils::Cosm); }
   //
-  void setTrackFlagSel(int tp, uint64_t f) { mTrackFlagSel[tp] = f; }
-  void setTrackFlagSelColl(uint64_t f) { setTrackFlagSel(utils::Coll, f); }
-  void setTrackFlagSelCosm(uint64_t f) { setTrackFlagSel(utils::Cosm, f); }
-  uint64_t getTrackFlagSel(int tp) const { return mTrackFlagSel[tp]; }
-  uint64_t getTrackFlagSelColl() const { return getTrackFlagSel(utils::Coll); }
-  uint64_t getTrackFlagSelCosm() const { return getTrackFlagSel(utils::Cosm); }
-  //
   void setNPointsSel(int tp, int n) { mNPointsSel[tp] = n; }
   void setNPointsSelColl(int n) { setNPointsSel(utils::Coll, n); }
   void setNPointsSelCosm(int n) { setNPointsSel(utils::Cosm, n); }
@@ -182,6 +173,7 @@ class AlignableDetector : public DOFSet
   void constrainOrphans(const double* sigma, const char* match = nullptr);
 
   virtual void writePedeInfo(FILE* parOut, const Option_t* opt = "") const;
+  virtual void writeLabeledPedeResults(FILE* parOut) const;
   virtual void writeCalibrationResults() const;
   virtual void writeAlignmentResults() const;
   //
@@ -209,7 +201,6 @@ class AlignableDetector : public DOFSet
   // Track selection
   bool mDisabled[utils::NTrackTypes] = {};         // detector disabled/enabled in the track
   bool mObligatory[utils::NTrackTypes] = {};       // detector must be present in the track
-  uint64_t mTrackFlagSel[utils::NTrackTypes] = {}; // flag for track selection
   int mNPointsSel[utils::NTrackTypes] = {};        // min number of points to require
   //
   int mUseErrorParam = 0;   // signal that points need to be updated using track info, 0 - no
@@ -218,19 +209,11 @@ class AlignableDetector : public DOFSet
   TObjArray mVolumes;  // all volumes of the detector
   //
   // this is transient info
-  int mNPoints = 0;         //! number of points from this detector
-  int mFirstPoint = 0;      //! entry of the 1st point
+  int mNPoints = 0; //! number of points from this detector
   //
   ClassDefOverride(AlignableDetector, 1); // base class for detector global alignment
 };
 
-//FIXME(milettri): needs AliESDtrack
-////_____________________________________________________
-//inline bool AlignableDetector::CheckFlags(const AliESDtrack* trc, int trtype) const
-//{
-//  // check if flags are ok
-//  return (trc->GetStatus() & mTrackFlagSel[trtype]) == mTrackFlagSel[trtype];
-//}
 } // namespace align
 } // namespace o2
 #endif
diff --git a/Detectors/Align/include/Align/AlignableDetectorITS.h b/Detectors/Align/include/Align/AlignableDetectorITS.h
index 2cd29ac8fe679..5f1816d8368b8 100644
--- a/Detectors/Align/include/Align/AlignableDetectorITS.h
+++ b/Detectors/Align/include/Align/AlignableDetectorITS.h
@@ -21,6 +21,7 @@
 #include "Align/utils.h"
 #include "ReconstructionDataFormats/TrackParametrizationWithError.h"
 #include "ReconstructionDataFormats/BaseCluster.h"
+#include "ITSMFTReconstruction/ChipMappingITS.h"
 
 namespace o2
 {
@@ -39,14 +40,12 @@ class AlignableDetectorITS : public AlignableDetector
  public:
   //
   using ClusterD = o2::BaseCluster;
-  enum ITSSel_t { kSPDNoSel,
-                  kSPDBoth,
-                  kSPDAny,
-                  kSPD0,
-                  kSPD1,
-                  kNSPDSelTypes };
-  //
-  AlignableDetectorITS() = default;
+  using OVL = o2::itsmft::ChipMappingITS::Overlaps;
+  enum EdgeFlags : int8_t { NONE = -1,
+                            LowRow = OVL::LowRow,
+                            HighRow = OVL::HighRow,
+                            Biased = 2 };
+  AlignableDetectorITS() = default; // RS FIXME do we need default c-tor?
   AlignableDetectorITS(Controller* ctr);
   ~AlignableDetectorITS() override = default;
   //
@@ -56,7 +55,7 @@ class AlignableDetectorITS : public AlignableDetector
   // RSTODO
   //  bool AcceptTrack(const AliESDtrack* trc, int trtype) const;
 
-  int processPoints(GIndex gid, bool inv) final;
+  int processPoints(GIndex gid, int npntCut, bool inv) final;
   bool prepareDetectorData() final;
 
   void SetAddErrorLr(int ilr, double sigY, double sigZ);
@@ -64,20 +63,11 @@ class AlignableDetectorITS : public AlignableDetector
   //
   void updatePointByTrackInfo(AlignmentPoint* pnt, const trackParam_t* t) const override;
   void setUseErrorParam(int v = 0) override;
-  void SetITSSelPattern(int trtype, ITSSel_t sel) { fITSPatt[trtype] = sel; }
-  void SetITSSelPatternColl(ITSSel_t sel = kSPDAny) { SetITSSelPattern(utils::Coll, sel); }
-  void SetITSSelPatternCosm(ITSSel_t sel = kSPDNoSel) { SetITSSelPattern(utils::Cosm, sel); }
-
-  int GetITSSelPattern(int tp) const { return fITSPatt[tp]; }
-  int GetITSSelPatternColl() const { return fITSPatt[utils::Coll]; }
-  int GetITSSelPatternCosm() const { return fITSPatt[utils::Cosm]; }
   //
   void setITSDictionary(const o2::itsmft::TopologyDictionary* d) { mITSDict = d; }
   //
   void Print(const Option_t* opt = "") const override;
   //
-  static const char* GetITSPattName(int sel) { return sel < kNSPDSelTypes ? fgkHitsSel[sel] : nullptr; }
-  //
  protected:
   //
   // -------- dummies --------
@@ -87,11 +77,10 @@ class AlignableDetectorITS : public AlignableDetector
  protected:
   //
   std::vector mITSClustersArray;
+  std::vector mOverlapCandidateID; // pool of indices for potentially overlapping clusters
+  std::vector mOverlapClusRef;     // 1st entry in mOverlapCandidateID for the overlapping cluster indices of each cluster
+  std::vector mOverlaps;
   const o2::itsmft::TopologyDictionary* mITSDict{nullptr}; // cluster patterns dictionary
-
-  int fITSPatt[utils::NTrackTypes]; // ITS hits selection pattern for coll/cosm tracks
-  //
-  static const char* fgkHitsSel[kNSPDSelTypes]; // ITS selection names
   //
   ClassDefOverride(AlignableDetectorITS, 1);
 };
diff --git a/Detectors/Align/include/Align/AlignableDetectorTOF.h b/Detectors/Align/include/Align/AlignableDetectorTOF.h
index 4897409260dfa..b5434bcf6ad3c 100644
--- a/Detectors/Align/include/Align/AlignableDetectorTOF.h
+++ b/Detectors/Align/include/Align/AlignableDetectorTOF.h
@@ -24,15 +24,16 @@ namespace o2
 namespace align
 {
 
-class AlignableDetectorTOF : public AlignableDetector
+class AlignableDetectorTOF final : public AlignableDetector
 {
  public:
-  AlignableDetectorTOF(const char* title = "");
-  virtual ~AlignableDetectorTOF();
+  AlignableDetectorTOF() = default;
+  AlignableDetectorTOF(Controller* ctr);
+  ~AlignableDetectorTOF() final = default;
   //
-  virtual void defineVolumes();
+  void defineVolumes() final;
   //
-  bool AcceptTrack(const AliESDtrack* trc, int trtype) const;
+  int processPoints(GIndex gid, int npntCut, bool inv) final;
   //
  protected:
   //
diff --git a/Detectors/Align/include/Align/AlignableDetectorTPC.h b/Detectors/Align/include/Align/AlignableDetectorTPC.h
index d513f983d44e4..af85e18bf2de1 100644
--- a/Detectors/Align/include/Align/AlignableDetectorTPC.h
+++ b/Detectors/Align/include/Align/AlignableDetectorTPC.h
@@ -10,8 +10,7 @@
 // or submit itself to any jurisdiction.
 
 /// @file   AlignableDetectorTPC.h
-/// @author ruben.shahoyan@cern.ch, michael.lettrich@cern.ch
-/// @since  2021-02-01
+/// @author ruben.shahoyan@cern.ch
 /// @brief  TPC detector wrapper
 
 #ifndef ALIGNABLEDETECTORTPC_H
@@ -24,23 +23,47 @@ namespace o2
 namespace align
 {
 
-class AlignableDetectorTPC : public AlignableDetector
+class AlignableDetectorTPC final : public AlignableDetector
 {
  public:
-  AlignableDetectorTPC(const char* title = "");
-  virtual ~AlignableDetectorTPC();
   //
-  virtual void defineVolumes();
-  //
-  bool AcceptTrack(const AliESDtrack* trc, int trtype) const;
+  AlignableDetectorTPC() = default;
+  AlignableDetectorTPC(Controller* ctr);
+  ~AlignableDetectorTPC() final = default;
+  void defineVolumes() final;
+  void Print(const Option_t* opt = "") const final;
   //
+  int processPoints(GIndex gid, int npntCut, bool inv) final;
+
+  void setTrackTimeStamp(float t) { mTrackTimeStamp = t; }
+  float getTrackTimeStamp() const { return mTrackTimeStamp; }
+
+  int getStack(int padrow) const
+  {
+    for (int i = 0; i < 4; i++) {
+      if (padrow <= mStackMinMaxRow[i].second) {
+        return i;
+      }
+    }
+    return -1;
+  }
+
+  int getDistanceToStackEdge(int padrow) const
+  {
+    // distance to the stack min or max padrow
+    auto st = getStack(padrow);
+    if (st < 0) {
+      return -999;
+    }
+    return std::min(padrow - mStackMinMaxRow[st].first, mStackMinMaxRow[st].second - padrow);
+  }
+
  protected:
   //
-  // -------- dummies --------
-  AlignableDetectorTPC(const AlignableDetectorTPC&);
-  AlignableDetectorTPC& operator=(const AlignableDetectorTPC&);
-  //
- protected:
+  float mTrackTimeStamp = 0.f; // use track timestamp in \mus
+  static constexpr int NSTACKS = 4;
+  const std::array, NSTACKS> mStackMinMaxRow = {std::pair{0, 62}, std::pair{63, 96}, std::pair{97, 126}, std::pair{127, 151}};
+
   ClassDef(AlignableDetectorTPC, 1);
 };
 } // namespace align
diff --git a/Detectors/Align/include/Align/AlignableDetectorTRD.h b/Detectors/Align/include/Align/AlignableDetectorTRD.h
index 9b193acb7d26e..4e7577b11055c 100644
--- a/Detectors/Align/include/Align/AlignableDetectorTRD.h
+++ b/Detectors/Align/include/Align/AlignableDetectorTRD.h
@@ -18,63 +18,58 @@
 #define ALIGNABLEDETECTORTRD_H
 
 #include "Align/AlignableDetector.h"
+#include "GPUTRDRecoParam.h"
 
 namespace o2
 {
 namespace align
 {
 
-class AlignableDetectorTRD : public AlignableDetector
+class AlignableDetectorTRD final : public AlignableDetector
 {
  public:
   //
-  enum { kCalibNRCCorrDzDtgl, // correction parameter for NonRC tracklets
-         kCalibDVT,           // global correction to Vdrift*t
-         kNCalibParams };     // calibration parameters
+  enum { CalibNRCCorrDzDtgl, // correction parameter for NonRC tracklets
+         CalibDVT,           // global correction to Vdrift*t
+         NCalibParams };     // calibration parameters
   //
-  AlignableDetectorTRD(const char* title = "");
-  virtual ~AlignableDetectorTRD();
+  AlignableDetectorTRD() = default; // RS FIXME do we need default c-tor?
+  AlignableDetectorTRD(Controller* ctr);
+  ~AlignableDetectorTRD() final = default;
+  void defineVolumes() final;
+  void Print(const Option_t* opt = "") const final;
+  const char* getCalibDOFName(int i) const final;
   //
-  virtual void defineVolumes();
-  virtual void Print(const Option_t* opt = "") const;
+  void writePedeInfo(FILE* parOut, const Option_t* opt = "") const final;
+  void writeLabeledPedeResults(FILE* parOut) const final;
   //
-  bool AcceptTrack(const AliESDtrack* trc, int trtype) const;
+  void setNonRCCorrDzDtgl(double v = 0.) { mNonRCCorrDzDtgl = v; }
+  double getNonRCCorrDzDtgl() const { return mNonRCCorrDzDtgl; }
+  double getNonRCCorrDzDtglWithCal() const { return getNonRCCorrDzDtgl() + getParVal(CalibNRCCorrDzDtgl); }
   //
-  virtual const char* getCalibDOFName(int i) const;
+  void setCorrDVT(double v = 0) { mCorrDVT = 0; }
+  double getCorrDVT() const { return mCorrDVT; }
+  double getCorrDVTWithCal() const { return getCorrDVT() + getParVal(CalibDVT); }
   //
-  virtual void writePedeInfo(FILE* parOut, const Option_t* opt = "") const;
+  double getCalibDOFVal(int id) const final;
+  double getCalibDOFValWithCal(int id) const final;
   //
-  void SetNonRCCorrDzDtgl(double v = 0) { fNonRCCorrDzDtgl = v; }
-  double GetNonRCCorrDzDtgl() const { return fNonRCCorrDzDtgl; }
-  double GetNonRCCorrDzDtglWithCal() const { return GetNonRCCorrDzDtgl() + getParVal(kCalibNRCCorrDzDtgl); }
-  //
-  void SetCorrDVT(double v = 0) { fCorrDVT = 0; }
-  double GetCorrDVT() const { return fCorrDVT; }
-  double GetCorrDVTWithCal() const { return GetCorrDVT() + getParVal(kCalibDVT); }
-  //
-  virtual double getCalibDOFVal(int id) const;
-  virtual double getCalibDOFValWithCal(int id) const;
-  //
-  const double* GetExtraErrRC() const { return fExtraErrRC; }
-  void SetExtraErrRC(double y = 0.2, double z = 1.0)
+  const double* getExtraErrRC() const { return mExtraErrRC; }
+  void setExtraErrRC(double y = 0.2, double z = 1.0)
   {
-    fExtraErrRC[0] = y;
-    fExtraErrRC[1] = z;
+    mExtraErrRC[0] = y;
+    mExtraErrRC[1] = z;
   }
-  //
- protected:
-  //
-  // -------- dummies --------
-  AlignableDetectorTRD(const AlignableDetectorTRD&);
-  AlignableDetectorTRD& operator=(const AlignableDetectorTRD&);
-  //
+
+  int processPoints(GIndex gid, int npntCut, bool inv) final;
+
  protected:
+  o2::gpu::GPUTRDRecoParam mRecoParam; // parameters required for TRD reconstruction
+  double mNonRCCorrDzDtgl = 0.;     // correction in Z for non-crossing tracklets
+  double mCorrDVT = 0.;             // correction to Vdrift*t
+  double mExtraErrRC[2] = {0., 0.}; // extra errors for RC tracklets
   //
-  double fNonRCCorrDzDtgl; // correction in Z for non-crossing tracklets
-  double fCorrDVT;         // correction to Vdrift*t
-  double fExtraErrRC[2];   // extra errors for RC tracklets
-  //
-  static const char* fgkCalibDOFName[kNCalibParams];
+  static const char* CalibDOFName[NCalibParams];
   //
   ClassDef(AlignableDetectorTRD, 1);
 };
diff --git a/Detectors/Align/include/Align/AlignableSensor.h b/Detectors/Align/include/Align/AlignableSensor.h
index fd1a64b392bf7..674f69b7c107c 100644
--- a/Detectors/Align/include/Align/AlignableSensor.h
+++ b/Detectors/Align/include/Align/AlignableSensor.h
@@ -21,7 +21,6 @@
 #include 
 #include "ReconstructionDataFormats/BaseCluster.h"
 #include "Align/AlignableVolume.h"
-#include "Align/DOFStatistics.h"
 #include "Align/utils.h"
 #include 
 
@@ -80,7 +79,7 @@ class AlignableSensor : public AlignableVolume
   virtual void updatePointByTrackInfo(AlignmentPoint* pnt, const trackParam_t* t) const;
   void updateL2GRecoMatrices(const std::vector& algArr, const TGeoHMatrix* cumulDelta) override;
   //
-  int finalizeStat(DOFStatistics& h) override;
+  int finalizeStat() override;
   //
   virtual void prepareMatrixClAlg();
   virtual void prepareMatrixClAlgReco();
@@ -100,11 +99,11 @@ class AlignableSensor : public AlignableVolume
   //
  protected:
   //
-  int mSID;                  // sensor id in detector
-  double mAddError[2];       // additional error increment for measurement
-  AlignableDetector* mDet;   // pointer on detector
-  TGeoHMatrix mMatClAlg;     // reference cluster alignment matrix in tracking frame, i.e. the matrix we want to correct
-  TGeoHMatrix mMatClAlgReco; // reco-time cluster alignment matrix in tracking frame
+  int mSID = -1;                     // sensor id in detector
+  double mAddError[2] = {0, 0};      // additional error increment for measurement
+  AlignableDetector* mDet = nullptr; // pointer on detector
+  TGeoHMatrix mMatClAlg;             // reference cluster alignment matrix in tracking frame, i.e. the matrix we want to correct
+  TGeoHMatrix mMatClAlgReco;         // reco-time cluster alignment matrix in tracking frame
 
   //
   ClassDefOverride(AlignableSensor, 1)
diff --git a/Detectors/Align/include/Align/AlignableSensorHMPID.h b/Detectors/Align/include/Align/AlignableSensorHMPID.h
index 6064078d1b923..867b72152c73c 100644
--- a/Detectors/Align/include/Align/AlignableSensorHMPID.h
+++ b/Detectors/Align/include/Align/AlignableSensorHMPID.h
@@ -29,7 +29,7 @@ namespace o2
 namespace align
 {
 
-class AlignableSensorHMPID : public AlignableSensor
+class AlignableSensorHMPID final : public AlignableSensor
 {
  public:
   AlignableSensorHMPID(const char* name = 0, int vid = 0, int iid = 0, int isec = 0);
diff --git a/Detectors/Align/include/Align/AlignableSensorITS.h b/Detectors/Align/include/Align/AlignableSensorITS.h
index de0e6d35629ec..7769eb6aa22f3 100644
--- a/Detectors/Align/include/Align/AlignableSensorITS.h
+++ b/Detectors/Align/include/Align/AlignableSensorITS.h
@@ -29,13 +29,15 @@ namespace o2
 namespace align
 {
 
-class AlignableSensorITS : public AlignableSensor
+class AlignableSensorITS final : public AlignableSensor
 {
  public:
   AlignableSensorITS() = default;
   AlignableSensorITS(const char* name, int vid, int iid, Controller* ctr);
   ~AlignableSensorITS() final = default;
   void prepareMatrixT2L() final;
+  void prepareMatrixL2G(bool reco = false) final;
+  void prepareMatrixL2GIdeal() final;
   //
  protected:
   //
diff --git a/Detectors/Align/include/Align/AlignableSensorTOF.h b/Detectors/Align/include/Align/AlignableSensorTOF.h
index 69cce00495e3f..d9e11dce98a0c 100644
--- a/Detectors/Align/include/Align/AlignableSensorTOF.h
+++ b/Detectors/Align/include/Align/AlignableSensorTOF.h
@@ -29,20 +29,21 @@ namespace o2
 namespace align
 {
 
-class AlignableSensorTOF : public AlignableSensor
+class AlignableSensorTOF final : public AlignableSensor
 {
  public:
-  AlignableSensorTOF(const char* name = 0, int vid = 0, int iid = 0, int isec = 0);
-  ~AlignableSensorTOF() = final;
+  AlignableSensorTOF() = default;
+  AlignableSensorTOF(const char* name, int vid, int iid, int isec, Controller* ctr);
+  ~AlignableSensorTOF() final = default;
   //
   void prepareMatrixT2L() final;
   //
-  int GetSector() const { return fSector; }
-  void SetSector(uint32_t sc) { fSector = (uint8_t)sc; }
+  int getSector() const { return mSector; }
+  void setSector(uint32_t sc) { mSector = (uint8_t)sc; }
   //
  protected:
   //
-  uint8_t fSector; // sector ID
+  uint8_t mSector; // sector ID
   //
   ClassDef(AlignableSensorTOF, 1)
 };
diff --git a/Detectors/Align/include/Align/AlignableSensorTPC.h b/Detectors/Align/include/Align/AlignableSensorTPC.h
index ee1b0caf5ff30..838e8c8272400 100644
--- a/Detectors/Align/include/Align/AlignableSensorTPC.h
+++ b/Detectors/Align/include/Align/AlignableSensorTPC.h
@@ -10,39 +10,35 @@
 // or submit itself to any jurisdiction.
 
 /// @file   AlignableSensorTPC.h
-/// @author ruben.shahoyan@cern.ch, michael.lettrich@cern.ch
-/// @since  2021-02-01
-/// @brief  TPC sensor (chamber)
+/// @author ruben.shahoyan@cern.ch
+/// @brief  TPC fake sensor (sector)
 
 #ifndef ALIGNABLESENSORTPC_H
 #define ALIGNABLESENSORTPC_H
 
 #include "Align/AlignableSensor.h"
-
-class TObjArray;
-//class AliTrackPointArray;
-//class AliESDtrack;
 class AlignmentPoint;
+class TObjArray;
 
 namespace o2
 {
 namespace align
 {
 
-class AlignableSensorTPC : public AlignableSensor
+class AlignableSensorTPC final : public AlignableSensor
 {
  public:
-  AlignableSensorTPC(const char* name = 0, int vid = 0, int iid = 0, int isec = 0);
-  ~AlignableSensorTPC() = final;
-  //
-  int GetSector() const { return fSector; }
-  void SetSector(uint32_t sc) { fSector = (uint8_t)sc; }
-  //
+  AlignableSensorTPC() = default;
+  AlignableSensorTPC(const char* name, int vid, int iid, int isec, Controller* ctr);
+  ~AlignableSensorTPC() final = default;
+  int getSector() const { return mSector; }
+  void setSector(int sc) { mSector = (uint8_t)sc; }
   void prepareMatrixT2L() final;
-  //
+  void prepareMatrixL2G(bool reco = false) final;
+  void prepareMatrixL2GIdeal() final;
+
  protected:
-  //
-  uint8_t fSector; // sector ID
+  uint8_t mSector = 0; // sector ID
 
   ClassDef(AlignableSensorTPC, 1)
 };
diff --git a/Detectors/Align/include/Align/AlignableSensorTRD.h b/Detectors/Align/include/Align/AlignableSensorTRD.h
index d5d0d1bb885d3..eabc3905e9e13 100644
--- a/Detectors/Align/include/Align/AlignableSensorTRD.h
+++ b/Detectors/Align/include/Align/AlignableSensorTRD.h
@@ -18,8 +18,6 @@
 #define ALIGNABLESENSORTRD_H
 
 #include "Align/AlignableSensor.h"
-//class AliTrackPointArray;
-//class AliESDtrack;
 class AlignmentPoint;
 class TObjArray;
 
@@ -28,22 +26,24 @@ namespace o2
 namespace align
 {
 
-class AlignableSensorTRD : public AlignableSensor
+class AlignableSensorTRD final : public AlignableSensor
 {
  public:
-  AlignableSensorTRD(const char* name = 0, int vid = 0, int iid = 0, int isec = 0);
-  ~AlignableSensorTRD() final;
-  //
-  int GetSector() const { return fSector; }
-  void SetSector(uint32_t sc) { fSector = (uint8_t)sc; }
-  //
-  void dPosTraDParCalib(const AlignmentPoint* pnt, double* deriv, int calibID, const AlignableVolume* parent = 0) const final;
-  //
+  AlignableSensorTRD() = default;
+  AlignableSensorTRD(const char* name, int vid, int iid, int isec, Controller* ctr);
+  ~AlignableSensorTRD() final = default;
+  int getSector() const { return mSector; }
+  void setSector(int sc) { mSector = (uint8_t)sc; }
+  void dPosTraDParCalib(const AlignmentPoint* pnt, double* deriv, int calibID, const AlignableVolume* parent = nullptr) const final;
+
+  void prepareMatrixL2G(bool reco = false) final;
+  void prepareMatrixL2GIdeal() final;
   void prepareMatrixT2L() final;
-  //
+  void prepareMatrixClAlg() final;
+  void prepareMatrixClAlgReco() final;
+
  protected:
-  //
-  uint8_t fSector; // sector ID
+  uint8_t mSector = 0; // sector ID
 
   ClassDef(AlignableSensorTRD, 1)
 };
diff --git a/Detectors/Align/include/Align/AlignableVolume.h b/Detectors/Align/include/Align/AlignableVolume.h
index 3c94d5836cd01..15b077eb2700f 100644
--- a/Detectors/Align/include/Align/AlignableVolume.h
+++ b/Detectors/Align/include/Align/AlignableVolume.h
@@ -29,7 +29,6 @@
 #include 
 #include 
 #include "DetectorsCommonDataFormats/AlignParam.h"
-#include "Align/DOFStatistics.h"
 #include "Align/DOFSet.h"
 #include 
 
@@ -107,16 +106,23 @@ class AlignableVolume : public DOFSet
     mDOF = pat;
     calcFree();
   }
+
+  void setMeasuredDOFPattern(uint32_t pat)
+  {
+    mDOFAsMeas = pat;
+  }
+  bool isNameMatching(const std::string& regexStr) const;
   bool isFreeDOF(int dof) const { return (mDOF & (0x1 << dof)) != 0; }
+  bool isMeasuredDOF(int dof) const { return isFreeDOF(dof) && ((mDOFAsMeas & (0x1 << dof)) != 0); }
   bool isCondDOF(int dof) const;
   uint32_t getFreeDOFPattern() const { return mDOF; }
   uint32_t getFreeDOFGeomPattern() const { return mDOF & kAllGeomDOF; }
   //
-  void addAutoConstraints(TObjArray* constrArr);
+  void addAutoConstraints();
   bool isChildrenDOFConstrained(int dof) const { return mConstrChild & 0x1 << dof; }
   uint8_t getChildrenConstraintPattern() const { return mConstrChild; }
   void constrainChildrenDOF(int dof) { mConstrChild |= 0x1 << dof; }
-  void uConstrainChildrenDOF(int dof) { mConstrChild &= ~(0x1 << dof); }
+  void unConstrainChildrenDOF(int dof) { mConstrChild &= ~(0x1 << dof); }
   void setChildrenConstrainPattern(uint32_t pat) { mConstrChild = pat; }
   bool hasChildrenConstraint() const { return mConstrChild; }
   //
@@ -138,8 +144,7 @@ class AlignableVolume : public DOFSet
   double getAlpTracking() const { return mAlp; }
   //
   int getNProcessedPoints() const { return mNProcPoints; }
-  virtual int finalizeStat(DOFStatistics& h);
-  void fillDOFStat(DOFStatistics& h) const;
+  virtual int finalizeStat();
   //
   int getNDOFGeomFree() const { return mNDOFGeomFree; }
   //
@@ -173,12 +178,12 @@ class AlignableVolume : public DOFSet
   void getDeltaT2LmodTRA(TGeoHMatrix& matMod, const double* delta, const TGeoHMatrix& relMat) const;
   //
   // creation of global matrices for storage
-  void createGloDeltaMatrix(TGeoHMatrix& deltaM) const;
-  void createLocDeltaMatrix(TGeoHMatrix& deltaM) const;
+  bool createGloDeltaMatrix(TGeoHMatrix& deltaM) const;
+  bool createLocDeltaMatrix(TGeoHMatrix& deltaM) const; // return true if the matrix is not unity
   void createPreGloDeltaMatrix(TGeoHMatrix& deltaM) const;
   void createPreLocDeltaMatrix(TGeoHMatrix& deltaM) const;
-  void createAlignmenMatrix(TGeoHMatrix& alg) const;
-  void createAlignmentObjects(std::vector& arr) const;
+  void createAlignmenMatrix(TGeoHMatrix& alg, const TGeoHMatrix* envelopeDelta = nullptr) const;
+  void createAlignmentObjects(std::vector& arr, const TGeoHMatrix* envelopeDelta = nullptr) const;
   //
   void setSkip(bool v = true) { SetBit(kSkipBit, v); }
   bool getSkip() const { return TestBit(kSkipBit); }
@@ -192,18 +197,25 @@ class AlignableVolume : public DOFSet
   bool ownsDOFID(int id) const;
   AlignableVolume* getVolOfDOFID(int id) const;
   //
+  bool isDummyEnvelope() const { return mIsDummyEnvelope; }
+  void setDummyEnvelope(bool v = true) { mIsDummyEnvelope = v; }
+  //
+  bool isDummy() const { return mIsDummy; }
+  void setDummy(bool v) { mIsDummy = v; }
+  //
   virtual bool isSensor() const { return false; }
   //
   virtual const char* getDOFName(int i) const;
   void Print(const Option_t* opt = "") const override;
   virtual void writePedeInfo(FILE* parOut, const Option_t* opt = "") const;
+  virtual void writeLabeledPedeResults(FILE* parOut) const;
   //
   static const char* getGeomDOFName(int i) { return i < kNDOFGeom ? sDOFName[i] : nullptr; }
   static void setDefGeomFree(uint8_t patt) { sDefGeomFree = patt; }
   static uint8_t getDefGeomFree() { return sDefGeomFree; }
   //
  protected:
-  void calcFree(bool condFree = false);
+  void calcFree(bool condFree = true);
   //
   // ------- dummies -------
   AlignableVolume(const AlignableVolume&);
@@ -217,6 +229,10 @@ class AlignableVolume : public DOFSet
   double mAlp = 0.;         // tracking frame alpa
   //
   uint32_t mDOF = 0;        // pattern of DOFs
+  uint32_t mDOFAsMeas = 0;  // consider DOF as measured with presigma error
+  bool mIsDummy = false;    // placeholder (e.g. inactive), used to have the numbering corresponding to position in the container
+  bool mIsDummyEnvelope = false; // some volumes are dummy envelopes for their children
+  //
   char mNDOFGeomFree = 0;   // number of free geom degrees of freedom
   uint8_t mConstrChild = 0; // bitpattern for constraints on children corrections
   //
diff --git a/Detectors/Align/include/Align/AlignmentPoint.h b/Detectors/Align/include/Align/AlignmentPoint.h
index 1e8b785f9a9b3..84b3f43f3d976 100644
--- a/Detectors/Align/include/Align/AlignmentPoint.h
+++ b/Detectors/Align/include/Align/AlignmentPoint.h
@@ -57,8 +57,7 @@ class AlignmentPoint
               kOptWSB = 0x1 << 13
 
   };
-  enum { kParY = 0 // track parameters
-         ,
+  enum { kParY = 0, // track parameters
          kParZ,
          kParSnp,
          kParTgl,
@@ -77,7 +76,6 @@ class AlignmentPoint
   //
   double getAlphaSens() const { return mAlphaSens; }
   double getXSens() const { return mXSens; }
-  double getXPoint() const { return mXSens + getXTracking(); }
   double getXTracking() const { return mXYZTracking[0]; }
   double getYTracking() const { return mXYZTracking[1]; }
   double getZTracking() const { return mXYZTracking[2]; }
@@ -193,7 +191,7 @@ class AlignmentPoint
   void incrementStat();
   //
   virtual void dumpCoordinates() const;
-  void print(uint16_t opt) const;
+  void print(uint16_t opt = 0) const;
   void clear();
   //
   bool isAfter(const AlignmentPoint& pnt) const;
diff --git a/Detectors/Align/include/Align/AlignmentTrack.h b/Detectors/Align/include/Align/AlignmentTrack.h
index c16d51086dfc0..cb69f11cbf85c 100644
--- a/Detectors/Align/include/Align/AlignmentTrack.h
+++ b/Detectors/Align/include/Align/AlignmentTrack.h
@@ -24,6 +24,7 @@
 #define ALIGNMENTTRACK_H
 
 #include "Align/AlignmentPoint.h"
+#include "ReconstructionDataFormats/GlobalTrackID.h"
 #include "ReconstructionDataFormats/Track.h"
 #include 
 #include 
@@ -38,8 +39,10 @@ class AlignmentTrack : public trackParam_t, public TObject
 {
  public:
   using trackParam_t = o2::track::TrackParametrizationWithError;
+  using trackPar_t = o2::track::TrackParametrization;
   using PropagatorD = o2::base::PropagatorD;
   using MatCorrType = PropagatorD::MatCorrType;
+  using GTrackID = o2::dataformats::GlobalTrackID;
   using trackParam_t::setParam;
   static constexpr double MaxDefStep = 3.0;
   static constexpr double MaxDefSnp = 0.95;
@@ -49,12 +52,9 @@ class AlignmentTrack : public trackParam_t, public TObject
          kResidDoneBit = BIT(16),
          kDerivDoneBit = BIT(17),
          kKalmanDoneBit = BIT(18) };
-  enum { kNKinParBOFF = 4 // N params for ExternalTrackParam part w/o field
-         ,
-         kNKinParBON = 5 // N params for ExternalTrackParam part with field
-         ,
-         kParY = 0 // track parameters
-         ,
+  enum { kNKinParBOFF = 4, // N params for ExternalTrackParam part w/o field
+         kNKinParBON = 5,  // N params for ExternalTrackParam part with field
+         kParY = 0,        // track parameters
          kParZ,
          kParSnp,
          kParTgl,
@@ -62,6 +62,8 @@ class AlignmentTrack : public trackParam_t, public TObject
   };
   AlignmentTrack() = default;
   ~AlignmentTrack() override = default;
+  void setCurrentTrackID(GTrackID id) { mCurrenTrackID = id; }
+  GTrackID getCurrentTrackID() const { return mCurrenTrackID; }
   void defineDOFs();
   int getNPoints() const { return mPoints.size(); }
   int getInnerPointID() const { return mInnerPointID; }
@@ -70,6 +72,8 @@ class AlignmentTrack : public trackParam_t, public TObject
   const AlignmentPoint* getPoint(int i) const { return mPoints[i]; }
   auto getPoints() { return mPoints; }
   auto& addDetectorPoint() { return mDetPoints.emplace_back(); }
+  std::vector& getDetPoints() { return mDetPoints; }
+  void suppressLastPoints(int n);
   void setRefPoint(AlignmentPoint* p) { mPoints.emplace_back(p); }
   int getNLocPar() const { return mNLocPar; }
   int getNLocExtPar() const { return mNLocExtPar; }
@@ -80,14 +84,14 @@ class AlignmentTrack : public trackParam_t, public TObject
   //
   template 
   void copyFrom(const o2::track::TrackParametrizationWithError
& trc);
-
-  bool propagateToPoint(trackParam_t& tr, const AlignmentPoint* pnt, double maxStep, double maxSnp = 0.95, MatCorrType mt = MatCorrType::USEMatCorrLUT, track::TrackLTIntegral* tLT = nullptr);
-  bool propagateParamToPoint(trackParam_t& tr, const AlignmentPoint* pnt, double maxStep = 3, double maxSnp = 0.95, MatCorrType mt = MatCorrType::USEMatCorrLUT);             // param only
-  bool propagateParamToPoint(trackParam_t* trSet, int nTr, const AlignmentPoint* pnt, double maxStep = 3, double maxSnp = 0.95, MatCorrType mt = MatCorrType::USEMatCorrLUT); // params only
+  bool propagateToPoint(trackParam_t& tr, trackPar_t* linRef, const AlignmentPoint* pnt, double maxStep, double maxSnp = 0.95, MatCorrType mt = MatCorrType::USEMatCorrLUT, track::TrackLTIntegral* tLT = nullptr, int signCorr = 0);
+  bool propagateParamToPoint(trackPar_t& tr, const AlignmentPoint* pnt, double maxStep = 3, double maxSnp = 0.95, MatCorrType mt = MatCorrType::USEMatCorrLUT, int signCorr = 0);             // param only
+  bool propagateParamToPoint(trackPar_t* trSet, int nTr, const AlignmentPoint* pnt, double maxStep = 3, double maxSnp = 0.95, MatCorrType mt = MatCorrType::USEMatCorrLUT, int signCorr = 0); // params only
   //
   bool calcResiduals(const double* params = nullptr);
   bool calcResidDeriv(double* params = nullptr);
   bool calcResidDerivGlo(AlignmentPoint* pnt);
+  bool testLocalSolution();
   //
   bool isCosmic() const { return TestBit(kCosmicBit); }
   void setCosmic(bool v = true) { SetBit(kCosmicBit, v); }
@@ -116,27 +120,29 @@ class AlignmentTrack : public trackParam_t, public TObject
   void imposePtBOff(double pt) { setQ2Pt(1. / pt); }
   // propagation methods
   void copyFrom(const trackParam_t* etp);
-  bool applyMatCorr(trackParam_t& trPar, const double* corrDiag, const AlignmentPoint* pnt);
-  bool applyMatCorr(trackParam_t* trSet, int ntr, const double* corrDiaf, const AlignmentPoint* pnt);
-  bool applyMatCorr(trackParam_t& trPar, const double* corrpar);
+  bool applyMatCorr(trackPar_t& trPar, const double* corrDiag, const AlignmentPoint* pnt);
+  bool applyMatCorr(trackPar_t* trSet, int ntr, const double* corrDiaf, const AlignmentPoint* pnt);
+  bool applyMatCorr(trackPar_t& trPar, const double* corrpar);
   //
   double getResidual(int dim, int pntID) const { return mResid[dim][pntID]; }
   const double* getDResDLoc(int dim, int pntID) const { return mDResDLoc[dim].data() + (pntID * mNLocPar); }
   const double* getDResDGlo(int dim, int id) const { return mDResDGlo[dim].data() + id; }
   const int* getGloParID() const { return mGloParID.data(); }
   //
-  void setParams(trackParam_t& tr, double x, double alp, const double* par, bool add);
-  void setParams(trackParam_t* trSet, int ntr, double x, double alp, const double* par, bool add);
-  void setParam(trackParam_t& tr, int par, double val);
-  void setParam(trackParam_t* trSet, int ntr, int par, double val);
-  void modParam(trackParam_t& tr, int par, double delta);
-  void modParam(trackParam_t* trSet, int ntr, int par, double delta);
+  void setParams(trackPar_t& tr, double x, double alp, const double* par, bool add);
+  void setParams(trackPar_t* trSet, int ntr, double x, double alp, const double* par, bool add);
+  void setParam(trackPar_t& tr, int par, double val);
+  void setParam(trackPar_t* trSet, int ntr, int par, double val);
+  void modParam(trackPar_t& tr, int par, double delta);
+  void modParam(trackPar_t* trSet, int ntr, int par, double delta);
   //
-  void richardsonDeriv(const trackParam_t* trSet, const double* delta,
+  void richardsonDeriv(const trackPar_t* trSet, const double* delta,
                        const AlignmentPoint* pnt, double& derY, double& derZ);
   //
   const double* getLocPars() const { return mLocPar.data(); }
+  std::vector& getLocParsV() { return mLocPar; }
   void setLocPars(const double* pars);
+  void addLocPars(const double* pars);
   //
  protected:
   //
@@ -165,6 +171,7 @@ class AlignmentTrack : public trackParam_t, public TObject
   double mChi2CosmUp = 0;                   // chi2 for cosmic upper leg
   double mChi2CosmDn = 0;                   // chi2 for cosmic down leg
   double mChi2Ini = 0;                      // chi2 with current residuals
+  GTrackID mCurrenTrackID = {};             // currently processed track ID
   std::vector mPoints{};   // alignment points pointers sorted in X
   std::vector mDetPoints{}; // alignment points added by detectors
   std::vector mResid[2];            // residuals array
@@ -173,13 +180,14 @@ class AlignmentTrack : public trackParam_t, public TObject
   std::vector mLocPar;              // local parameters array
   std::vector mGloParID;               // IDs of relevant global params
  private:
-  bool propagate(trackParam_t& tr, const AlignmentPoint* pnt, double maxStep, double maxSnp, MatCorrType mt, track::TrackLTIntegral* tLT);
+  bool propagate(trackParam_t& tr, trackPar_t* linRef, const AlignmentPoint* pnt, double maxStep, double maxSnp, MatCorrType mt, track::TrackLTIntegral* tLT, int signCorr = 0);
+  bool propagate(trackPar_t& tr, const AlignmentPoint* pnt, double maxStep, double maxSnp, MatCorrType mt, track::TrackLTIntegral* tLT, int signCorr = 0);
   //
   ClassDefOverride(AlignmentTrack, 2)
 };
 
 //____________________________________________________________________________________________
-inline void AlignmentTrack::setParams(trackParam_t& tr, double x, double alp, const double* par, bool add)
+inline void AlignmentTrack::setParams(trackPar_t& tr, double x, double alp, const double* par, bool add)
 {
   // set track params
   const double kDefQ2PtCosm = 1;
@@ -199,7 +207,7 @@ inline void AlignmentTrack::setParams(trackParam_t& tr, double x, double alp, co
 }
 
 //____________________________________________________________________________________________
-inline void AlignmentTrack::setParams(trackParam_t* trSet, int ntr, double x, double alp, const double* par, bool add)
+inline void AlignmentTrack::setParams(trackPar_t* trSet, int ntr, double x, double alp, const double* par, bool add)
 {
   // set parames for multiple tracks (VECTORIZE THIS)
   if (!add) { // full parameter supplied
@@ -218,14 +226,14 @@ inline void AlignmentTrack::setParams(trackParam_t* trSet, int ntr, double x, do
 }
 
 //____________________________________________________________________________________________
-inline void AlignmentTrack::setParam(trackParam_t& tr, int par, double val)
+inline void AlignmentTrack::setParam(trackPar_t& tr, int par, double val)
 {
   // set track parameter
   tr.setParam(val, par);
 }
 
 //____________________________________________________________________________________________
-inline void AlignmentTrack::setParam(trackParam_t* trSet, int ntr, int par, double val)
+inline void AlignmentTrack::setParam(trackPar_t* trSet, int ntr, int par, double val)
 {
   // set parames for multiple tracks (VECTORIZE THIS)
   for (int i = 0; i < ntr; ++i) {
@@ -234,7 +242,7 @@ inline void AlignmentTrack::setParam(trackParam_t* trSet, int ntr, int par, doub
 }
 
 //____________________________________________________________________________________________
-inline void AlignmentTrack::modParam(trackParam_t& tr, int par, double delta)
+inline void AlignmentTrack::modParam(trackPar_t& tr, int par, double delta)
 {
   // modify track parameter
   const auto val = tr.getParam(par) + delta;
@@ -242,10 +250,10 @@ inline void AlignmentTrack::modParam(trackParam_t& tr, int par, double delta)
 }
 
 //____________________________________________________________________________________________
-inline void AlignmentTrack::modParam(trackParam_t* trSet, int ntr, int par, double delta)
+inline void AlignmentTrack::modParam(trackPar_t* trSet, int ntr, int par, double delta)
 {
   // modify track parameter (VECTORIZE THOS)
-  for (size_t i = 0; i < ntr; ++i) {
+  for (int i = 0; i < ntr; ++i) {
     modParam(trSet[i], par, delta);
   }
 }
diff --git a/Detectors/Align/include/Align/Controller.h b/Detectors/Align/include/Align/Controller.h
index bab0b95a81b38..90abf2025d1c3 100644
--- a/Detectors/Align/include/Align/Controller.h
+++ b/Detectors/Align/include/Align/Controller.h
@@ -26,11 +26,15 @@
 #include "DetectorsBase/GeometryManager.h"
 #include "DetectorsBase/Propagator.h"
 #include "Align/AlignmentTrack.h"
+#include "Align/AlgTrcDbg.h"
 #include "ReconstructionDataFormats/PrimaryVertex.h"
-// #include "AliSymMatrix.h" FIXME(milettri): needs AliSymMatrix
+#include "ReconstructionDataFormats/TrackCosmics.h"
+#include "DataFormatsTPC/VDriftCorrFact.h"
+#include "CorrectionMapsHelper.h"
 
 #include "Align/Millepede2Record.h"
 #include "Align/ResidualsController.h"
+#include "Align/GeometricalConstraint.h"
 
 #include 
 #include 
@@ -40,6 +44,8 @@
 #include 
 #include 
 #include "Align/utils.h"
+#include "Framework/TimingInfo.h"
+#include "Align/AlignableDetector.h"
 
 // can be fwd declared if we don't require root dict.
 //class TTree;
@@ -48,6 +54,8 @@
 #include 
 #include 
 #include "Align/Mille.h"
+// #include "GPUO2ExternalUser.h"
+// #include "DataFormatsTPC/WorkflowHelper.h"
 
 namespace o2
 {
@@ -55,6 +63,19 @@ namespace globaltracking
 {
 class RecoContainer;
 }
+namespace trd
+{
+class TrackletTransformer;
+}
+namespace utils
+{
+class TreeStreamRedirector;
+}
+
+namespace gpu
+{
+class GPUParam;
+}
 
 namespace align
 {
@@ -62,31 +83,32 @@ namespace align
 //class Mille;
 
 class EventVertex;
-class AlignableDetector;
 class AlignableVolume;
 class AlignmentPoint;
 class ResidualsControllerFast;
-class GeometricalConstraint;
-class DOFStatistics;
 
-class Controller : public TObject
+class Controller final : public TObject
 {
  public:
   struct ProcStat {
-    enum { kInput,
-           kAccepted,
-           kNStatCl };
-    enum { kRun,
-           kEventColl,
-           kEventCosm,
-           kTrackColl,
-           kTrackCosm,
-           kMaxStat };
-    std::array, kNStatCl> data{};
+    enum {
+      kInput,
+      kAccepted,
+      kNStatCl
+    };
+    enum {
+      kVertices,
+      kTracks,
+      kTracksWithVertex,
+      kCosmic,
+      kMaxStat
+    };
+    std::array, kNStatCl> data{};
     void print() const;
   };
 
   using DetID = o2::detectors::DetID;
+  using GTrackID = o2::dataformats::GlobalTrackID;
 
   enum { kNLrSkip = 4 };
   enum { kITS,
@@ -99,56 +121,24 @@ class Controller : public TObject
   enum { kCosmLow,
          kCosmUp,
          kNCosmLegs };
-  enum MPOut_t { kMille = BIT(0),
-                 kMPRec = BIT(1),
-                 kContR = BIT(2) };
   enum { kInitGeomDone = BIT(14),
          kInitDOFsDone = BIT(15),
          kMPAlignDone = BIT(16) };
-  //
-  enum {     // STAT histo entries
-    kRunDone // input runs
-    ,
-    kEvInp // input events
-    ,
-    kEvVtx // after vtx selection
-    ,
-    kTrackInp // input tracks
-    ,
-    kTrackFitInp // input to ini fit
-    ,
-    kTrackFitInpVC // those with vertex constraint
-    ,
-    kTrackProcMatInp // input to process materials
-    ,
-    kTrackResDerInp // input to resid/deriv calculation
-    ,
-    kTrackStore // stored tracks
-    ,
-    kTrackAcc // tracks accepted
-    ,
-    kTrackControl // control tracks filled
-    //
-    ,
-    kNHVars
-  };
 
   Controller() = default;
-  Controller(DetID::mask_t detmask);
+  Controller(DetID::mask_t detmask, GTrackID::mask_t trcmask, bool cosmic = false, bool useMC = false, int instID = 0);
   ~Controller() final;
 
   void expandGlobalsBy(int n);
   void process();
+  void processCosmic();
 
-  //  bool LoadRefOCDB(); FIXME(milettri): needs OCDB
-  //  bool LoadRecoTimeOCDB(); FIXME(milettri): needs OCDB
   bool getUseRecoOCDB() const { return mUseRecoOCDB; }
   void setUseRecoOCDB(bool v = true) { mUseRecoOCDB = v; }
 
   void initDetectors();
   void initDOFs();
-  void terminate(bool dostat = true);
-  void setStatHistoLabels(TH1* h) const;
+  void terminate();
   //
   void setInitGeomDone() { SetBit(kInitGeomDone); }
   bool getInitGeomDone() const { return TestBit(kInitGeomDone); }
@@ -163,15 +153,14 @@ class Controller : public TObject
   //
   void addDetector(AlignableDetector* det);
   //
-  void addConstraint(const GeometricalConstraint* cs) { mConstraints.AddLast((TObject*)cs); }
-  int getNConstraints() const { return mConstraints.GetEntriesFast(); }
-  const TObjArray* getConstraints() const { return &mConstraints; }
-  const GeometricalConstraint* getConstraint(int i) const { return (GeometricalConstraint*)mConstraints[i]; }
+  int getNConstraints() const { return mConstraints.size(); }
+  const std::vector& getConstraints() const { return mConstraints; }
+  std::vector& getConstraints() { return mConstraints; }
+  const GeometricalConstraint& getConstraint(int i) const { return mConstraints[i]; }
+
   void addAutoConstraints();
   //
-  void acknowledgeNewRun(int run);
-  void setRunNumber(int run);
-  int getRunNumber() const { return mRunNumber; }
+  void setTimingInfo(const o2::framework::TimingInfo& ti);
   bool getFieldOn() const { return mFieldOn; }
   void setFieldOn(bool v = true) { mFieldOn = v; }
   int getTracksType() const { return mTracksType; }
@@ -179,7 +168,8 @@ class Controller : public TObject
   bool isCosmic() const { return mTracksType == utils::Cosm; }
   bool isCollision() const { return mTracksType == utils::Coll; }
   void setCosmic(bool v = true) { mTracksType = v ? utils::Cosm : utils::Coll; }
-  float getStat(int cls, int tp) const { return mStat.data[cls][tp]; }
+  auto getStat(int cls, int tp) const { return mStat.data[cls][tp]; }
+  auto& getStat() const { return mStat; }
   //
   bool checkDetectorPattern(DetID::mask_t patt) const;
   bool checkDetectorPoints(const int* npsel) const;
@@ -189,7 +179,7 @@ class Controller : public TObject
   //  const AliESDVertex* GetVertex() const { return fVertex; } FIXME(milettri): needs AliESDVertex
   //
   //----------------------------------------
-  bool readParameters(const char* parfile = "millepede.res", bool useErrors = true);
+  bool readParameters(const std::string& parfile = "millepede.res", bool useErrors = true);
   auto& getGloParVal() { return mGloParVal; }
   auto& getGloParErr() { return mGloParErr; }
   auto& getGloParLab() { return mGloParLab; }
@@ -197,6 +187,7 @@ class Controller : public TObject
   int parID2Label(int i) const { return getGloParLab(i); }
   int label2ParID(int lab) const;
   AlignableVolume* getVolOfDOFID(int id) const;
+  AlignableVolume* getVolOfLabel(int label) const;
   AlignableDetector* getDetOfDOFID(int id) const;
   //
   AlignmentPoint* getRefPoint() const { return mRefPoint.get(); }
@@ -209,104 +200,38 @@ class Controller : public TObject
   const o2::globaltracking::RecoContainer* getRecoContainer() const { return mRecoData; }
   void setRecoContainer(const o2::globaltracking::RecoContainer* cont) { mRecoData = cont; }
 
-  //  bool ProcessEvent(const AliESDEvent* esdEv); FIXME(milettri): needs AliESDEvent
-  //  bool ProcessTrack(const AliESDtrack* esdTr); FIXME(milettri): needs AliESDtrack
-  //  bool ProcessTrack(const AliESDCosmicTrack* esdCTr); FIXME(milettri): needs AliESDCosmicTrack
-  //  uint32_t AcceptTrack(const AliESDtrack* esdTr, bool strict = true) const; FIXME(milettri): needs AliESDtrack
-  //  uint32_t AcceptTrackCosmic(const AliESDtrack* esdPairCosm[kNCosmLegs]) const; FIXME(milettri): needs AliESDtrack
-  //  bool CheckSetVertex(const AliESDVertex* vtx); FIXME(milettri): needs AliESDVertex
   bool addVertexConstraint(const o2::dataformats::PrimaryVertex& vtx);
   int getNDetectors() const { return mNDet; }
-  AlignableDetector* getDetector(DetID id) const { return mDetectors[id]; }
+  AlignableDetector* getDetector(DetID id) const { return mDetectors[id].get(); }
 
   EventVertex* getVertexSensor() const { return mVtxSens.get(); }
   //
   void resetForNextTrack();
   int getNDOFs() const { return mGloParVal.size(); }
   //----------------------------------------
-  // output related
-  void setMPDatFileName(const char* name = "mpData");
-  void setMPParFileName(const char* name = "mpParams.txt");
-  void setMPConFileName(const char* name = "mpConstraints.txt");
-  void setMPSteerFileName(const char* name = "mpSteer.txt");
-  void setResidFileName(const char* name = "mpControlRes.root");
-  void setOutCDBPath(const char* name = "local://outOCDB");
-  void setOutCDBComment(const char* cm = nullptr) { mOutCDBComment = cm; }
-  void setOutCDBResponsible(const char* v = nullptr) { mOutCDBResponsible = v; }
-  //  void SetOutCDBRunRange(int rmin = 0, int rmax = 999999999); FIXME(milettri): needs OCDB
-  int* getOutCDBRunRange() const { return (int*)mOutCDBRunRange; }
-  int getOutCDBRunMin() const { return mOutCDBRunRange[0]; }
-  int getOutCDBRunMax() const { return mOutCDBRunRange[1]; }
   float getControlFrac() const { return mControlFrac; }
   void setControlFrac(float v = 1.) { mControlFrac = v; }
-  //  void writeCalibrationResults() const; FIXME(milettri): needs OCDB
+  void writeCalibrationResults() const;
   void applyAlignmentFromMPSol();
-  const char* getOutCDBComment() const { return mOutCDBComment.c_str(); }
-  const char* getOutCDBResponsible() const { return mOutCDBResponsible.c_str(); }
-  const char* getOutCDBPath() const { return mOutCDBPath.c_str(); }
-  const char* getMPDatFileName() const { return mMPDatFileName.c_str(); }
-  const char* getResidFileName() const { return mResidFileName.c_str(); }
-  const char* getMPParFileName() const { return mMPParFileName.c_str(); }
-  const char* getMPConFileName() const { return mMPConFileName.c_str(); }
-  const char* getMPSteerFileName() const { return mMPSteerFileName.c_str(); }
-  //
-  bool fillMPRecData();
+  //
+  bool fillMPRecData(o2::dataformats::GlobalTrackID tid);
+  bool fillControlData(o2::dataformats::GlobalTrackID tid);
   bool fillMilleData();
-  bool fillControlData();
-  void setDoKalmanResid(bool v = true) { mDoKalmanResid = v; }
-  void setMPOutType(int t) { mMPOutType = t; }
-  void produceMPData(bool v = true)
-  {
-    if (v) {
-      mMPOutType |= kMille;
-    } else {
-      mMPOutType &= ~kMille;
-    }
-  }
-  void produceMPRecord(bool v = true)
-  {
-    if (v) {
-      mMPOutType |= kMPRec;
-    } else {
-      mMPOutType &= ~kMPRec;
-    }
-  }
-  void produceControlRes(bool v = true)
-  {
-    if (v) {
-      mMPOutType |= kContR;
-    } else {
-      mMPOutType &= ~kContR;
-    }
-  }
-  int getMPOutType() const { return mMPOutType; }
-  bool getDoKalmanResid() const { return mDoKalmanResid; }
-  bool getProduceMPData() const { return mMPOutType & kMille; }
-  bool getProduceMPRecord() const { return mMPOutType & kMPRec; }
-  bool getProduceControlRes() const { return mMPOutType & kContR; }
+
   void closeMPRecOutput();
   void closeMilleOutput();
   void closeResidOutput();
   void initMPRecOutput();
   void initMIlleOutput();
   void initResidOutput();
-  bool storeProcessedTrack(int what);
+  bool storeProcessedTrack(o2::dataformats::GlobalTrackID tid = {});
+  void extractDbgTrack();
   void printStatistics() const;
-  bool getMilleTXT() const { return !mMilleOutBin; }
-  void setMilleTXT(bool v = true) { mMilleOutBin = !v; }
   //
   void genPedeSteerFile(const Option_t* opt = "") const;
+  void writeLabeledPedeResults() const;
   void writePedeConstraints() const;
   void checkConstraints(const char* params = nullptr);
-  DOFStatistics& GetDOFStat() { return mDOFStat; }
-  void setDOFStat(const DOFStatistics& st) { mDOFStat = st; }
-  TH1* getHistoStat() const { return mHistoStat; }
-  void detachHistoStat() { setHistoStat(nullptr); }
-  void setHistoStat(TH1F* h) { mHistoStat = h; }
-  void fillStatHisto(int type, float w = 1);
-  void createStatHisto();
-  void fixLowStatFromDOFStat(int thresh = 40);
-  void loadStat(const char* flname);
   //
   //----------------------------------------
   //
@@ -320,10 +245,9 @@ class Controller : public TObject
   Char_t* getDOFLabelTxt(int idf) const;
   //
   static Char_t* getDetNameByDetID(int id) { return (Char_t*)sDetectorName[id]; } //RSREM
-  static void mPRec2Mille(const char* mprecfile, const char* millefile = "mpData.mille", bool bindata = true);
-  static void mPRec2Mille(TTree* mprTree, const char* millefile = "mpData.mille", bool bindata = true);
+  static void MPRec2Mille(const std::string& mprecfile, const std::string& millefile = "mpData.mille", bool bindata = true);
+  static void MPRec2Mille(TTree* mprTree, const std::string& millefile = "mpData.mille", bool bindata = true);
   //
-  //  AliSymMatrix* BuildMatrix(TVectorD& vec); FIXME(milettri): needs AliSymMatrix
   bool testLocalSolution();
   //
   // fast check of solution using derivatives
@@ -336,6 +260,33 @@ class Controller : public TObject
   void setDetectorsMask(DetID::mask_t m) { mDetMask = m; }
   DetID::mask_t getDetectorsMask() const { return mDetMask; }
 
+  void setTrackSourceMask(GTrackID::mask_t m) { mMPsrc = m; }
+  GTrackID::mask_t getTrackSourceMask() const { return mMPsrc; }
+
+  void setTRDTransformer(const o2::trd::TrackletTransformer* trans) { mTRDTransformer = trans; }
+  void setTRDTrigRecFilterActive(bool v) { mTRDTrigRecFilterActive = v; }
+  void setAllowAfterburnerTracks(bool v) { mAllowAfterburnerTracks = v; }
+
+  const o2::trd::TrackletTransformer* getTRDTransformer() const { return mTRDTransformer; }
+  bool getTRDTrigRecFilterActive() const { return mTRDTrigRecFilterActive; }
+  bool getAllowAfterburnerTracks() const { return mAllowAfterburnerTracks; }
+
+  void setTPCVDrift(const o2::tpc::VDriftCorrFact& v);
+  void setTPCCorrMaps(o2::gpu::CorrectionMapsHelper* maph);
+  o2::gpu::CorrectionMapsHelper* getTPCCorrMaps() { return mTPCCorrMapsHelper; }
+  const o2::tpc::VDriftCorrFact& getTPCVDrift() const { return mTPCDrift; }
+
+  int getInstanceID() const { return mInstanceID; }
+  void setInstanceID(int i) { mInstanceID = i; }
+
+  int getDebugOutputLevel() const { return mDebugOutputLevel; }
+  void setDebugOutputLevel(int i) { mDebugOutputLevel = i; }
+  void setDebugStream(o2::utils::TreeStreamRedirector* d) { mDBGOut = d; }
+  o2::utils::TreeStreamRedirector* getDebugStream() { return mDBGOut; }
+
+  void setTPCParam(const o2::gpu::GPUParam* par) { mTPCParam = par; }
+  const o2::gpu::GPUParam* getTPCParam() const { return mTPCParam; }
+
  protected:
   //
   // --------- dummies -----------
@@ -345,19 +296,31 @@ class Controller : public TObject
  protected:
   //
   DetID::mask_t mDetMask{};
-
+  GTrackID::mask_t mMPsrc{};
+  std::vector mTrackSources;
+  o2::framework::TimingInfo mTimingInfo{};
+  int mInstanceID = 0; // instance in case of pipelining
+  int mRunNumber = 0;
   int mNDet = 0;                             // number of deectors participating in the alignment
   int mNDOFs = 0;                            // number of degrees of freedom
-  int mRunNumber = -1;                       // current run number
+  bool mUseMC = false;
   bool mFieldOn = false;                     // field on flag
   int mTracksType = utils::Coll;             // collision/cosmic event type
+  float mMPRecOutFraction = 0.;
+  float mControlFraction = 0.;
   std::unique_ptr mAlgTrack; // current alignment track
+  AlgTrcDbg mAlgTrackDbg;                    // current alignment track debug version
   const o2::globaltracking::RecoContainer* mRecoData = nullptr; // externally set RecoContainer
+  const o2::trd::TrackletTransformer* mTRDTransformer = nullptr;  // TRD tracket transformer
+  bool mTRDTrigRecFilterActive = false;                           // select TRD triggers processed with ITS
+  bool mAllowAfterburnerTracks = false;                           // allow using ITS-TPC afterburner tracks
+
+  const o2::gpu::GPUParam* mTPCParam = nullptr;
 
-  std::array mDetectors{}; // detectors participating in the alignment
+  std::array, DetID::nDetectors> mDetectors{}; // detectors participating in the alignment
 
   std::unique_ptr mVtxSens; // fake sensor for the vertex
-  TObjArray mConstraints{};              // array of constraints
+  std::vector mConstraints{}; // array of constraints
   //
   // Track selection
   std::array mObligatoryDetPattern{}; // pattern of obligatory detectors
@@ -365,17 +328,19 @@ class Controller : public TObject
   std::vector mGloParVal; // parameters for DOFs
   std::vector mGloParErr; // errors for DOFs
   std::vector mGloParLab;   // labels for DOFs
-  std::vector mOrderedLbl;  //ordered labels
-  std::vector mLbl2ID;      //Label order in mOrderedLbl -> parID
+  std::unordered_map mLbl2ID; // Labels mapping to parameter ID
   //
   std::unique_ptr mRefPoint; //! reference point for track definition
   //
+  int mDebugOutputLevel = 0;
+  o2::utils::TreeStreamRedirector* mDBGOut = nullptr;
+
   // statistics
   ProcStat mStat{}; // processing statistics
+  int mNTF = 0;
   //
   // output related
   float mControlFrac = 1.0;                    //  fraction of tracks to process control residuals
-  int mMPOutType = kMille | kMPRec | kContR;   // What to store as an output, see storeProcessedTrack
   std::unique_ptr mMille;               //! Mille interface
   Millepede2Record mMPRecord;                  //! MP record
   Millepede2Record* mMPRecordPtr = &mMPRecord; //! MP record
@@ -386,33 +351,21 @@ class Controller : public TObject
   std::unique_ptr mResidTree; //! tree to store control residuals
   std::unique_ptr mMPRecFile; //! file to store MP record tree
   std::unique_ptr mResidFile; //! file to store control residuals tree
-  TArrayF mMilleDBuffer;        //! buffer for Mille Derivatives output
-  TArrayI mMilleIBuffer;        //! buffer for Mille Indecis output
-  std::string mMPDatFileName{"mpData"};            //  file name for records binary data output
-  std::string mMPParFileName{"mpParams.txt"};      //  file name for MP params
-  std::string mMPConFileName{"mpConstraints.txt"}; //  file name for MP constraints
-  std::string mMPSteerFileName{"mpSteer.txt"};     //  file name for MP steering
-  std::string mResidFileName{"mpContolRes.root"};  //  file name for optional control residuals
-  bool mMilleOutBin = true;                        //  optionally text output for Mille debugging
-  bool mDoKalmanResid = true;                      //  calculate residuals with smoothed kalman in the ControlRes
-  //
-  std::string mOutCDBPath{};        // output OCDB path
-  std::string mOutCDBComment{};     // optional comment to add to output cdb objects
-  std::string mOutCDBResponsible{}; // optional responsible for output metadata
-  int mOutCDBRunRange[2] = {};      // run range for output storage
-  //
-  DOFStatistics mDOFStat;     // stat of entries per dof
-  TH1F* mHistoStat = nullptr; // histo with general statistics
+  std::string mMilleFileName{};      //!
   //
   // input related
   int mRefRunNumber = 0;    // optional run number used for reference
   int mRefOCDBLoaded = 0;   // flag/counter for ref.OCDB loading
   bool mUseRecoOCDB = true; // flag to preload reco-time calib objects
+
+  o2::tpc::VDriftCorrFact mTPCDrift{};
+  o2::gpu::CorrectionMapsHelper* mTPCCorrMapsHelper = nullptr;
+
   //
   static const int sSkipLayers[kNLrSkip];          // detector layers for which we don't need module matrices
   static const Char_t* sDetectorName[kNDetectors]; // names of detectors //RSREM
-  static const Char_t* sHStatName[kNHVars];        // names for stat.bins in the stat histo
   static const Char_t* sMPDataExt;                 // extension for MP2 binary data
+  static const Char_t* sMPDataTxtExt;              // extension for MP2 txt data
   //
   ClassDefOverride(Controller, 1)
 };
diff --git a/Detectors/Align/include/Align/DOFSet.h b/Detectors/Align/include/Align/DOFSet.h
index 20e638f951780..fc27d34b783d7 100644
--- a/Detectors/Align/include/Align/DOFSet.h
+++ b/Detectors/Align/include/Align/DOFSet.h
@@ -68,13 +68,14 @@ class DOFSet : public TNamed
   float* getParVals();
   float* getParErrs();
   int* getParLabs();
+  bool varsSet() const { return mFirstParGloID != -1; }
 
   Controller* mController = nullptr;
   int mNDOFs = 0;          // number of DOFs
   int mNDOFsFree = 0;      // numer of DOFs free
   int mNCalibDOFs = 0;     // number of calibDOFs
   int mNCalibDOFsFree = 0; // number of calibDOFs free
-  int mFirstParGloID = 0;  // ID of the 1st parameter in the global results array
+  int mFirstParGloID = -1; // ID of the 1st parameter in the global results array
 
   ClassDefOverride(DOFSet, 1);
 };
diff --git a/Detectors/Align/include/Align/DOFStatistics.h b/Detectors/Align/include/Align/DOFStatistics.h
deleted file mode 100644
index 7bb25d7c7f8ea..0000000000000
--- a/Detectors/Align/include/Align/DOFStatistics.h
+++ /dev/null
@@ -1,59 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-/// @file   DOFStatistics.h
-/// @author ruben.shahoyan@cern.ch, michael.lettrich@cern.ch
-/// @since  2021-02-01
-/// @brief  Mergable Object for statistics of points used by each DOF
-
-#ifndef DOFStatistics_H
-#define DOFStatistics_H
-
-#include 
-#include 
-#include 
-
-class TH1F;
-class TCollection;
-
-namespace o2
-{
-namespace align
-{
-
-class Controller;
-
-class DOFStatistics : public TNamed
-{
- public:
-  explicit DOFStatistics(int n = 0) : TNamed("DOFStatistics", "DOF statistics"), mStat{n, 0} {};
-
-  inline int getNDOFs() const noexcept { return mStat.size(); }
-  inline int getStat(int idf) const noexcept { return idf < getNDOFs() ? mStat[idf] : 0; }
-  inline const int* getStat() const noexcept { return mStat.data(); };
-  inline void setStat(int idf, int v) { mStat[idf] = v; }
-  inline void addStat(int idf, int v) { mStat[idf] += v; }
-  inline int getNMerges() const noexcept { return mNMerges; }
-  std::unique_ptr buildHistogram(Controller* st) const;
-  void Print(Option_t* opt) const final { std::cout << "NDOFs: " << mStat.size() << " NMerges: " << mNMerges << "\n"; };
-  int64_t merge(TCollection* list);
-
- protected:
-
- protected:
-  int mNMerges{1};        // number of merges
-  std::vector mStat; // statistics per DOF
-
-  ClassDef(DOFStatistics, 1);
-};
-} // namespace align
-} // namespace o2
-#endif
diff --git a/Detectors/Align/include/Align/GeometricalConstraint.h b/Detectors/Align/include/Align/GeometricalConstraint.h
index c252d4591f4e3..70c0e03d1fb03 100644
--- a/Detectors/Align/include/Align/GeometricalConstraint.h
+++ b/Detectors/Align/include/Align/GeometricalConstraint.h
@@ -28,8 +28,7 @@
 #define GEOMETRICALCONSTRAINT_H
 
 #include 
-#include 
-#include 
+#include 
 #include "Align/AlignableVolume.h"
 
 namespace o2
@@ -37,24 +36,28 @@ namespace o2
 namespace align
 {
 
-class GeometricalConstraint : public TNamed
+class GeometricalConstraint
 {
  public:
   enum { kNDOFGeom = AlignableVolume::kNDOFGeom };
-  enum { kNoJacobianBit = BIT(14) };
   //
-  GeometricalConstraint(const char* name = nullptr, const char* title = nullptr);
-  ~GeometricalConstraint() override;
-  //
-  void setParent(const AlignableVolume* par);
+  const std::string getName() const { return mName; }
+  void setName(const std::string& n) { mName = n; }
+  void setParent(const AlignableVolume* par)
+  {
+    mParent = par;
+    if (mName.empty()) {
+      mName = par->getSymName();
+    }
+  }
   const AlignableVolume* getParent() const { return mParent; }
   //
-  int getNChildren() const { return mChildren.GetEntriesFast(); }
-  AlignableVolume* getChild(int i) const { return (AlignableVolume*)mChildren[i]; }
+  int getNChildren() const { return mChildren.size(); }
+  const AlignableVolume* getChild(int i) const { return mChildren[i]; }
   void addChild(const AlignableVolume* v)
   {
     if (v) {
-      mChildren.AddLast((AlignableVolume*)v);
+      mChildren.push_back(v);
     }
   }
   //
@@ -62,33 +65,30 @@ class GeometricalConstraint : public TNamed
   uint8_t getConstraintPattern() const { return mConstraint; }
   void constrainDOF(int dof) { mConstraint |= 0x1 << dof; }
   void unConstrainDOF(int dof) { mConstraint &= ~(0x1 << dof); }
-  void setConstrainPattern(uint32_t pat) { mConstraint = pat; }
+  void setConstrainPattern(uint8_t pat) { mConstraint = pat; }
   bool hasConstraint() const { return mConstraint; }
   double getSigma(int i) const { return mSigma[i]; }
   void setSigma(int i, double s = 0) { mSigma[i] = s; }
   //
-  void setNoJacobian(bool v = true) { SetBit(kNoJacobianBit, v); }
-  bool getNoJacobian() const { return TestBit(kNoJacobianBit); }
+  void setNoJacobian(bool v = true) { mNoJacobian = v; }
+  bool getNoJacobian() const { return mNoJacobian; }
   //
-  void constrCoefGeom(const TGeoHMatrix& matRD, float* jac /*[kNDOFGeom][kNDOFGeom]*/) const;
-  //
-  void Print(const Option_t* opt = "") const final;
-  virtual void writeChildrenConstraints(FILE* conOut) const;
-  virtual void checkConstraint() const;
-  virtual const char* getDOFName(int i) const { return AlignableVolume::getGeomDOFName(i); }
-  //
- protected:
-  // ------- dummies -------
-  GeometricalConstraint(const GeometricalConstraint&);
-  GeometricalConstraint& operator=(const GeometricalConstraint&);
+  void constrCoefGeom(const TGeoHMatrix& matRD, double* jac /*[kNDOFGeom][kNDOFGeom]*/) const;
   //
+  void print() const;
+  void writeChildrenConstraints(FILE* conOut) const;
+  void checkConstraint() const;
+  const char* getDOFName(int i) const { return AlignableVolume::getGeomDOFName(i); }
+
  protected:
-  uint32_t mConstraint;           // bit pattern of constraint
-  double mSigma[kNDOFGeom];       // optional sigma if constraint is gaussian
-  const AlignableVolume* mParent; // parent volume for contraint, lab if 0
-  TObjArray mChildren;            // volumes subjected to constraints
+  bool mNoJacobian = false;                      // flag that Jacobian is not needed
+  uint8_t mConstraint = 0;                       // bit pattern of constraint
+  double mSigma[kNDOFGeom] = {};                 // optional sigma if constraint is gaussian
+  const AlignableVolume* mParent = nullptr;      // parent volume for contraint, lab if 0
+  std::vector mChildren; // volumes subjected to constraints
+  std::string mName{};
   //
-  ClassDef(GeometricalConstraint, 2);
+  ClassDefNV(GeometricalConstraint, 2);
 };
 
 } // namespace align
diff --git a/Detectors/Align/include/Align/Mille.h b/Detectors/Align/include/Align/Mille.h
index 82de5084033be..9dc6245ab50f3 100644
--- a/Detectors/Align/include/Align/Mille.h
+++ b/Detectors/Align/include/Align/Mille.h
@@ -42,30 +42,23 @@ namespace align
 class Mille
 {
  public:
-  Mille(const char* outFileName, bool asBinary = true, bool writeZero = false);
-  ~Mille();
-
-  void mille(int NLC, const float* derLc, int NGL, const float* derGl,
-             const int* label, float rMeas, float sigma);
+  Mille(const std::string& outFileName, bool asBinary = true, bool writeZero = false);
+  void mille(int NLC, const float* derLc, int NGL, const float* derGl, const int* label, float rMeas, float sigma);
   void special(int nSpecial, const float* floatings, const int* integers);
-  void kill();
-  int end();
+  void clear();
+  int finalise();
+  void kill() { clear(); };        // alias to old Mille method
+  int end() { return finalise(); } //  alias to old Mille method
 
  private:
-  void newSet();
-  bool checkBufferSize(int nLocal, int nGlobal);
-
-  std::ofstream myOutFile; ///< C-binary for output
-  bool myAsBinary;         ///< if false output as text
-  bool myWriteZero;        ///< if true also write out derivatives/labels ==0
-  /// buffer size for ints and floats
-  int myBufferSize;      ///< buffer size for ints and floats
-  TArrayI myBufferInt;   ///< to collect labels etc.
-  TArrayF myBufferFloat; ///< to collect derivatives etc.
-  int myBufferPos;       ///< position in buffer
-  bool myHasSpecial;     ///< if true, special(..) already called for this record
   /// largest label allowed: 2^31 - 1
-  enum { myMaxLabel = (0xFFFFFFFF - (1 << 31)) };
+  static constexpr int MaxLabel = 0x7fffffff;
+  std::ofstream mOutFile = {};     ///< C-binary for output
+  bool mAsBinary = true;           ///< if false output as text
+  bool mWriteZero = false;         ///< if true also write out derivatives/labels ==0
+  bool mHasSpecial = false;        ///< if true, special(..) already called for this record
+  std::vector mBufferInt;     ///< to collect labels etc.
+  std::vector mBufferFloat; ///< to collect derivatives etc.
 };
 
 } // namespace align
diff --git a/Detectors/Align/include/Align/Millepede2Record.h b/Detectors/Align/include/Align/Millepede2Record.h
index 9733fd321a88a..800c17481d534 100644
--- a/Detectors/Align/include/Align/Millepede2Record.h
+++ b/Detectors/Align/include/Align/Millepede2Record.h
@@ -17,7 +17,8 @@
 #ifndef MILLEPEDE2RECORD_H
 #define MILLEPEDE2RECORD_H
 
-#include 
+#include 
+#include "ReconstructionDataFormats/GlobalTrackID.h"
 
 namespace o2
 {
@@ -26,22 +27,22 @@ namespace align
 
 class AlignmentTrack;
 
-class Millepede2Record : public TObject
+class Millepede2Record
 {
  public:
-  enum { kCosmicBit = BIT(14) };
+  enum { CosmicBit = 0x1 };
   //
-  Millepede2Record();
-  ~Millepede2Record() final;
+  Millepede2Record() = default;
+  ~Millepede2Record();
   //
-  int getRun() const { return GetUniqueID(); }
-  void setRun(int r) { SetUniqueID(r); }
-  uint32_t getTimeStamp() const { return mTimeStamp; }
-  void setTimeStamp(uint32_t t) { mTimeStamp = t; }
-  uint32_t getTrackID() const { return mTrackID; }
-  void setTrackID(uint32_t t) { mTrackID = t; }
-  bool isCosmic() const { return TestBit(kCosmicBit); }
-  void setCosmic(bool v = true) { SetBit(kCosmicBit, v); }
+  int getRun() const { return mRunNumber; }
+  void setRun(int r) { mRunNumber = r; }
+  uint32_t getFirstTFOrbit() const { return mFirstTFOrbit; }
+  void setFirstTFOrbit(uint32_t v) { mFirstTFOrbit = v; }
+  o2::dataformats::GlobalTrackID getTrackID() const { return mTrackID; }
+  void setTrackID(o2::dataformats::GlobalTrackID t) { mTrackID = t; }
+  bool isCosmic() const { return testBit(CosmicBit); }
+  void setCosmic(bool v = true) { setBit(CosmicBit, v); }
   //
   int getNVarGlo() const { return mNVarGlo; }
   void setNVarGlo(int n) { mNVarGlo = n; }
@@ -65,51 +66,61 @@ class Millepede2Record : public TObject
   const int16_t* getArrLabLoc() const { return mIDLoc; }
   const int* getArrLabGlo() const { return mIDGlo; }
   //
-  bool fillTrack(AlignmentTrack* trc, const int* id2Lab = nullptr);
+  bool fillTrack(AlignmentTrack& trc, const std::vector& id2Lab);
   void dummyRecord(float res, float err, float dGlo, int labGlo);
   //
   void resize(int nresid, int nloc, int nglo);
   //
-  void Clear(const Option_t* opt = "") final;
-  void Print(const Option_t* opt = "") const final;
+  void clear();
+  void print() const;
   //
  protected:
   //
-  // ------- dummies --------
-  Millepede2Record(const Millepede2Record&);
-  Millepede2Record& operator=(const Millepede2Record&);
-  //
- protected:
-  //
-  uint32_t mTrackID;   // track in the event
-  uint32_t mTimeStamp; // event time stamp
-  int mNResid;         // number of residuals for the track (=2 npoints)
-  int mNVarLoc;        // number of local variables for the track
-  int mNVarGlo;        // number of global variables defined
-  int mNDLocTot;       // total number of non-zero local derivatives
-  int mNDGloTot;       // total number of non-zero global derivatives
-  int mNMeas;          // number of measured points
-  float mChi2Ini;      // chi2 of initial kalman fit
-  float mQ2Pt;         // q/pt at ref point
-  float mTgl;          // dip angle at ref point
-  //
-  int16_t* mNDLoc; //[mNResid] number of non-0 local derivatives per residual
-  int* mNDGlo;     //[mNResid] number of non-0 global derivatives per residual
-  int* mVolID;     //[mNResid] volume id + 1 (0 - not a volume)
-  float* mResid;   //[mNResid] residuals
-  float* mResErr;  //[mNResid] error associated to residual
-  //
-  int16_t* mIDLoc; //[mNDLocTot] ID of local variables for non-0 local derivatives
-  int* mIDGlo;     //[mNDGloTot] ID of global variables for non-0 global derivatives
-  float* mDLoc;    //[mNDLocTot] non-0 local derivatives
-  float* mDGlo;    //[mNDGloTot] non-0 global derivatives
+  uint16_t mBits = 0;
+  int mRunNumber = 0;
+  uint32_t mFirstTFOrbit = 0;                // event time stamp
+  o2::dataformats::GlobalTrackID mTrackID{}; // track in the event
+  int mNResid = 0;                           // number of residuals for the track (=2 npoints)
+  int mNVarLoc = 0;                          // number of local variables for the track
+  int mNVarGlo = 0;                          // number of global variables defined
+  int mNDLocTot = 0;                         // total number of non-zero local derivatives
+  int mNDGloTot = 0;                         // total number of non-zero global derivatives
+  int mNMeas = 0;                            // number of measured points
+  float mChi2Ini = 0;                        // chi2 of initial kalman fit
+  float mQ2Pt = 0;                           // q/pt at ref point
+  float mTgl = 0;                            // dip angle at ref point
+  //
+  int16_t* mMeasType = nullptr; //[mNResid] measurement type (0,1: Y, Z, 10+j : j-th material correction param)
+  int16_t* mNDLoc = nullptr; //[mNResid] number of non-0 local derivatives per residual
+  int* mNDGlo = nullptr;     //[mNResid] number of non-0 global derivatives per residual
+  int* mVolID = nullptr;     //[mNResid] volume id + 1 (0 - not a volume)
+  float* mResid = nullptr;   //[mNResid] residuals
+  float* mResErr = nullptr;  //[mNResid] error associated to residual
+  //
+  int16_t* mIDLoc = nullptr; //[mNDLocTot] ID of local variables for non-0 local derivatives
+  int* mIDGlo = nullptr;     //[mNDGloTot] ID of global variables for non-0 global derivatives
+  float* mDLoc = nullptr;    //[mNDLocTot] non-0 local derivatives
+  float* mDGlo = nullptr;    //[mNDGloTot] non-0 global derivatives
   //
   // aux info
-  int mNResidBook;   //! number of slots booked for residuals
-  int mNDLocTotBook; //! number of slots booked for local derivatives
-  int mNDGloTotBook; //! number of slots booked for global derivatives
-  //
-  ClassDef(Millepede2Record, 4);
+  int mNResidBook = 0;   //! number of slots booked for residuals
+  int mNDLocTotBook = 0; //! number of slots booked for local derivatives
+  int mNDGloTotBook = 0; //! number of slots booked for global derivatives
+  //
+  void setBit(uint16_t b, bool v)
+  {
+    if (v) {
+      mBits |= b;
+    } else {
+      mBits &= ~(b & 0xffff);
+    }
+  }
+  bool testBit(uint16_t b) const
+  {
+    return mBits & b;
+  }
+
+  ClassDefNV(Millepede2Record, 2);
 };
 } // namespace align
 } // namespace o2
diff --git a/Detectors/Align/include/Align/ResidualsController.h b/Detectors/Align/include/Align/ResidualsController.h
index a42c4333f4e25..8e54f54e0fcfa 100644
--- a/Detectors/Align/include/Align/ResidualsController.h
+++ b/Detectors/Align/include/Align/ResidualsController.h
@@ -17,8 +17,9 @@
 #ifndef RESIDUALSCONTROLLER_H
 #define RESIDUALSCONTROLLER_H
 
-#include 
+#include "ReconstructionDataFormats/GlobalTrackID.h"
 #include 
+#include 
 
 namespace o2
 {
@@ -27,38 +28,42 @@ namespace align
 
 class AlignmentTrack;
 
-class ResidualsController : public TObject
+class ResidualsController
 {
  public:
-  enum { kCosmicBit = BIT(14),
-         kVertexBit = BIT(15),
-         kKalmanDoneBit = BIT(16) };
-  //
-  ResidualsController();
-  ~ResidualsController() final;
-  //
-  void setRun(int r) { mRun = r; }
+  enum {
+    CosmicBit = 0x1,
+    VertexBit = 0x1 << 1,
+    KalmanBit = 0x1 << 2
+  };
+  //
+  ResidualsController() = default;
+  ~ResidualsController();
+  //
+  int getRun() const { return mRunNumber; }
+  void setRun(int r) { mRunNumber = r; }
+  uint32_t getFirstTFOrbit() const { return mFirstTFOrbit; }
+  void setFirstTFOrbit(uint32_t v) { mFirstTFOrbit = v; }
+  o2::dataformats::GlobalTrackID getTrackID() const { return mTrackID; }
+  void setTrackID(o2::dataformats::GlobalTrackID t) { mTrackID = t; }
   void setBz(float v) { mBz = v; }
-  void setTimeStamp(uint32_t v) { mTimeStamp = v; }
-  void setTrackID(uint32_t v) { mTrackID = v; }
+  float getBz() const { return mBz; }
+
   void setNPoints(int n)
   {
     mNPoints = n;
     resize(n);
   }
   //
-  bool isCosmic() const { return TestBit(kCosmicBit); }
-  bool hasVertex() const { return TestBit(kVertexBit); }
-  void setCosmic(bool v = kTRUE) { SetBit(kCosmicBit, v); }
-  void setHasVertex(bool v = kTRUE) { SetBit(kVertexBit, v); }
+  bool isCosmic() const { return testBit(CosmicBit); }
+  void setCosmic(bool v = true) { setBit(CosmicBit, v); }
+
+  bool hasVertex() const { return testBit(VertexBit); }
+  void setHasVertex(bool v = true) { setBit(VertexBit, v); }
   //
-  bool getKalmanDone() const { return TestBit(kKalmanDoneBit); }
-  void setKalmanDone(bool v = kTRUE) { SetBit(kKalmanDoneBit, v); }
+  bool getKalmanDone() const { return testBit(KalmanBit); }
+  void setKalmanDone(bool v = true) { setBit(KalmanBit, v); }
   //
-  int getRun() const { return mRun; }
-  float getBz() const { return mBz; }
-  uint32_t getTimeStamp() const { return mTimeStamp; }
-  uint32_t getTrackID() const { return mTrackID; }
   int getNPoints() const { return mNPoints; }
   int getNBook() const { return mNBook; }
   float getChi2() const { return mChi2; }
@@ -100,49 +105,57 @@ class ResidualsController : public TObject
   float getYLab(int i) const;
   float getZLab(int i) const;
   //
-  bool fillTrack(AlignmentTrack* trc, bool doKalman = kTRUE);
+  bool fillTrack(AlignmentTrack& trc, bool doKalman = kTRUE);
   void resize(int n);
-  void Clear(const Option_t* opt = "") final;
-  void Print(const Option_t* opt = "re") const final;
+  void clear();
+  void print(const Option_t* opt = "re") const;
   //
  protected:
   //
-  // -------- dummies --------
-  ResidualsController(const ResidualsController&);
-  ResidualsController& operator=(const ResidualsController&);
-  //
- protected:
-  //
-  int mRun;            // run
-  float mBz;           // field
-  uint32_t mTimeStamp; // event time
-  uint32_t mTrackID;   // track ID
-  int mNPoints;        // n meas points
-  int mNBook;          //! booked lenfth
-  float mChi2;         //  chi2 after solution
-  float mChi2Ini;      //  chi2 before solution
-  float mChi2K;        //  chi2 from kalman
-  float mQ2Pt;         //  Q/Pt at reference point
-  float* mX;           //[mNPoints] tracking X of cluster
-  float* mY;           //[mNPoints] tracking Y of cluster
-  float* mZ;           //[mNPoints] tracking Z of cluster
-  float* mSnp;         //[mNPoints] track Snp
-  float* mTgl;         //[mNPoints] track Tgl
-  float* mAlpha;       //[mNPoints] track alpha
-  float* mDY;          //[mNPoints] Y residual (track - meas)
-  float* mDZ;          //[mNPoints] Z residual (track - meas)
-  float* mDYK;         //[mNPoints] Y residual (track - meas) Kalman
-  float* mDZK;         //[mNPoints] Z residual (track - meas) Kalman
-  float* mSigY2;       //[mNPoints] Y err^2
-  float* mSigYZ;       //[mNPoints] YZ err
-  float* mSigZ2;       //[mNPoints] Z err^2
-  float* mSigY2K;      //[mNPoints] Y err^2 of Kalman track smoothing
-  float* mSigYZK;      //[mNPoints] YZ err  of Kalman track smoothing
-  float* mSigZ2K;      //[mNPoints] Z err^2 of Kalman track smoothing
-  int* mVolID;         //[mNPoints] volume id (0 for vertex constraint)
-  int* mLabel;         //[mNPoints] label of the volume
-  //
-  ClassDef(ResidualsController, 2);
+  uint16_t mBits = 0;
+  int mRunNumber = 0;                        // run
+  float mBz = 0;                             // field
+  uint32_t mFirstTFOrbit = 0;                // event time stamp
+  o2::dataformats::GlobalTrackID mTrackID{}; // track in the event
+  int mNPoints = 0;                          // n meas points
+  int mNBook = 0;                            //! booked length
+  float mChi2 = 0;                           //  chi2 after solution
+  float mChi2Ini = 0;                        //  chi2 before solution
+  float mChi2K = 0;                          //  chi2 from kalman
+  float mQ2Pt = 0;                           //  Q/Pt at reference point
+  float* mX = nullptr;                       //[mNPoints] tracking X of cluster
+  float* mY = nullptr;                       //[mNPoints] tracking Y of cluster
+  float* mZ = nullptr;                       //[mNPoints] tracking Z of cluster
+  float* mSnp = nullptr;                     //[mNPoints] track Snp
+  float* mTgl = nullptr;                     //[mNPoints] track Tgl
+  float* mAlpha = nullptr;                   //[mNPoints] track alpha
+  float* mDY = nullptr;                      //[mNPoints] Y residual (track - meas)
+  float* mDZ = nullptr;                      //[mNPoints] Z residual (track - meas)
+  float* mDYK = nullptr;                     //[mNPoints] Y residual (track - meas) Kalman
+  float* mDZK = nullptr;                     //[mNPoints] Z residual (track - meas) Kalman
+  float* mSigY2 = nullptr;                   //[mNPoints] Y err^2
+  float* mSigYZ = nullptr;                   //[mNPoints] YZ err
+  float* mSigZ2 = nullptr;                   //[mNPoints] Z err^2
+  float* mSigY2K = nullptr;                  //[mNPoints] Y err^2 of Kalman track smoothing
+  float* mSigYZK = nullptr;                  //[mNPoints] YZ err  of Kalman track smoothing
+  float* mSigZ2K = nullptr;                  //[mNPoints] Z err^2 of Kalman track smoothing
+  int* mVolID = nullptr;                     //[mNPoints] volume id (0 for vertex constraint)
+  int* mLabel = nullptr;                     //[mNPoints] label of the volume
+  //
+  void setBit(uint16_t b, bool v)
+  {
+    if (v) {
+      mBits |= b;
+    } else {
+      mBits &= ~(b & 0xffff);
+    }
+  }
+  bool testBit(uint16_t b) const
+  {
+    return mBits & b;
+  }
+
+  ClassDefNV(ResidualsController, 1);
 };
 } // namespace align
 } // namespace o2
diff --git a/Detectors/Align/include/Align/ResidualsControllerFast.h b/Detectors/Align/include/Align/ResidualsControllerFast.h
index 171bfb45757d2..5a2f947db9cca 100644
--- a/Detectors/Align/include/Align/ResidualsControllerFast.h
+++ b/Detectors/Align/include/Align/ResidualsControllerFast.h
@@ -24,7 +24,7 @@ namespace o2
 namespace align
 {
 
-class ResidualsControllerFast : public TObject
+class ResidualsControllerFast final : public TObject
 {
  public:
   enum { kCosmicBit = BIT(14),
diff --git a/Detectors/Align/include/Align/utils.h b/Detectors/Align/include/Align/utils.h
index 68d5f60faf65d..36589c4e5f460 100644
--- a/Detectors/Align/include/Align/utils.h
+++ b/Detectors/Align/include/Align/utils.h
@@ -46,119 +46,6 @@ enum { Coll,
        Cosm,
        NTrackTypes };
 
-//_________________________________________________________________________________
-inline constexpr double sectorDAlpha() noexcept
-{
-  return constants::math::PI / 9;
-};
-
-//_________________________________________________________________________________
-inline constexpr double sector2Alpha(int sect) noexcept
-{
-  // get barrel sector alpha in -pi:pi format
-  if (sect > 8) {
-    sect -= 18;
-  }
-  return (sect + 0.5) * sectorDAlpha();
-};
-
-//_________________________________________________________________________________
-inline int phi2Sector(double phi)
-{
-  // get barrel sector from phi in -pi:pi format
-  int sect = math_utils::nintd((phi * constants::math::Rad2Deg - 10) / 20.);
-  if (sect < 0) {
-    sect += 18;
-  }
-  return sect;
-};
-
-//_________________________________________________________________________________
-template 
-inline constexpr void bringTo02Pi(F& phi) noexcept
-{
-  // bring phi to 0-2pi range
-  if (phi < 0) {
-    phi += constants::math::TwoPI;
-  } else if (phi > constants::math::TwoPI) {
-    phi -= constants::math::TwoPI;
-  }
-};
-
-//_________________________________________________________________________________
-template 
-inline constexpr void bringToPiPM(F& phi) noexcept
-{
-  // bring phi to -pi:pi range
-  if (phi > constants::math::PI) {
-    phi -= constants::math::TwoPI;
-  }
-};
-
-//_________________________________________________________________________________
-template 
-inline constexpr bool okForPhiMin(F phiMin, F phi) noexcept
-{
-  // check if phi is above the phiMin, phi's must be in 0-2pi range
-  const F dphi = phi - phiMin;
-  if ((dphi > 0 && dphi < constants::math::PI) || dphi < -constants::math::PI) {
-    return true;
-  } else {
-    return false;
-  }
-};
-
-//_________________________________________________________________________________
-template 
-inline constexpr bool okForPhiMax(F phiMax, F phi) noexcept
-{
-  // check if phi is below the phiMax, phi's must be in 0-2pi range
-  const F dphi = phi - phiMax;
-  if ((dphi < 0 && dphi > -constants::math::PI) || dphi > constants::math::PI) {
-    return true;
-  } else {
-    return false;
-  }
-};
-
-//_________________________________________________________________________________
-template 
-constexpr F meanPhiSmall(F phi0, F phi1)
-{
-  // return mean phi, assume phis in 0:2pi
-  F phi;
-  if (!okForPhiMin(phi0, phi1)) {
-    phi = phi0;
-    phi0 = phi1;
-    phi1 = phi;
-  }
-  if (phi0 > phi1) {
-    phi = (phi1 - (constants::math::TwoPI - phi0)) / 2; // wrap
-  } else {
-    phi = (phi0 + phi1) / 2;
-  }
-  bringTo02Pi(phi);
-  return phi;
-};
-
-//_________________________________________________________________________________
-template 
-constexpr F deltaPhiSmall(F phi0, F phi1) noexcept
-{
-  // return delta phi, assume phi is in 0:2pi
-  F del;
-  if (!okForPhiMin(phi0, phi1)) {
-    del = phi0;
-    phi0 = phi1;
-    phi1 = del;
-  }
-  del = phi1 - phi0;
-  if (del < 0) {
-    del += constants::math::TwoPI;
-  }
-  return del;
-};
-
 //_________________________________________________________________________________
 template 
 inline constexpr bool smallerAbs(F d, F tolD) noexcept
diff --git a/Detectors/Align/macro/CMakeLists.txt b/Detectors/Align/macro/CMakeLists.txt
new file mode 100644
index 0000000000000..54c8715b074b2
--- /dev/null
+++ b/Detectors/Align/macro/CMakeLists.txt
@@ -0,0 +1,27 @@
+# Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+# See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+# All rights not expressly granted are reserved.
+#
+# This software is distributed under the terms of the GNU General Public
+# License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+#
+# In applying this license CERN does not waive the privileges and immunities
+# granted to it by virtue of its status as an Intergovernmental Organization
+# or submit itself to any jurisdiction.
+
+install(FILES algconf.C
+              algDump.C
+              MPRec2Mille.C
+        DESTINATION share/Detectors/Align/macro)
+
+o2_add_test_root_macro(algconf.C
+                       PUBLIC_LINK_LIBRARIES O2::Align
+                       LABELS align COMPILE_ONLY)
+
+o2_add_test_root_macro(algDump.C
+                       PUBLIC_LINK_LIBRARIES O2::Align
+                       LABELS align COMPILE_ONLY)
+
+o2_add_test_root_macro(MPRec2Mille.C
+                       PUBLIC_LINK_LIBRARIES O2::Align
+                       LABELS align COMPILE_ONLY)
diff --git a/Detectors/Align/macro/MPRec2Mille.C b/Detectors/Align/macro/MPRec2Mille.C
new file mode 100644
index 0000000000000..2af14e33ae77e
--- /dev/null
+++ b/Detectors/Align/macro/MPRec2Mille.C
@@ -0,0 +1,180 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#if !defined(__CINT__) || defined(__MAKECINT__)
+#include "Align/Millepede2Record.h"
+#include "Align/Mille.h"
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#endif
+
+using namespace o2::align;
+
+// convert MPRecord to Mille format
+const char* recBranchName = "mprec";
+const char* recTreeName = "mpTree";
+const char* defOutName = "mpData";
+const int defSplit = -200; // default output chunk size in MB
+
+TChain* loadMPrecChain(const char* inpData, const char* chName = recTreeName);
+int convertAndStore(Millepede2Record* rec, Mille* mille);
+bool processMPRec(Millepede2Record* rec);
+std::vector buffLoc;
+
+void MPRec2Mille(const char* inpName,              // name of MPRecord file or list of files
+                 const char* outname = defOutName, // out file name
+                 int split = defSplit              // 0: no split, >0: on N tracks,<0: on size in MB
+)
+{
+  TChain* mprChain = loadMPrecChain(inpName);
+  if (!mprChain)
+    return;
+  int nEnt = mprChain->GetEntries();
+  //
+  TBranch* br = mprChain->GetBranch(recBranchName);
+  if (!br) {
+    printf("provided tree does not contain branch mprec\n");
+    return;
+  }
+  //
+  Millepede2Record mp, *mprec = ∓
+  br->SetAddress(&mprec);
+  //
+  TString mln = outname;
+  if (mln.IsNull()) {
+    mln = inpName; // use inpname + ".mille"
+  }
+  if (mln.EndsWith(".mille") > 0) {
+    mln.Resize(mln.Last('.'));
+  }
+  printf(">>%s \n<<%s%s%s\n", inpName, mln.Data(), split ? "_XXX" : "", ".mille");
+  if (split) {
+    printf("Split on %d %s\n", TMath::Abs(split), split > 0 ? "tracks" : "MB");
+  }
+  //
+  TString milleName;
+  std::unique_ptr mille;
+  int cntTr = 0, cntTot = 0, cntMille = 0;
+  double sizeW = 0., sizeWTot = 0.;
+  if (split < 0) {
+    split *= 1000000;
+  }
+  for (int i = 0; i < nEnt; i++) {
+    mprChain->GetEntry(i);
+    //
+    if (!processMPRec(mprec))
+      continue; // preprocess and skip if needed
+    //
+    if (!mille || (split > 0 && ++cntTr > split) || (split < 0 && sizeW > -split)) { // start new mille file
+      cntTr = sizeW = 0;
+      milleName = split ? Form("%s_%03d.%s", mln.Data(), cntMille, "mille") : Form("%s.%s", mln.Data(), "mille");
+      cntMille++;
+      printf("Opening output file %s\n", milleName.Data());
+      mille = std::make_unique(milleName.Data());
+    }
+    cntTot++;
+    int nbwr = convertAndStore(mprec, mille.get());
+    sizeW += nbwr;
+    sizeWTot += nbwr;
+  }
+  mille.reset();
+  br->SetAddress(0);
+  delete mprChain;
+  //
+  printf("converted %d tracks out of %d\n(%ld MB in %d %s%s.mille files written)\n",
+         cntTot, nEnt, long(sizeWTot / 1e6), cntMille, mln.Data(), split ? "_XXX" : "");
+  //
+}
+
+//_________________________________________________________
+int convertAndStore(Millepede2Record* rec, Mille* mille)
+{
+  // convert and store the record
+  int nr = rec->getNResid(); // number of residual records
+  int nloc = rec->getNVarLoc();
+  const float* recDGlo = rec->getArrGlo();
+  const float* recDLoc = rec->getArrLoc();
+  const short* recLabLoc = rec->getArrLabLoc();
+  const int* recLabGlo = rec->getArrLabGlo();
+  //
+  for (int ir = 0; ir < nr; ir++) {
+    buffLoc.clear();
+    buffLoc.resize(nloc);
+    int ndglo = rec->getNDGlo(ir);
+    int ndloc = rec->getNDLoc(ir);
+    // fill 0-suppressed array from MPRecord to non-0-suppressed array of Mille
+    for (int l = ndloc; l--;) {
+      buffLoc[recLabLoc[l]] = recDLoc[l];
+    }
+    //
+    mille->mille(nloc, buffLoc.data(), ndglo, recDGlo, recLabGlo, rec->getResid(ir), rec->getResErr(ir));
+    //
+    recLabGlo += ndglo; // next record
+    recDGlo += ndglo;
+    recLabLoc += ndloc;
+    recDLoc += ndloc;
+  }
+  return mille->end(); // bytes written
+  //
+}
+
+//____________________________________________________________________
+TChain* loadMPrecChain(const char* inpData, const char* chName)
+{
+  TChain* chain = new TChain(chName);
+  //
+  TString inpDtStr = inpData;
+  if (inpDtStr.EndsWith(".root")) {
+    chain->AddFile(inpData);
+  } else {
+    //
+    ifstream inpf(inpData);
+    if (!inpf.good()) {
+      printf("Failed on input filename %s\n", inpData);
+      return 0;
+    }
+    //
+    TString flName;
+    flName.ReadLine(inpf);
+    while (!flName.IsNull()) {
+      flName = flName.Strip(TString::kBoth, ' ');
+      if (flName.BeginsWith("//") || flName.BeginsWith("#")) {
+        flName.ReadLine(inpf);
+        continue;
+      }
+      flName = flName.Strip(TString::kBoth, ',');
+      flName = flName.Strip(TString::kBoth, '"');
+      printf("Adding %s\n", flName.Data());
+      chain->AddFile(flName.Data());
+      flName.ReadLine(inpf);
+    }
+  }
+  //
+  int n = chain->GetEntries();
+  if (n < 1) {
+    printf("Obtained chain is empty\n");
+    return 0;
+  } else
+    printf("Opened %s chain with %d entries\n", chName, n);
+  return chain;
+}
+
+//_________________________________________________________
+bool processMPRec(Millepede2Record* rec)
+{
+  // put here user code
+  //
+  return true;
+}
diff --git a/Detectors/Align/macro/algDump.C b/Detectors/Align/macro/algDump.C
new file mode 100644
index 0000000000000..2d1dfcbaaa0de
--- /dev/null
+++ b/Detectors/Align/macro/algDump.C
@@ -0,0 +1,93 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#if !defined(__CLING__) || defined(__ROOTCLING__)
+#include "DetectorsBase/GeometryManager.h"
+#include "CommonUtils/TreeStreamRedirector.h"
+#include "ITSMFTReconstruction/ChipMappingITS.h"
+#include "DetectorsCommonDataFormats/DetID.h"
+#include "DataFormatsTRD/Constants.h"
+#include "TRDBase/Geometry.h"
+#include "TOFBase/Geo.h"
+#include 
+#endif
+
+using DetID = o2::detectors::DetID;
+
+void algDump(const std::string& geom = "", const std::string& outname = "algdump.root")
+{
+  o2::base::GeometryManager::loadGeometry(geom.c_str());
+  o2::utils::TreeStreamRedirector outstream(outname.c_str(), "recreate");
+  TGeoHMatrix* matAlg = nullptr;
+  TGeoHMatrix matOrig;
+  TVector3 pos0, pos;
+  DetID det;
+
+  auto procSens = [](TGeoHMatrix& mat) {
+    double loc[3] = {0., 0., 0.}, glo[3];
+    mat.LocalToMaster(loc, glo);
+    return TVector3(glo[0], glo[1], glo[2]);
+  };
+
+  auto store = [&outstream, &det, &pos, &pos0](int lr, int sid, int sidLr) {
+    outstream << "gm"
+              << "det=" << det.getID() << "lr=" << lr << "sid=" << sid << "sidlr=" << sidLr << "pos0=" << pos0 << "pos=" << pos << "\n";
+    printf("xx %d %d %d %f %f %f\n", det.getID(), lr, sid, pos0[0], pos0[1], pos0[1]);
+  };
+
+  det = DetID("ITS");
+  o2::itsmft::ChipMappingITS mpits;
+  for (int ic = 0; ic < mpits.getNChips(); ic++) {
+    int lr = mpits.getLayer(ic);
+    int ic0 = ic - mpits.getFirstChipsOnLayer(lr);
+    matAlg = o2::base::GeometryManager::getMatrix(det, ic);
+    o2::base::GeometryManager::getOriginalMatrix(det, ic, matOrig);
+    pos0 = procSens(matOrig);
+    pos = procSens(*matAlg);
+    store(lr, ic, ic0);
+  }
+
+  det = DetID("TRD");
+  for (int ilr = 0; ilr < o2::trd::constants::NLAYER; ilr++) {                                 // layer
+    for (int ich = 0; ich < o2::trd::constants::NSTACK * o2::trd::constants::NSECTOR; ich++) { // chamber
+      int isector = ich / o2::trd::constants::NSTACK;
+      int istack = ich % o2::trd::constants::NSTACK;
+      uint16_t sid = o2::trd::Geometry::getDetector(ilr, istack, isector);
+      const char* symname = Form("TRD/sm%02d/st%d/pl%d", isector, istack, ilr);
+      if (!gGeoManager->GetAlignableEntry(symname)) {
+        continue;
+      }
+      matAlg = o2::base::GeometryManager::getMatrix(det, sid);
+      o2::base::GeometryManager::getOriginalMatrix(det, sid, matOrig);
+      pos0 = procSens(matOrig);
+      pos = procSens(*matAlg);
+      store(ilr, sid, ich);
+    }
+  }
+
+  det = DetID("TOF");
+  int cnt = -1;
+  for (int isc = 0; isc < 18; isc++) {
+    for (int istr = 1; istr <= o2::tof::Geo::NSTRIPXSECTOR; istr++) { // strip
+      const char* symname = Form("TOF/sm%02d/strip%02d", isc, istr);
+      cnt++;
+      if (!gGeoManager->GetAlignableEntry(symname)) {
+        continue;
+      }
+      matAlg = o2::base::GeometryManager::getMatrix(det, cnt);
+      o2::base::GeometryManager::getOriginalMatrix(det, cnt, matOrig);
+      pos0 = procSens(matOrig);
+      pos = procSens(*matAlg);
+      store(0, cnt, cnt);
+    }
+  }
+  outstream.Close();
+}
diff --git a/Detectors/Align/macro/algconf.C b/Detectors/Align/macro/algconf.C
new file mode 100644
index 0000000000000..190c8739eaa34
--- /dev/null
+++ b/Detectors/Align/macro/algconf.C
@@ -0,0 +1,193 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#if !defined(__CLING__) || defined(__ROOTCLING__)
+#include "Align/Controller.h"
+#include "Align/AlignableVolume.h"
+#include "Align/AlignableDetectorITS.h"
+#include "Align/AlignableDetectorTPC.h"
+#include "Align/AlignableDetectorTRD.h"
+#include "Align/AlignableDetectorTOF.h"
+#endif
+#include "Framework/Logger.h"
+
+using namespace o2::align;
+
+void configITS(Controller* c, int par);
+void configTPC(Controller* c, int par);
+void configTRD(Controller* c, int par);
+void configTOF(Controller* c, int par);
+
+int algconf(Controller* c, int par)
+{
+  LOG(info) << "calling algconf with " << c << " " << par;
+
+  if (c->getDetector(o2::detectors::DetID::ITS)) {
+    configITS(c, par);
+  }
+  if (c->getDetector(o2::detectors::DetID::TPC)) {
+    configTPC(c, par);
+  }
+  if (c->getDetector(o2::detectors::DetID::TRD)) {
+    configTRD(c, par);
+  }
+  if (c->getDetector(o2::detectors::DetID::TOF)) {
+    configTOF(c, par);
+  }
+  c->Print("long");
+  LOG(info) << "user config done";
+  return 0;
+}
+
+/*
+  SymNames for volumes
+  ITS
+  ITS/ITSULayer/ITSUHalfBarrel,  X=0:6, Y=0,1
+  ITS/ITSULayer/ITSUHalfBarrel/ITSUStave
+  ITS/ITSULayer/ITSUHalfBarrel/ITSUStave/ITSUHalfStave, W=0,1 starting from L3
+  ITS/ITSULayer/ITSUHalfBarrel/ITSUStave/ITSUHalfStave/ITSUModule
+  ITS/ITSULayer/ITSUHalfBarrel/ITSUStave/ITSUChip for L0,1,2
+  ITS/ITSULayer/ITSUHalfBarrel/ITSUStave/ITSUHalfStave/ITSUModule/ITSUChip for L3-6
+
+  TRD
+  TRD/sm, S=00:17
+  TRD/sm/st/pl
, T=0-6 (stack)
+
+  TOF
+  TOF/sm, S=00:17
+  TOF/sm, T=00:91
+*/
+
+void configITS(Controller* c, int par)
+{
+  const double kCondSig[AlignableVolume::kNDOFGeom] = {0.2, 0.2, 0.3, 1., 1., 1.}; // precondition sigmas
+  AlignableDetectorITS* its = (AlignableDetectorITS*)c->getDetector(o2::detectors::DetID::ITS);
+  if (!its) {
+    LOG(warn) << "No ITS";
+    return;
+  }
+
+  auto volITS = its->getVolume("ITS"); // envelope volume
+
+  volITS->setChildrenConstrainPattern(AlignableVolume::kDOFBitTX | AlignableVolume::kDOFBitTY | AlignableVolume::kDOFBitTZ); // no auto constraint
+  volITS->setFreeDOFPattern(0);                                                                                              // fix
+
+  its->setFreeDOFPattern(0, -1, ".+ITSUChip[0-9]+$");      // Fix sensors
+  its->setFreeDOFPattern(0, -1, ".+ITSUModule[0-9]+$");    // Fix modules on OB
+  its->setFreeDOFPattern(0, -1, ".+ITSUHalfStave[0-9]+$"); // Fix halfstaves on OB
+  //  its->setFreeDOFPattern(0, -1, ".+ITSUStave[0-9]+$");   // Fix staves
+  //  its->setFreeDOFPattern(0, -1, ".+ITSUHalfBarrel[0-9]+$");   // Fix half-barrels
+
+  for (int iv = its->getNVolumes(); iv--;) {
+    auto vol = its->getVolume(iv);
+    for (int idf = AlignableVolume::kNDOFGeom; idf--;) {
+      if (std::abs(vol->getParErr(idf)) < 1e-6) { // there is not yet condition
+        vol->setParErr(idf, kCondSig[idf]);       // set condition
+      }
+    }
+    if (!vol->isSensor()) {
+      // prevent global shift of children in the containers
+      vol->setChildrenConstrainPattern(AlignableVolume::kDOFBitTX | AlignableVolume::kDOFBitTY | AlignableVolume::kDOFBitTZ);
+    }
+  }
+  /*
+  auto nvol = its->getNVolumes();
+  for (int i=0;igetVolume(i);
+    vol->Print();
+  }
+  */
+}
+
+void configTPC(Controller* c, int par)
+{
+  const double kCondSig[AlignableVolume::kNDOFGeom] = {1., 1., 1., 1., 1., 1.}; // precondition sigmas
+  AlignableDetectorTPC* tpc = (AlignableDetectorTPC*)c->getDetector(o2::detectors::DetID::TPC);
+  if (!tpc) {
+    LOG(info) << "No TPC";
+    return;
+  }
+
+  tpc->setFreeDOFPattern(0, -1, "TPC/sec[0-9]+$"); // Fix sectors
+  auto tpcvol = tpc->getVolume("TPC_envelope");    // envelope
+  if (!tpcvol || !tpcvol->isDummyEnvelope()) {
+    LOG(fatal) << "Could not find TPC_envelope volume";
+  }
+  for (int idf = AlignableVolume::kNDOFGeom; idf--;) {
+    tpcvol->setParErr(idf, kCondSig[idf]); // set condition
+  }
+}
+
+void configTRD(Controller* c, int par)
+{
+  const double kCondSig[AlignableVolume::kNDOFGeom] = {0.2, 0.2, 0.3, 1., 1., 1.}; // precondition sigmas
+  AlignableDetectorTRD* trd = (AlignableDetectorTRD*)c->getDetector(o2::detectors::DetID::TRD);
+  if (!trd) {
+    LOG(warn) << "No TRD";
+    return;
+  }
+
+  trd->setFreeDOFPattern(0, -1, ".+/pl[0-9]+$"); // Fix chambers
+  // trd->setFreeDOFPattern(0, -1, ".+/sm[0-9]+$");   // Fix sectors
+
+  for (int iv = trd->getNVolumes(); iv--;) {
+    auto vol = trd->getVolume(iv);
+    for (int idf = AlignableVolume::kNDOFGeom; idf--;) {
+      if (std::abs(vol->getParErr(idf)) < 1e-6) { // there is not yet condition
+        vol->setParErr(idf, kCondSig[idf]);       // set condition
+      }
+    }
+    if (!vol->isSensor()) {
+      // prevent global shift of children in the containers
+      vol->setChildrenConstrainPattern(AlignableVolume::kDOFBitTX | AlignableVolume::kDOFBitTY | AlignableVolume::kDOFBitTZ);
+    }
+  }
+  /*
+  auto nvol = trd->getNVolumes();
+  for (int i=0;igetVolume(i);
+    vol->Print();
+  }
+  */
+}
+
+void configTOF(Controller* c, int par)
+{
+  const double kCondSig[AlignableVolume::kNDOFGeom] = {0.2, 0.2, 0.3, 1., 1., 1.}; // precondition sigmas
+  AlignableDetectorTOF* tof = (AlignableDetectorTOF*)c->getDetector(o2::detectors::DetID::TOF);
+  if (!tof) {
+    LOG(warn) << "No TOF";
+    return;
+  }
+
+  tof->setFreeDOFPattern(0, -1, ".+/strip[0-9]+$"); // Fix strips
+  // tof->setFreeDOFPattern(0, -1, ".+/sm[0-9]+$");   // Fix sectors
+
+  for (int iv = tof->getNVolumes(); iv--;) {
+    auto vol = tof->getVolume(iv);
+    for (int idf = AlignableVolume::kNDOFGeom; idf--;) {
+      if (std::abs(vol->getParErr(idf)) < 1e-6) { // there is not yet condition
+        vol->setParErr(idf, kCondSig[idf]);       // set condition
+      }
+    }
+    if (!vol->isSensor()) {
+      // prevent global shift of children in the containers
+      vol->setChildrenConstrainPattern(AlignableVolume::kDOFBitTX | AlignableVolume::kDOFBitTY | AlignableVolume::kDOFBitTZ);
+    }
+  }
+  /*
+  auto nvol = tof->getNVolumes();
+  for (int i=0;igetVolume(i);
+    vol->Print();
+  }
+  */
+}
diff --git a/Detectors/Align/src/AlgPntDbg.cxx b/Detectors/Align/src/AlgPntDbg.cxx
new file mode 100644
index 0000000000000..b52492b3a5cc4
--- /dev/null
+++ b/Detectors/Align/src/AlgPntDbg.cxx
@@ -0,0 +1,22 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// @file   AlgPntDbg.h
+
+#include "Align/AlgPntDbg.h"
+
+namespace o2
+{
+namespace align
+{
+
+} // namespace align
+} // namespace o2
diff --git a/Detectors/Align/src/AlgTrcDbg.cxx b/Detectors/Align/src/AlgTrcDbg.cxx
new file mode 100644
index 0000000000000..157ca941449b7
--- /dev/null
+++ b/Detectors/Align/src/AlgTrcDbg.cxx
@@ -0,0 +1,23 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// @file   AlgTrcDbg.h
+
+#include "Align/AlgTrcDbg.h"
+#include "Framework/Logger.h"
+
+namespace o2
+{
+namespace align
+{
+
+} // namespace align
+} // namespace o2
diff --git a/Detectors/Align/src/AlignLinkDef.h b/Detectors/Align/src/AlignLinkDef.h
index d5c5747054862..66f3220d720ca 100644
--- a/Detectors/Align/src/AlignLinkDef.h
+++ b/Detectors/Align/src/AlignLinkDef.h
@@ -8,7 +8,6 @@
 // In applying this license CERN does not waive the privileges and immunities
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
-
 #ifdef __CLING__
 
 #pragma link off all globals;
@@ -18,18 +17,18 @@
 #pragma link C++ class o2::align::DOFSet + ;
 #pragma link C++ class o2::align::AlignableDetector + ;
 #pragma link C++ class o2::align::AlignableDetectorITS + ;
-//#pragma link C++ class o2::align::AlignableDetectorTOF + ;
-//#pragma link C++ class o2::align::AlignableDetectorTPC + ;
-//#pragma link C++ class o2::align::AlignableDetectorTRD + ;
-//#pragma link C++ class o2::align::AlignableDetectorHMPID + ;
+#pragma link C++ class o2::align::AlignableDetectorTOF + ;
+#pragma link C++ class o2::align::AlignableDetectorTPC + ;
+#pragma link C++ class o2::align::AlignableDetectorTRD + ;
+// #pragma link C++ class o2::align::AlignableDetectorHMPID + ;
 #pragma link C++ class o2::align::Millepede2Record + ;
 #pragma link C++ class o2::align::AlignmentPoint + ;
 #pragma link C++ class o2::align::AlignableSensor + ;
 #pragma link C++ class o2::align::AlignableSensorITS + ;
-//#pragma link C++ class o2::align::AlignableSensorTOF + ;
-//#pragma link C++ class o2::align::AlignableSensorTPC + ;
-//#pragma link C++ class o2::align::AlignableSensorTRD + ;
-//#pragma link C++ class o2::align::AlignableSensorHMPID + ;
+#pragma link C++ class o2::align::AlignableSensorTOF + ;
+#pragma link C++ class o2::align::AlignableSensorTPC + ;
+#pragma link C++ class o2::align::AlignableSensorTRD + ;
+// #pragma link C++ class o2::align::AlignableSensorHMPID + ;
 #pragma link C++ class o2::align::Controller + ;
 #pragma link C++ class o2::align::AlignmentTrack + ;
 #pragma link C++ class o2::align::AlignableVolume + ;
@@ -37,10 +36,18 @@
 #pragma link C++ class o2::align::ResidualsController + ;
 #pragma link C++ class o2::align::ResidualsControllerFast + ;
 #pragma link C++ class o2::align::GeometricalConstraint + ;
-#pragma link C++ class o2::align::DOFStatistics + ;
 #pragma link C++ class o2::align::utils;
 
+#pragma link C++ class o2::align::AlgPntDbg + ;
+#pragma link C++ class o2::align::AlgTrcDbg + ;
+#pragma link C++ class std::vector < o2::align::AlgPntDbg> + ;
+
 #pragma link C++ class o2::align::AlignConfig + ;
 #pragma link C++ class o2::conf::ConfigurableParamHelper < o2::align::AlignConfig> + ;
 
+#pragma link C++ class std::vector < o2::track::TrackParametrization < double>> + ;
+
+#include "TVector3.h"
+#pragma link C++ class std::vector < TVector3> + ;
+
 #endif
diff --git a/Detectors/Align/src/AlignableDetector.cxx b/Detectors/Align/src/AlignableDetector.cxx
index 50a09d22e0094..68e602c85251d 100644
--- a/Detectors/Align/src/AlignableDetector.cxx
+++ b/Detectors/Align/src/AlignableDetector.cxx
@@ -19,23 +19,16 @@
 #include "Align/AlignableSensor.h"
 #include "Align/Controller.h"
 #include "Align/AlignmentTrack.h"
-#include "Align/DOFStatistics.h"
 #include "Align/GeometricalConstraint.h"
 #include "DetectorsBase/GRPGeomHelper.h"
+#include "CommonUtils/NameConf.h"
 #include "Framework/Logger.h"
-//#include "AliGeomManager.h"
-//#include "AliCDBManager.h"
-//#include "AliCDBMetaData.h"
-//#include "AliCDBEntry.h"
-//#include "AliAlignObj.h"
-//#include "AliCDBId.h"
-//#include "AliExternalTrackParam.h"
-//#include "AliAlignObjParams.h"
 #include 
 #include 
 #include 
 #include 
 #include 
+#include 
 
 ClassImp(o2::align::AlignableDetector);
 
@@ -49,6 +42,8 @@ namespace align
 //____________________________________________
 AlignableDetector::AlignableDetector(DetID id, Controller* ctr) : DOFSet(id.getName(), ctr), mDetID(id)
 {
+  mVolumes.SetOwner(true);
+  mSensors.SetOwner(false); // sensors are just pointers on particular volumes
 }
 
 //____________________________________________
@@ -60,7 +55,7 @@ AlignableDetector::~AlignableDetector()
 }
 
 //____________________________________________
-int AlignableDetector::processPoints(GIndex gid, bool inv)
+int AlignableDetector::processPoints(GIndex gid, int npntCut, bool inv)
 {
   // Create alignment points corresponding to this detector, recalibrate/realign them to the
   // level of the "starting point" for the alignment/calibration session.
@@ -121,7 +116,7 @@ void AlignableDetector::updateL2GRecoMatrices()
 void AlignableDetector::reset()
 {
   // prepare for the next track processing
-  mNPoints = mFirstPoint = 0;
+  mNPoints = 0;
 }
 
 //_________________________________________________________
@@ -182,11 +177,11 @@ void AlignableDetector::defineMatrices()
   TIter next(&mVolumes);
   AlignableVolume* vol(nullptr);
   while ((vol = (AlignableVolume*)next())) {
-    // modified global-local matrix
-    vol->prepareMatrixL2G();
-    // ideal global-local matrix
-    vol->prepareMatrixL2GIdeal();
-    //
+    if (vol->isDummy() || vol->isDummyEnvelope()) {
+      continue;
+    }
+    vol->prepareMatrixL2G();      // modified global-local matrix
+    vol->prepareMatrixL2GIdeal(); // ideal global-local matrix
   }
   // Now set tracking-local matrix (MUST be done after ALL L2G matrices are done!)
   // Attention: for sensor it is a real tracking matrix extracted from
@@ -195,6 +190,9 @@ void AlignableDetector::defineMatrices()
   // see its definition in the AlignableVolume::PrepateMatrixT2L
   next.Reset();
   while ((vol = (AlignableVolume*)next())) {
+    if (vol->isDummy()) {
+      continue;
+    }
     vol->prepareMatrixT2L();
     if (vol->isSensor()) {
       ((AlignableSensor*)vol)->prepareMatrixClAlg();
@@ -338,10 +336,7 @@ void AlignableDetector::Print(const Option_t* opt) const
   printf("Obligatory in Collisions: %7s | Cosmic: %7s\n",
          isObligatory(Coll) ? " YES " : "  NO ", isObligatory(Cosm) ? " YES " : "  NO ");
   //
-  fmt::printf("Sel. flags in Collisions: {:05#x}%05 | Cosmic: 0x{:05#x}%05\n", mTrackFlagSel[Coll], mTrackFlagSel[Cosm]);
-  //
-  printf("Min.points in Collisions: %7d | Cosmic: %7d\n",
-         mNPointsSel[Coll], mNPointsSel[Cosm]);
+  printf("Min.points in Collisions: %7d | Cosmic: %7d\n", mNPointsSel[Coll], mNPointsSel[Cosm]);
   //
   if (!(IsDisabledColl() && IsDisabledCosm()) && opts.Contains("long")) {
     for (int iv = 0; iv < getNVolumes(); iv++) {
@@ -414,6 +409,23 @@ void AlignableDetector::writePedeInfo(FILE* parOut, const Option_t* opt) const
   //
 }
 
+//______________________________________________________
+void AlignableDetector::writeLabeledPedeResults(FILE* parOut) const
+{
+  // contribute to params and constraints template files for PEDE
+  fprintf(parOut, "\n!!\t\tDetector:\t%s\tNDOFs: %d\n", mDetID.getName(), getNDOFs());
+  //
+  // parameters
+  int nvol = getNVolumes();
+  for (int iv = 0; iv < nvol; iv++) { // call for root level volumes, they will take care of their children
+    AlignableVolume* vol = getVolume(iv);
+    if (!vol->getParent()) {
+      vol->writeLabeledPedeResults(parOut);
+    }
+  }
+  //
+}
+
 //______________________________________________________
 void AlignableDetector::writeCalibrationResults() const
 {
@@ -426,28 +438,19 @@ void AlignableDetector::writeCalibrationResults() const
 //______________________________________________________
 void AlignableDetector::writeAlignmentResults() const
 {
-  LOG(fatal) << __PRETTY_FUNCTION__ << " is disabled";
-  //FIXME(lettrich): needs OCDB
-  //  // store updated alignment
-  //  TClonesArray* arr = new TClonesArray("AliAlignObjParams", 10);
-  //  //
-  //  int nvol = getNVolumes();
-  //  for (int iv = 0; iv < nvol; iv++) {
-  //    AlignableVolume* vol = getVolume(iv);
-  //    // call only for top level objects, they will take care of children
-  //    if (!vol->getParent()){
-  //      vol->createAlignmentObjects(arr);}
-  //  }
-  //  //
-  //  AliCDBManager* man = AliCDBManager::Instance();
-  //  AliCDBMetaData* md = new AliCDBMetaData();
-  //  md->SetResponsible(mController->getOutCDBResponsible());
-  //  md->SetComment(mController->getOutCDBResponsible());
-  //  //
-  //  AliCDBId id(Form("%s/Align/Data", mDetID.getName()), mController->getOutCDBRunMin(), mController->getOutCDBRunMax());
-  //  man->Put(arr, id, md);
-  //  //
-  //  delete arr;
+  std::vector arr;
+  int nvol = getNVolumes();
+  for (int iv = 0; iv < nvol; iv++) {
+    AlignableVolume* vol = getVolume(iv);
+    // call only for top level objects, they will take care of children
+    if (!vol->getParent()) {
+      vol->createAlignmentObjects(arr);
+    }
+  }
+  TFile outalg(fmt::format("alignment{}.root", getName()).c_str(), "recreate");
+  outalg.WriteObjectAny(&arr, "std::vector", o2::base::NameConf::CCDBOBJECT.data());
+  outalg.Close();
+  LOGP(info, "storing {} alignment in {}", getName(), outalg.GetName());
 }
 
 //______________________________________________________
@@ -491,15 +494,13 @@ void AlignableDetector::terminate()
   //  if (isDisabled()) return;
   int nvol = getNVolumes();
   mNProcPoints = 0;
-  auto& st = mController->GetDOFStat();
   for (int iv = 0; iv < nvol; iv++) {
     AlignableVolume* vol = getVolume(iv);
     // call init for root level volumes, they will take care of their children
     if (!vol->getParent()) {
-      mNProcPoints += vol->finalizeStat(st);
+      mNProcPoints += vol->finalizeStat();
     }
   }
-  fillDOFStat(st); // fill stat for calib dofs
 }
 
 //________________________________________
@@ -510,7 +511,7 @@ void AlignableDetector::addAutoConstraints() const
   for (int iv = 0; iv < nvol; iv++) { // call for root level volumes, they will take care of their children
     AlignableVolume* vol = getVolume(iv);
     if (!vol->getParent()) {
-      vol->addAutoConstraints((TObjArray*)mController->getConstraints());
+      vol->addAutoConstraints();
     }
   }
 }
@@ -530,24 +531,21 @@ void AlignableDetector::fixNonSensors()
 }
 
 //________________________________________
-int AlignableDetector::selectVolumes(TObjArray* arr, int lev, const char* match)
+int AlignableDetector::selectVolumes(std::vector cont, int lev, const std::string& regexStr)
 {
   // select volumes matching to pattern and/or hierarchy level
   //
-  if (!arr) {
-    return 0;
-  }
+  std::regex selRegEx(regexStr);
   int nadd = 0;
-  TString mts = match, syms;
   for (int i = getNVolumes(); i--;) {
     AlignableVolume* vol = getVolume(i);
     if (lev >= 0 && vol->countParents() != lev) {
       continue;
     } // wrong level
-    if (!mts.IsNull() && !(syms = vol->getSymName()).Contains(mts)) {
+    if (!regexStr.empty() && !std::regex_match(vol->getSymName(), selRegEx)) {
       continue;
-    } //wrong name
-    arr->AddLast(vol);
+    }
+    cont.push_back(vol);
     nadd++;
   }
   //
@@ -555,40 +553,39 @@ int AlignableDetector::selectVolumes(TObjArray* arr, int lev, const char* match)
 }
 
 //________________________________________
-void AlignableDetector::setFreeDOFPattern(uint32_t pat, int lev, const char* match)
+void AlignableDetector::setFreeDOFPattern(uint32_t pat, int lev, const std::string& regexStr)
 {
-  // set free DOFs to volumes matching either to hierarchy level or
-  // whose name contains match
+  // set free DOFs to volumes matching either to hierarchy level or whose name contains match
   //
-  TString mts = match, syms;
+  std::regex selRegEx(regexStr);
   for (int i = getNVolumes(); i--;) {
     AlignableVolume* vol = getVolume(i);
     if (lev >= 0 && vol->countParents() != lev) {
       continue;
     } // wrong level
-    if (!mts.IsNull() && !(syms = vol->getSymName()).Contains(mts)) {
+    if (!regexStr.empty() && !std::regex_match(vol->getSymName(), selRegEx)) {
       continue;
-    } //wrong name
+    } // wrong name
     vol->setFreeDOFPattern(pat);
   }
   //
 }
 
 //________________________________________
-void AlignableDetector::setDOFCondition(int dof, float condErr, int lev, const char* match)
+void AlignableDetector::setDOFCondition(int dof, float condErr, int lev, const std::string& regexStr)
 {
   // set condition for DOF of volumes matching either to hierarchy level or
   // whose name contains match
   //
-  TString mts = match, syms;
+  std::regex selRegEx(regexStr);
   for (int i = getNVolumes(); i--;) {
     AlignableVolume* vol = getVolume(i);
     if (lev >= 0 && vol->countParents() != lev) {
       continue;
     } // wrong level
-    if (!mts.IsNull() && !(syms = vol->getSymName()).Contains(mts)) {
+    if (!regexStr.empty() && !std::regex_match(vol->getSymName(), selRegEx)) {
       continue;
-    } //wrong name
+    } // wrong name
     if (dof >= vol->getNDOFs()) {
       continue;
     }
@@ -610,14 +607,14 @@ void AlignableDetector::constrainOrphans(const double* sigma, const char* match)
   // sigma>0 : dof constrained by gaussian constraint
   //
   TString mts = match, syms;
-  GeometricalConstraint* constr = new GeometricalConstraint();
+  auto cstr = getController()->getConstraints().emplace_back();
   for (int i = 0; i < AlignableVolume::kNDOFGeom; i++) {
     if (sigma[i] >= 0) {
-      constr->constrainDOF(i);
+      cstr.constrainDOF(i);
     } else {
-      constr->unConstrainDOF(i);
+      cstr.unConstrainDOF(i);
     }
-    constr->setSigma(i, sigma[i]);
+    cstr.setSigma(i, sigma[i]);
   }
   for (int i = getNVolumes(); i--;) {
     AlignableVolume* vol = getVolume(i);
@@ -627,14 +624,12 @@ void AlignableDetector::constrainOrphans(const double* sigma, const char* match)
     if (!mts.IsNull() && !(syms = vol->getSymName()).Contains(mts)) {
       continue;
     } //wrong name
-    constr->addChild(vol);
+    cstr.addChild(vol);
   }
   //
-  if (!constr->getNChildren()) {
+  if (!cstr.getNChildren()) {
     LOG(info) << "No volume passed filter " << match;
-    delete constr;
-  } else {
-    ((TObjArray*)mController->getConstraints())->Add(constr);
+    getController()->getConstraints().pop_back();
   }
 }
 
@@ -674,7 +669,7 @@ void AlignableDetector::calcFree(bool condFix)
   mNCalibDOFsFree = 0;
   for (int i = 0; i < mNCalibDOFs; i++) {
     if (!isFreeDOF(i)) {
-      if (condFix) {
+      if (condFix && varsSet()) {
         setParErr(i, -999);
       }
       continue;
@@ -684,20 +679,6 @@ void AlignableDetector::calcFree(bool condFix)
   //
 }
 
-//______________________________________________________
-void AlignableDetector::fillDOFStat(DOFStatistics& st) const
-{
-  // fill statistics info hist
-  int ndf = getNCalibDOFs();
-  int dof0 = getFirstParGloID();
-  int stat = getNProcessedPoints();
-  for (int idf = 0; idf < ndf; idf++) {
-    int dof = idf + dof0;
-    st.addStat(dof, stat);
-  }
-  //
-}
-
 //______________________________________________________
 void AlignableDetector::writeSensorPositions(const char* outFName)
 {
diff --git a/Detectors/Align/src/AlignableDetectorITS.cxx b/Detectors/Align/src/AlignableDetectorITS.cxx
index 0a984b6f225ae..7387ae4620bf3 100644
--- a/Detectors/Align/src/AlignableDetectorITS.cxx
+++ b/Detectors/Align/src/AlignableDetectorITS.cxx
@@ -18,8 +18,10 @@
 #include "Align/AlignableVolume.h"
 #include "Align/AlignableSensorITS.h"
 #include "Align/Controller.h"
+#include "Align/AlignConfig.h"
 #include "ITSBase/GeometryTGeo.h"
 #include "DataFormatsITSMFT/TopologyDictionary.h"
+#include "DataFormatsITSMFT/TrkClusRef.h"
 #include "DataFormatsITS/TrackITS.h"
 #include "DataFormatsGlobalTracking/RecoContainer.h"
 #include "ITStracking/IOUtils.h"
@@ -34,17 +36,14 @@ namespace o2
 namespace align
 {
 
-const char* AlignableDetectorITS::fgkHitsSel[AlignableDetectorITS::kNSPDSelTypes] =
-  {"SPDNoSel", "SPDBoth", "SPDAny", "SPD0", "SPD1"};
-
 //____________________________________________
 AlignableDetectorITS::AlignableDetectorITS(Controller* ctr) : AlignableDetector(DetID::ITS, ctr)
 {
   // default c-tor
-  setUseErrorParam();
-  SetITSSelPatternColl();
-  SetITSSelPatternCosm();
+  o2::itsmft::ChipMappingITS mp;
+  mOverlaps = mp.getOverlapsInfo();
 }
+
 /*
 //____________________________________________
 void AlignableDetectorITS::initGeom()
@@ -100,7 +99,7 @@ void AlignableDetectorITS::defineVolumes()
     if (ich != chID) {
       throw std::runtime_error(fmt::format("mismatch between counter {} and composed {} chip IDs", ich, chID));
     }
-    addVolume(sens = new AlignableSensorITS(o2::base::GeometryManager::getSymbolicName(mDetID, ich), chID, getSensLabel(chID), mController));
+    addVolume(sens = new AlignableSensorITS(o2::base::GeometryManager::getSymbolicName(mDetID, ich), chID, getSensLabel(ich), mController));
     int lay = 0, hba, sta = 0, ssta = 0, modd = 0, chip = 0;
     geom->getChipId(chID, lay, hba, sta, ssta, modd, chip);
     AlignableVolume* parVol = sym2vol[modd < 0 ? geom->composeSymNameStave(lay, hba, sta) : geom->composeSymNameModule(lay, hba, sta, ssta, modd)];
@@ -113,7 +112,7 @@ void AlignableDetectorITS::defineVolumes()
 }
 
 //____________________________________________
-int AlignableDetectorITS::processPoints(GIndex gid, bool inv)
+int AlignableDetectorITS::processPoints(GIndex gid, int npntCut, bool inv)
 {
   // Extract the points corresponding to this detector, recalibrate/realign them to the
   // level of the "starting point" for the alignment/calibration session.
@@ -122,70 +121,260 @@ int AlignableDetectorITS::processPoints(GIndex gid, bool inv)
   //
   auto algTrack = mController->getAlgTrack();
   auto recoData = mController->getRecoContainer();
-  const auto tracks = recoData->getITSTracks();
-  if (tracks.empty()) {
-    return -1; // source not loaded?
-  }
-  const auto& track = tracks[gid.getIndex()];
-  const auto& clusIdx = recoData->getITSTracksClusterRefs();
-  // do we want to apply some cuts?
-  int clEntry = track.getFirstClusterEntry();
-  mNPoints = 0;
-  mFirstPoint = algTrack->getNPoints();
-  for (int icl = track.getNumberOfClusters(); icl--;) {
-    const auto& clus = mITSClustersArray[clusIdx[clEntry++]];
-    auto* sensor = getSensor(clus.getSensorID());
+  const auto& algConf = AlignConfig::Instance();
+  int npoints = 0;
+  auto procClus = [this, inv, &npoints, &algTrack](const ClusterD& clus) {
+    auto* sensor = this->getSensor(clus.getSensorID());
     auto& pnt = algTrack->addDetectorPoint();
-
-    if (!getUseErrorParam()) {
-      const auto* sysE = sensor->getAddError(); // additional syst error
-      pnt.setYZErrTracking(clus.getSigmaY2() + sysE[0] * sysE[0], clus.getSigmaYZ(), clus.getSigmaZ2() + sysE[1] * sysE[1]);
-    } else { // errors will be calculated just before using the point in the fit, using track info
-      pnt.setYZErrTracking(0., 0., 0.);
+    const auto* sysE = sensor->getAddError(); // additional syst error
+    pnt.setYZErrTracking(clus.getSigmaY2() + sysE[0] * sysE[0], clus.getSigmaYZ(), clus.getSigmaZ2() + sysE[1] * sysE[1]);
+    if (this->getUseErrorParam()) { // errors will be calculated just before using the point in the fit, using track info
       pnt.setNeedUpdateFromTrack();
     }
     pnt.setXYZTracking(clus.getX(), clus.getY(), clus.getZ());
     pnt.setSensor(sensor);
     pnt.setAlphaSens(sensor->getAlpTracking());
     pnt.setXSens(sensor->getXTracking());
-    pnt.setDetID(mDetID);
+    pnt.setDetID(this->mDetID);
     pnt.setSID(sensor->getSID());
-    //
     pnt.setContainsMeasurement();
+    pnt.setInvDir(inv);
     pnt.init();
-    mNPoints++;
+    npoints++;
+  };
+  std::array clusIDs{};
+  int nOverlaps = 0;
+  if (gid.getSource() == GIndex::ITS) {
+    const auto tracks = recoData->getITSTracks();
+    if (tracks.empty()) {
+      return -1; // source not loaded?
+    }
+    const auto& track = tracks[gid.getIndex()];
+    if (track.getNClusters() < npntCut) {
+      return -1;
+    }
+    const auto& clusIdx = recoData->getITSTracksClusterRefs();
+    // do we want to apply some cuts?
+    int clEntry = track.getFirstClusterEntry();
+    int preevSensID = -1;
+    bool errReported = false;
+    for (int icl = track.getNumberOfClusters(); icl--;) { // clusters refs are stored from outer to inner layers, we loop in inner -> outer direction
+      const auto& clus = mITSClustersArray[(clusIDs[npoints] = clusIdx[clEntry + icl])];
+      if (clus.getSensorID() < preevSensID && !errReported) { // clusters are ordered from outer to inner layer, hence decreasing sensorID
+        std::string errstr{};
+        for (int ie = track.getNumberOfClusters(); ie--;) {
+          errstr += fmt::format(" {}", mITSClustersArray[clusIdx[clEntry + ie]].getSensorID());
+        }
+        LOGP(error, "wrong ITS clusters order? : chips {}", errstr);
+        errReported = true;
+      }
+      preevSensID = clus.getSensorID();
+      if (clus.getBits()) { // overlapping clusters will have bit set
+        if (clus.isBitSet(EdgeFlags::Biased)) {
+          continue;
+        }
+        if (clus.getCount()) {
+          nOverlaps++;
+        }
+      }
+      procClus(clus);
+    }
+  } else { // ITSAB
+    const auto& trkITSABref = recoData->getITSABRefs()[gid.getIndex()];
+    const auto& ABTrackClusIdx = recoData->getITSABClusterRefs();
+    int nCl = trkITSABref.getNClusters();
+    int clEntry = trkITSABref.getFirstEntry();
+    for (int icl = 0; icl < nCl; icl++) { // clusters are stored from inner to outer layers
+      const auto& clus = mITSClustersArray[(clusIDs[npoints] = ABTrackClusIdx[clEntry + icl])];
+      if (clus.getBits()) { // overlapping clusters will have bit set
+        if (clus.isBitSet(EdgeFlags::Biased)) {
+          continue;
+        }
+        if (clus.getCount()) {
+          nOverlaps++;
+        }
+      }
+      procClus(clus);
+    }
+  }
+  if (npoints < npntCut) { // reset points to original start
+    algTrack->suppressLastPoints(npoints);
+    npoints = 0;
+    return 0;
+  }
+
+  // do we need to process overlaps?
+  if (nOverlaps) {
+    auto trcProp = mController->getRecoContainer()->getTrackParam(mController->getAlgTrack()->getCurrentTrackID());
+    trcProp.resetCovariance(10);
+    auto propagator = o2::base::Propagator::Instance();
+    for (int icl = 0; icl < npoints; icl++) {
+      const auto& clus = mITSClustersArray[clusIDs[icl]];
+      float alp = getSensor(clus.getSensorID())->getAlpTracking();
+      if (!trcProp.rotate(alp) ||
+          !propagator->propagateToX(trcProp, clus.getX(), propagator->getNominalBz(), algConf.maxSnp, algConf.maxStep, base::Propagator::MatCorrType(algConf.matCorType)) ||
+          !trcProp.update(clus)) {
+        break;
+      }
+
+      if (clus.getCount()) { // there is an overlap, find best matching cluster
+        nOverlaps--;
+        int bestClusID = -1, clusIDtoCheck = mOverlapClusRef[clusIDs[icl]];
+        float bestChi2 = algConf.ITSOverlapMaxChi2;
+        auto trPropOvl = trcProp;
+        for (int iov = 0; iov < clus.getCount(); iov++) {
+          int clusOvlID = mOverlapCandidateID[clusIDtoCheck];
+          const auto& clusOvl = mITSClustersArray[clusOvlID];
+          if (iov == 0) {
+            if (!trPropOvl.rotate(getSensor(clusOvl.getSensorID())->getAlpTracking()) ||
+                !propagator->propagateToX(trPropOvl, clusOvl.getX(), propagator->getNominalBz(), algConf.maxSnp, algConf.maxStep, base::Propagator::MatCorrType::USEMatCorrNONE)) {
+              break;
+            }
+          }
+          auto chi2 = trPropOvl.getPredictedChi2(clusOvl);
+          if (chi2 < bestChi2) {
+            bestChi2 = chi2;
+            bestClusID = clusOvlID;
+          }
+        }
+        if (bestClusID != -1) { // account overlapping cluster
+          procClus(mITSClustersArray[bestClusID]);
+        }
+      }
+      if (!nOverlaps) {
+        break;
+      }
+    }
   }
-  return track.getNumberOfClusters(); // RS
+  mNPoints += npoints;
+  return npoints;
 }
 
 //____________________________________________
 bool AlignableDetectorITS::prepareDetectorData()
 {
   // prepare TF data for processing: convert clusters
+  const auto& algConf = AlignConfig::Instance();
   auto recoData = mController->getRecoContainer();
   const auto clusITS = recoData->getITSClusters();
+  const auto clusITSROF = recoData->getITSClustersROFRecords();
   const auto patterns = recoData->getITSClustersPatterns();
   auto pattIt = patterns.begin();
+  mITSClustersArray.clear();
   mITSClustersArray.reserve(clusITS.size());
+  if (algConf.ITSOverlapMargin > 0) {
+    mOverlapClusRef.clear();
+    mOverlapClusRef.resize(clusITS.size(), -1);
 
-  for (auto& c : clusITS) {
-    auto* sensor = getSensor(c.getSensorID());
-    double sigmaY2, sigmaZ2, sigmaYZ = 0, locXYZC[3], traXYZ[3];
-    auto locXYZ = o2::its::ioutils::extractClusterDataA(c, pattIt, mITSDict, sigmaY2, sigmaZ2); // local ideal coordinates
-    const auto& matAlg = sensor->getMatrixClAlg();                                              // local alignment matrix !!! RS FIXME
-    matAlg.LocalToMaster(locXYZ.data(), locXYZC);                                               // aligned point in the local fram
-    const auto& mat = sensor->getMatrixT2L();                                                   // RS FIXME check if correct
-    mat.MasterToLocal(locXYZC, traXYZ);
-    /*
-    if (applyMisalignment) {
-      auto lrID = chmap.getLayer(c.getSensorID());
-      sigmaY2 += conf.sysErrY2[lrID];
-      sigmaZ2 += conf.sysErrZ2[lrID];
-    }
-    */
-    auto& cl3d = mITSClustersArray.emplace_back(c.getSensorID(), traXYZ[0], traXYZ[1], traXYZ[2], sigmaY2, sigmaZ2, sigmaYZ); // local --> tracking
+    mOverlapCandidateID.clear();
+    mOverlapCandidateID.reserve(clusITS.size());
   }
+  static std::vector edgeClusters;
+  int ROFCount = 0;
+  int16_t curSensID = -1;
+  struct ROFChipEntry {
+    int16_t rofCount = -1;
+    int chipFirstEntry = -1;
+  };
+  std::array chipROFStart{}; // fill only for clusters with overlaps
+
+  for (const auto& rof : clusITSROF) {
+    int maxic = rof.getFirstEntry() + rof.getNEntries();
+    edgeClusters.clear();
+    for (int ic = rof.getFirstEntry(); ic < maxic; ic++) {
+      const auto& c = clusITS[ic];
+      int16_t sensID = c.getSensorID();
+      auto* sensor = getSensor(sensID);
+      double sigmaY2, sigmaZ2, sigmaYZ = 0, locXYZC[3], traXYZ[3];
+      auto pattItCopy = pattIt;
+      auto locXYZ = o2::its::ioutils::extractClusterDataA(c, pattIt, mITSDict, sigmaY2, sigmaZ2); // local ideal coordinates
+      const auto& matAlg = sensor->getMatrixClAlg();                                              // local alignment matrix !!! RS FIXME
+      matAlg.LocalToMaster(locXYZ.data(), locXYZC);                                               // aligned point in the local frame
+      const auto& mat = sensor->getMatrixT2L();                                                   // RS FIXME check if correct
+      mat.MasterToLocal(locXYZC, traXYZ);
+      auto& cl3d = mITSClustersArray.emplace_back(sensID, traXYZ[0], traXYZ[1], traXYZ[2], sigmaY2, sigmaZ2, sigmaYZ); // local --> tracking
+
+      if (algConf.ITSOverlapMargin > 0) {
+        // fill chips overlaps info for clusters whose center is within of the algConf.ITSOverlapMargin distance from the chip min or max row edge
+        // but the pixel closest to this edge has distance of at least algConf.ITSOverlapEdgeRows from the edge
+        int row = 0, col = 0;
+        o2::itsmft::SegmentationAlpide::localToDetectorUnchecked(locXYZ[0], locXYZ[2], row, col);
+        int drow = row < o2::itsmft::SegmentationAlpide::NRows / 2 ? row : o2::itsmft::SegmentationAlpide::NRows - row - 1; // distance to the edge
+        if (drow * o2::itsmft::SegmentationAlpide::PitchRow < algConf.ITSOverlapMargin) {                                   // rough check is passed, check if the edge cluster is indeed good
+          cl3d.setBit(row < o2::itsmft::SegmentationAlpide::NRows / 2 ? EdgeFlags::LowRow : EdgeFlags::HighRow);            // flag that this is an edge cluster and indicate the low/high row side
+          // check if it is not too close to the edge (to be biased)
+          if (algConf.ITSOverlapEdgeRows > 0) { // is there a restriction?
+            auto pattID = c.getPatternID();
+            drow = c.getRow();
+            if (pattID != itsmft::CompCluster::InvalidPatternID) {
+              if (!mITSDict->isGroup(pattID)) {
+                const auto& patt = mITSDict->getPattern(pattID); // reference pixel is min row/col corner
+                if (row > o2::itsmft::SegmentationAlpide::NRows / 2) {
+                  drow = o2::itsmft::SegmentationAlpide::NRows - 1 - (drow + patt.getRowSpan() - 1);
+                }
+              } else { // group: reference pixel is the one containing the COG
+                o2::itsmft::ClusterPattern patt(pattItCopy);
+                drow = row < o2::itsmft::SegmentationAlpide::NRows / 2 ? drow - patt.getRowSpan() / 2 : o2::itsmft::SegmentationAlpide::NRows - 1 - (drow + patt.getRowSpan() / 2 - 1);
+              }
+            } else {
+              o2::itsmft::ClusterPattern patt(pattItCopy); // reference pixel is min row/col corner
+              if (row > o2::itsmft::SegmentationAlpide::NRows / 2) {
+                drow = o2::itsmft::SegmentationAlpide::NRows - 1 - (drow + patt.getRowSpan() - 1);
+              }
+            }
+            if (drow < algConf.ITSOverlapEdgeRows) { // too close to the edge, flag this
+              cl3d.setBit(EdgeFlags::Biased);
+            }
+          }
+          if (!cl3d.isBitSet(EdgeFlags::Biased)) {
+            if (chipROFStart[sensID].rofCount != ROFCount) { // remember 1st entry
+              chipROFStart[sensID].rofCount = ROFCount;
+              chipROFStart[sensID].chipFirstEntry = edgeClusters.size(); // remember 1st entry of edge cluster for this chip
+            }
+            edgeClusters.push_back(ic);
+          }
+        }
+      }
+    } // clusters of ROF
+    // relate edge clusters of ROF to each other
+    int prevSensID = -1;
+    for (auto ic : edgeClusters) {
+      auto& cl = mITSClustersArray[ic];
+      int sensID = cl.getSensorID();
+      auto ovl = mOverlaps[sensID];
+      int ovlCount = 0;
+      for (int ir = 0; ir < OVL::NSides; ir++) {
+        if (ovl.rowSide[ir] == OVL::NONE) { // no overlap from this row side
+          continue;
+        }
+        int chipOvl = ovl.rowSide[ir]; // look for overlaps with this chip
+        // are there clusters with overlaps on chipOvl?
+        if (chipROFStart[chipOvl].rofCount == ROFCount) {
+          auto oClusID = edgeClusters[chipROFStart[chipOvl].chipFirstEntry];
+          while (oClusID < int(mITSClustersArray.size())) {
+            auto oClus = mITSClustersArray[oClusID];
+            if (oClus.getSensorID() != sensID) {
+              break; // no more clusters on the overlapping chip
+            }
+            if (oClus.isBitSet(ovl.rowSideOverlap[ir]) &&                       // make sure that the edge cluster is on the right side of the row
+                !oClus.isBitSet(EdgeFlags::Biased) &&                           // not too close to the edge
+                std::abs(oClus.getZ() - cl.getZ()) < algConf.ITSOverlapMaxDZ) { // apply fiducial cut on Z distance of 2 clusters
+              // register overlaping cluster
+              if (!ovlCount) { // 1st overlap
+                mOverlapClusRef[ic] = mOverlapCandidateID.size();
+              }
+              mOverlapCandidateID.push_back(oClusID);
+              ovlCount++;
+            }
+            oClusID++;
+          }
+        }
+      }
+      cl.setCount(std::min(127, ovlCount));
+    }
 
+    ROFCount++;
+  } // loop over ROFs
   return true;
 }
 
@@ -193,26 +382,7 @@ bool AlignableDetectorITS::prepareDetectorData()
 void AlignableDetectorITS::Print(const Option_t* opt) const
 {
   AlignableDetector::Print(opt);
-  printf("Sel.pattern   Collisions: %7s | Cosmic: %7s\n",
-         GetITSPattName(fITSPatt[Coll]), GetITSPattName(fITSPatt[Cosm]));
-}
-
-/*
-// RSTODO
-//____________________________________________
-bool AlignableDetectorITS::AcceptTrack(const AliESDtrack* trc, int trtype) const
-{
-  // test if detector had seed this track
-  if (!CheckFlags(trc, trtype))
-    return false;
-  if (trc->GetNcls(0) < mNPointsSel[trtype])
-    return false;
-  if (!CheckHitPattern(trc, GetITSSelPattern(trtype)))
-    return false;
-  //
-  return true;
 }
-*/
 
 //____________________________________________
 void AlignableDetectorITS::SetAddErrorLr(int ilr, double sigY, double sigZ)
@@ -239,7 +409,7 @@ void AlignableDetectorITS::SetSkipLr(int ilr)
 //_________________________________________________
 void AlignableDetectorITS::setUseErrorParam(int v)
 {
-  // set type of points error parameterization
+  // set type of points error parameterization // RS DO WE NEED THIS?
   mUseErrorParam = v;
 }
 
diff --git a/Detectors/Align/src/AlignableDetectorTOF.cxx b/Detectors/Align/src/AlignableDetectorTOF.cxx
index 6dc79c7bd685f..9fa78e133a25d 100644
--- a/Detectors/Align/src/AlignableDetectorTOF.cxx
+++ b/Detectors/Align/src/AlignableDetectorTOF.cxx
@@ -18,9 +18,9 @@
 #include "Align/AlignableVolume.h"
 #include "Align/AlignableSensorTOF.h"
 #include "Align/Controller.h"
-//#include "AliGeomManager.h"
-//#include "AliTOFGeometry.h"
-//#include "AliESDtrack.h"
+#include "DataFormatsGlobalTracking/RecoContainer.h"
+#include "DataFormatsTOF/Cluster.h"
+#include "TOFBase/Geo.h"
 #include 
 
 ClassImp(o2::align::AlignableDetectorTOF);
@@ -31,17 +31,9 @@ namespace align
 {
 
 //____________________________________________
-AlignableDetectorTOF::AlignableDetectorTOF(const char* title)
+AlignableDetectorTOF::AlignableDetectorTOF(Controller* ctr) : AlignableDetector(DetID::TOF, ctr)
 {
   // default c-tor
-  SetNameTitle(Controller::getDetNameByDetID(Controller::kTOF), title);
-  setDetID(Controller::kTOF);
-}
-
-//____________________________________________
-AlignableDetectorTOF::~AlignableDetectorTOF()
-{
-  // d-tor
 }
 
 //____________________________________________
@@ -49,38 +41,92 @@ void AlignableDetectorTOF::defineVolumes()
 {
   // define TOF volumes
   //
-  const int kNSect = 18, kNStrips = AliTOFGeometry::NStripA() + 2 * AliTOFGeometry::NStripB() + 2 * AliTOFGeometry::NStripC();
+  constexpr int NSect = 18;
   int labDet = getDetLabel();
-  AlignableSensorTOF* strip = 0;
+  AlignableSensorTOF* strip = nullptr;
   //
   //  AddVolume( volTOF = new AlignableVolume("TOF") ); // no main volume, why?
-  AlignableVolume* sect[kNSect] = {0};
-  //
-  for (int isc = 0; isc < kNSect; isc++) {
-    int iid = labDet + (1 + isc) * 100;
-    addVolume(sect[isc] = new AlignableVolume(Form("TOF/sm%02d", isc), iid));
-  }
+  AlignableVolume* sect[NSect] = {};
+  AlignableVolume* volTOF = nullptr; // fictious envelope
+  addVolume(volTOF = new AlignableVolume("TOF_envelope", getDetLabel(), mController));
+  volTOF->setDummyEnvelope();
   //
   int cnt = 0;
-  for (int isc = 0; isc < kNSect; isc++) {
-    for (int istr = 1; istr <= kNStrips; istr++) { // strip
-      int iid = labDet + (1 + isc) * 100 + (1 + istr);
-      int vid = AliGeomManager::LayerToVolUID(AliGeomManager::kTOF, cnt++);
+  for (int isc = 0; isc < NSect; isc++) {
+    for (int istr = 1; istr <= o2::tof::Geo::NSTRIPXSECTOR; istr++) { // strip
       const char* symname = Form("TOF/sm%02d/strip%02d", isc, istr);
-      if (!gGeoManager->GetAlignableEntry(symname))
-        continue;
-      addVolume(strip = new AlignableSensorTOF(symname, vid, iid, isc));
+      addVolume(strip = new AlignableSensorTOF(symname, o2::base::GeometryManager::getSensID(DetID::TOF, cnt), getSensLabel(cnt), isc, mController));
+      cnt++;
+      if (!gGeoManager->GetAlignableEntry(symname)) {
+        strip->setDummy(true);
+        //        continue;
+      }
+      if (!sect[isc]) {
+        sect[isc] = new AlignableVolume(Form("TOF/sm%02d", isc), getNonSensLabel(isc), mController);
+        sect[isc]->setParent(volTOF);
+      }
       strip->setParent(sect[isc]);
     } // strip
   }   // layer
-  //
+
+  for (int isc = 0; isc < NSect; isc++) {
+    if (sect[isc]) {
+      addVolume(sect[isc]);
+    }
+  }
 }
 
 //____________________________________________
-bool AlignableDetectorTOF::AcceptTrack(const AliESDtrack* trc, int trtype) const
+int AlignableDetectorTOF::processPoints(GIndex gid, int npntCut, bool inv)
 {
-  // test if detector had seed this track
-  return CheckFlags(trc, trtype);
+  // Extract the points corresponding to this detector, recalibrate/realign them to the
+  // level of the "starting point" for the alignment/calibration session.
+  // If inv==true, the track propagates in direction of decreasing tracking X
+  // (i.e. upper leg of cosmic track)
+  //
+
+  int npoints = 0;
+  auto algTrack = mController->getAlgTrack();
+  auto recoData = mController->getRecoContainer();
+  auto TOFClusters = recoData->getTOFClusters();
+  if (TOFClusters.empty()) {
+    return -1; // source not loaded?
+  }
+  const auto& clus = TOFClusters[gid.getIndex()];
+  int det[5] = {}, ch = clus.getMainContributingChannel();
+  o2::tof::Geo::getVolumeIndices(ch, det);
+  int sensID = o2::tof::Geo::getStripNumberPerSM(det[1], det[2]) + clus.getSector() * o2::tof::Geo::NSTRIPXSECTOR;
+  auto* sensor = (AlignableSensorTOF*)getSensor(sensID);
+  if (sensor->isDummy()) {
+    LOGP(error, "Dummy sensor {} is referred by a track", sensID);
+    return 0;
+  }
+  double loc[3] = {(det[4] + 0.5) * o2::tof::Geo::XPAD - o2::tof::Geo::XHALFSTRIP, 0., (det[3] - 0.5) * o2::tof::Geo::ZPAD}, locCorr[3] = {}, traCorr[3] = {};
+  const auto& matAlg = sensor->getMatrixClAlg();
+  matAlg.LocalToMaster(loc, locCorr);
+  // rotate to tracking
+  const auto& matT2L = sensor->getMatrixT2L();
+  matT2L.MasterToLocal(locCorr, traCorr);
+  //
+  auto& pnt = algTrack->addDetectorPoint();
+
+  const auto* sysE = sensor->getAddError(); // additional syst error
+  pnt.setYZErrTracking(clus.getSigmaY2() + sysE[0] * sysE[0], clus.getSigmaYZ(), clus.getSigmaZ2() + sysE[1] * sysE[1]);
+  if (getUseErrorParam()) { // errors will be calculated just before using the point in the fit, using track info
+    pnt.setNeedUpdateFromTrack();
+  }
+  pnt.setXYZTracking(traCorr[0], traCorr[1], traCorr[2]);
+  pnt.setSensor(sensor);
+  pnt.setAlphaSens(sensor->getAlpTracking());
+  pnt.setXSens(sensor->getXTracking());
+  pnt.setDetID(mDetID);
+  pnt.setSID(sensor->getSID());
+  pnt.setContainsMeasurement();
+  pnt.setInvDir(inv);
+  pnt.init();
+  npoints++;
+  mNPoints += npoints;
+  return npoints;
 }
 
 } // namespace align
diff --git a/Detectors/Align/src/AlignableDetectorTPC.cxx b/Detectors/Align/src/AlignableDetectorTPC.cxx
index 63fac7dd9cac6..980ded2d8ff2f 100644
--- a/Detectors/Align/src/AlignableDetectorTPC.cxx
+++ b/Detectors/Align/src/AlignableDetectorTPC.cxx
@@ -10,84 +10,255 @@
 // or submit itself to any jurisdiction.
 
 /// @file   AlignableDetectorTPC.h
-/// @author ruben.shahoyan@cern.ch, michael.lettrich@cern.ch
-/// @since  2021-02-01
+/// @author ruben.shahoyan@cern.ch
 /// @brief  TPC detector wrapper
 
 #include "Align/AlignableDetectorTPC.h"
 #include "Align/AlignableVolume.h"
 #include "Align/AlignableSensorTPC.h"
 #include "Align/Controller.h"
-//#include "AliGeomManager.h"
-//#include "AliESDtrack.h"
-#include "Framework/Logger.h"
+#include "Align/AlignConfig.h"
+#include "DataFormatsGlobalTracking/RecoContainer.h"
+#include "DataFormatsTPC/Constants.h"
+#include "DataFormatsTPC/TrackTPC.h"
+#include "DataFormatsTPC/WorkflowHelper.h"
+#include 
 #include 
-
-ClassImp(o2::align::AlignableDetectorTPC);
+#include "GPUO2ExternalUser.h"
+#include "DataFormatsTPC/WorkflowHelper.h"
+#include "GPUParam.inc"
 
 namespace o2
 {
 namespace align
 {
+using namespace TMath;
 
 //____________________________________________
-AlignableDetectorTPC::AlignableDetectorTPC(const char* title)
+AlignableDetectorTPC::AlignableDetectorTPC(Controller* ctr) : AlignableDetector(DetID::TPC, ctr)
 {
   // default c-tor
-  SetNameTitle(Controller::getDetNameByDetID(Controller::kTPC), title);
-  setDetID(Controller::kTPC);
 }
 
 //____________________________________________
-AlignableDetectorTPC::~AlignableDetectorTPC()
+void AlignableDetectorTPC::defineVolumes()
 {
-  // d-tor
+  // define fictious TPC envelope and sector volumes
+  AlignableVolume* volTPC = nullptr;
+  int labDet = getDetLabel();
+  const int NSectors = o2::tpc::constants::MAXSECTOR / 2;
+
+  addVolume(volTPC = new AlignableVolume("TPC_envelope", getDetLabel(), mController));
+  volTPC->setDummyEnvelope();
+
+  for (int isec = 0; isec < o2::tpc::constants::MAXSECTOR; isec++) {
+    int isecSide = isec % NSectors;
+    const char* symname = Form("TPC/sec%02d", isec);
+    AlignableSensorTPC* sector = new AlignableSensorTPC(symname, o2::base::GeometryManager::getSensID(mDetID, isec), getSensLabel(isec), isec, mController);
+    sector->setParent(volTPC);
+    addVolume(sector);
+  }
 }
 
 //____________________________________________
-void AlignableDetectorTPC::defineVolumes()
+void AlignableDetectorTPC::Print(const Option_t* opt) const
 {
-  // define TPC volumes
-  //
-  const int kNSect = 18, kAC = 2, kIOROC = 2;
-  const char* kSide[kAC] = {"A", "C"};
-  const char* kROC[kIOROC] = {"Inner", "Outer"};
-  //  AlignableSensorTPC *chamb=0;
-  //
-  int labDet = getDetLabel();
-  AlignableVolume* volTPC = new AlignableVolume("ALIC_1/TPC_M_1", labDet);
-  addVolume(volTPC);
+  // print info
+  AlignableDetector::Print(opt);
+}
+
+//____________________________________________
+int AlignableDetectorTPC::processPoints(GIndex gid, int npntCut, bool inv)
+{
+  // Extract the points corresponding to this detector, recalibrate/realign them to the
+  // level of the "starting point" for the alignment/calibration session.
+  // If inv==true, the track propagates in direction of decreasing tracking X
+  // (i.e. upper leg of cosmic track)
   //
+  const auto& algConf = AlignConfig::Instance();
+  const auto recoData = mController->getRecoContainer();
+  const auto& trk = recoData->getTrack(gid);
+  gsl::span TPCShMap = recoData->clusterShMapTPC;
+
+  int nClus = trk.getNClusters();
+  if (nClus < npntCut) {
+    return -1;
+  }
+  int npointsIni = mNPoints;
+  auto prop = o2::base::Propagator::Instance(); // float version!
+  constexpr float TAN10 = 0.17632698;
+  const auto clusterIdxStruct = recoData->getTPCTracksClusterRefs();
+  const auto clusterNativeAccess = recoData->inputsTPCclusters->clusterIndex;
+  bool fail = false;
+  auto algTrack = mController->getAlgTrack();
+
+  o2::track::TrackParCov trkParam = inv ? trk : trk.getOuterParam(); // we refit outer param inward
+  trkParam.resetCovariance();
+  float bzkG = prop->getNominalBz(), qptB5Scale = std::abs(bzkG) > 0.1 ? std::abs(bzkG) / 5.006680f : 1.f;
+  float q2pt2 = trkParam.getQ2Pt() * trkParam.getQ2Pt(), q2pt2Wgh = q2pt2 * qptB5Scale * qptB5Scale;
+  float err2 = (100.f + q2pt2Wgh) / (1.f + q2pt2Wgh) * q2pt2; // -> 100 for high pTs, -> 1 for low pTs.
+  trkParam.setCov(err2, 14);                                  // 100% error
+
+  int direction = inv ? -1 : 1;
+  int start = inv ? nClus - 1 : 0;
+  int stop = inv ? -1 : nClus;
+  const o2::tpc::ClusterNative* cl = nullptr;
+  uint8_t sector, row = 0, currentSector = 0, currentRow = 0;
+  short clusterState = 0, nextState;
+  float tOffset = mTrackTimeStamp / (o2::constants::lhc::LHCBunchSpacingMUS * 8);
+  bool stopLoop = false;
+  int npoints = 0;
+  for (int i = start; i != stop; i += cl ? 0 : direction) {
+    float x, y, z, xTmp, yTmp, zTmp, charge = 0.f;
+    int clusters = 0;
+    double combRow = 0;
 
-  for (int roc = 0; roc < kIOROC; roc++) {     // inner/outer
-    for (int side = 0; side < kAC; side++) {   // A/C
-      for (int isc = 0; isc < kNSect; isc++) { // sector ID
-        const char* symname = Form("TPC/Endcap%s/Sector%d/%sChamber", kSide[side], isc + 1, kROC[roc]);
-        if (!gGeoManager->GetAlignableEntry(symname)) {
-          AliErrorF("Did not find alignable %s", symname);
+    while (true) {
+      if (!cl) {
+        auto clTmp = &trk.getCluster(clusterIdxStruct, i, clusterNativeAccess, sector, row);
+        if (row < algConf.minTPCPadRow) {
+          if (!inv) { // inward refit: all other clusters padrow will be <= minumum (with outward refit the following clusters have a chance to be accepted)
+            stopLoop = true;
+          }
+          break;
+        } else if (row > algConf.maxTPCPadRow) {
+          if (inv) { // outward refit: all other clusters padrow will be >= maximum (with inward refit the following clusters have a chance to be accepted)
+            stopLoop = true;
+          }
+          break;
+        }
+        if (algConf.discardEdgePadrows > 0 && getDistanceToStackEdge(row) < algConf.discardEdgePadrows) {
+          if (i + direction != stop) {
+            i += direction;
+            continue;
+          } else {
+            stopLoop = true;
+            break;
+          }
+        }
+        mController->getTPCCorrMaps()->Transform(sector, row, clTmp->getPad(), clTmp->getTime(), xTmp, yTmp, zTmp, tOffset);
+        if (algConf.discardSectorEdgeDepth > 0) {
+          if (std::abs(yTmp) + algConf.discardSectorEdgeDepth > xTmp * TAN10) {
+            if (i + direction != stop) {
+              i += direction;
+              continue;
+            } else {
+              stopLoop = true;
+              break;
+            }
+          }
+        }
+
+        cl = clTmp;
+        nextState = TPCShMap[cl - clusterNativeAccess.clustersLinear];
+      }
+      if (clusters == 0 || (sector == currentSector && std::abs(row - currentRow) < algConf.maxTPCRowsCombined)) {
+        if (clusters == 1) {
+          x *= charge;
+          y *= charge;
+          z *= charge;
+          combRow *= charge;
+        }
+        if (clusters == 0) {
+          x = xTmp;
+          y = yTmp;
+          z = zTmp;
+          // mController->getTPCCorrMaps()->Transform(sector, row, cl->getPad(), cl->getTime(), x, y, z, tOffset);
+          currentRow = row;
+          currentSector = sector;
+          charge = cl->qTot;
+          clusterState = nextState;
+          combRow = row;
+          LOGP(debug, "starting a supercluster at row {} of sector {} -> {},{},{}", currentRow, currentSector, x, y, z);
+        } else {
+          // float xx, yy, zz;
+          // mController->getTPCCorrMaps()->Transform(sector, row, cl->getPad(), cl->getTime(), xx, yy, zz, tOffset);
+          x += xTmp * cl->qTot;
+          y += yTmp * cl->qTot;
+          z += zTmp * cl->qTot;
+          combRow += row * cl->qTot;
+          charge += cl->qTot;
+          clusterState |= nextState;
+          npntCut--;
+          LOGP(debug, "merging cluster #{} at row {} to a supercluster starting at row {} ", clusters + 1, row, currentRow);
+        }
+        cl = nullptr;
+        clusters++;
+        if (i + direction != stop) {
+          i += direction;
           continue;
         }
-        int iid = side * kNSect + isc;
-        uint16_t vid = AliGeomManager::LayerToVolUID(AliGeomManager::kTPC1 + roc, iid);
-        iid = labDet + (1 + side) * 10000 + (1 + isc) * 100 + (1 + roc);
-        AlignableSensorTPC* sens = new AlignableSensorTPC(symname, vid, iid, isc);
-        sens->setParent(volTPC);
-        addVolume(sens);
-      } // sector ID
-    }   // A/C
-  }     // inner/outer
-  //
-}
+      }
+      break;
+    }
+    if (stopLoop) {
+      break;
+    }
+    if (clusters == 0) {
+      continue;
+    } else if (clusters > 1) {
+      x /= charge;
+      y /= charge;
+      z /= charge;
+      currentRow = combRow / charge;
+      LOGP(debug, "Combined cluster of {} subclusters: row {} , {},{},{}", clusters, currentRow, x, y, z);
+    }
 
-//____________________________________________
-bool AlignableDetectorTPC::AcceptTrack(const AliESDtrack* trc, int trtype) const
-{
-  // test if detector had seed this track
-  if (!CheckFlags(trc, trtype))
-    return false;
-  if (trc->GetNcls(1) < mNPointsSel[trtype])
-    return false;
-  return true;
+    if (!trkParam.rotate(math_utils::detail::sector2Angle(currentSector)) || !prop->PropagateToXBxByBz(trkParam, x, algConf.maxSnp)) {
+      break;
+    }
+    if (!npoints) {
+      trkParam.setZ(z);
+    }
+
+    auto* sectSensor = (AlignableSensorTPC*)getSensor(currentSector);
+    const auto* sysE = sectSensor->getAddError(); // additional syst error
+
+    std::array p = {y, z};
+    std::array c = {0, 0, 0};
+    mController->getTPCParam()->GetClusterErrors2(sector, currentRow, z, trkParam.getSnp(), trkParam.getTgl(), -1.f, 0.f, 0.f, c[0], c[2]); // TODO: Note this disables occupancy / charge components of the error estimation
+    mController->getTPCParam()->UpdateClusterError2ByState(clusterState, c[0], c[2]);
+    int nrComb = std::abs(row - currentRow) + 1;
+    if (nrComb > 1) {
+      float fact = 1. / std::sqrt(nrComb);
+      c[0] *= fact;
+      c[2] *= fact;
+    }
+    if (sysE[0] > 0.f) {
+      c[0] += sysE[0] * sysE[0];
+    }
+    if (sysE[1] > 0.f) {
+      c[2] += sysE[1] * sysE[1];
+    }
+
+    if (!trkParam.update(p, c)) {
+      break;
+    }
+
+    auto& pnt = algTrack->addDetectorPoint();
+    pnt.setYZErrTracking(c[0], c[1], c[2]);
+    if (getUseErrorParam()) { // errors will be calculated just before using the point in the fit, using track info
+      pnt.setNeedUpdateFromTrack();
+    }
+    pnt.setXYZTracking(x, y, z);
+    pnt.setSensor(sectSensor);
+    pnt.setAlphaSens(sectSensor->getAlpTracking());
+    pnt.setXSens(sectSensor->getXTracking());
+    pnt.setDetID(mDetID);
+    pnt.setSID(sectSensor->getSID());
+    pnt.setContainsMeasurement();
+    pnt.setInvDir(inv);
+    pnt.init();
+    npoints++;
+  }
+  if (npoints < npntCut) {
+    algTrack->suppressLastPoints(npoints);
+    mNPoints = npointsIni;
+    npoints = -1;
+  }
+  mNPoints += npoints;
+
+  return npoints;
 }
 
 } // namespace align
diff --git a/Detectors/Align/src/AlignableDetectorTRD.cxx b/Detectors/Align/src/AlignableDetectorTRD.cxx
index 3b7a2177e0e60..080d0f72b2516 100644
--- a/Detectors/Align/src/AlignableDetectorTRD.cxx
+++ b/Detectors/Align/src/AlignableDetectorTRD.cxx
@@ -18,105 +18,85 @@
 #include "Align/AlignableVolume.h"
 #include "Align/AlignableSensorTRD.h"
 #include "Align/Controller.h"
-//#include "AliGeomManager.h"
-//#include "AliESDtrack.h"
-//#include "AliTRDgeometry.h"
-#include 
+#include "Align/AlignConfig.h"
+#include "TRDBase/Geometry.h"
+#include "TRDBase/TrackletTransformer.h"
+#include "DataFormatsGlobalTracking/RecoContainer.h"
+#include "DataFormatsTRD/Constants.h"
+#include "DataFormatsTRD/TrackTRD.h"
+#include "DataFormatsTRD/Tracklet64.h"
+#include "DataFormatsTRD/CalibratedTracklet.h"
+#include "GPUO2InterfaceConfiguration.h"
 #include 
-
-using namespace TMath;
-
-ClassImp(o2::align::AlignableDetectorTRD);
+#include 
 
 namespace o2
 {
 namespace align
 {
-
-const char* AlignableDetectorTRD::fgkCalibDOFName[AlignableDetectorTRD::kNCalibParams] = {"DZdTglNRC", "DVDriftT"};
+using namespace TMath;
+const char* AlignableDetectorTRD::CalibDOFName[AlignableDetectorTRD::NCalibParams] = {"DZdTglNRC", "DVDriftT"};
 
 //____________________________________________
-AlignableDetectorTRD::AlignableDetectorTRD(const char* title)
-  : AlignableDetector(), fNonRCCorrDzDtgl(0), fCorrDVT(0)
+AlignableDetectorTRD::AlignableDetectorTRD(Controller* ctr) : AlignableDetector(DetID::TRD, ctr)
 {
   // default c-tor
-  SetNameTitle(Controller::getDetNameByDetID(Controller::kTRD), title);
-  setDetID(Controller::kTRD);
-  fExtraErrRC[0] = fExtraErrRC[1] = 0;
-  //
-  // ad hoc correction
-  SetNonRCCorrDzDtgl();
-  SetExtraErrRC();
-  mNCalibDOFs = kNCalibParams;
-}
-
-//____________________________________________
-AlignableDetectorTRD::~AlignableDetectorTRD()
-{
-  // d-tor
 }
 
 //____________________________________________
 void AlignableDetectorTRD::defineVolumes()
 {
   // define TRD volumes
-  //
-  const int kNSect = 18, kNStacks = 5, kNLayers = 6;
-  AlignableSensorTRD* chamb = 0;
-  //
-  int labDet = getDetLabel();
-  //  AddVolume( volTRD = new AlignableVolume("TRD") ); // no main volume, why?
-  AlignableVolume* sect[kNSect] = {0};
-  //
-  for (int ilr = 0; ilr < kNLayers; ilr++) {            // layer
-    for (int ich = 0; ich < kNStacks * kNSect; ich++) { // chamber
-      int isector = ich / AliTRDgeometry::Nstack();
-      int istack = ich % AliTRDgeometry::Nstack();
-      //int lid       = AliTRDgeometry::GetDetector(ilr,istack,isector);
-      int iid = labDet + (1 + ilr) * 10000 + (1 + isector) * 100 + (1 + istack);
+  auto geo = o2::trd::Geometry::instance();
+  geo->createPadPlaneArray();
+  geo->createClusterMatrixArray(); // ideal T2L matrices
+
+  AlignableSensorTRD* chamb = nullptr;
+  AlignableVolume* sect[o2::trd::constants::NSECTOR]{};
+  AlignableVolume* volTRD = nullptr; // fictious envelope
+
+  addVolume(volTRD = new AlignableVolume("TRD_envelope", getDetLabel(), mController));
+  volTRD->setDummyEnvelope();
+
+  for (int ilr = 0; ilr < o2::trd::constants::NLAYER; ilr++) {                                 // layer
+    for (int ich = 0; ich < o2::trd::constants::NSTACK * o2::trd::constants::NSECTOR; ich++) { // chamber
+      int isector = ich / o2::trd::constants::NSTACK;
+      int istack = ich % o2::trd::constants::NSTACK;
+      uint16_t sid = o2::trd::Geometry::getDetector(ilr, istack, isector);
       const char* symname = Form("TRD/sm%02d/st%d/pl%d", isector, istack, ilr);
-      if (!gGeoManager->GetAlignableEntry(symname))
-        continue;
-      uint16_t vid = AliGeomManager::LayerToVolUID(AliGeomManager::kTRD1 + ilr, ich);
-      addVolume(chamb = new AlignableSensorTRD(symname, vid, iid /*lid*/, isector));
-      iid = labDet + (1 + isector) * 100;
-      if (!sect[isector])
-        sect[isector] = new AlignableVolume(Form("TRD/sm%02d", isector), iid);
+      addVolume(chamb = new AlignableSensorTRD(symname, o2::base::GeometryManager::getSensID(mDetID, sid), getSensLabel(sid), isector, mController));
+      if (!gGeoManager->GetAlignableEntry(symname)) {
+        chamb->setDummy(true);
+        //        continue;
+      }
+      if (!sect[isector]) {
+        sect[isector] = new AlignableVolume(Form("TRD/sm%02d", isector), getNonSensLabel(isector), mController);
+        sect[isector]->setParent(volTRD);
+      }
       chamb->setParent(sect[isector]);
     } // chamber
   }   // layer
   //
-  for (int isc = 0; isc < kNSect; isc++) {
-    if (sect[isc])
+  for (int isc = 0; isc < o2::trd::constants::NSECTOR; isc++) {
+    if (sect[isc]) {
       addVolume(sect[isc]);
+    }
   }
-  //
-}
-
-//____________________________________________
-bool AlignableDetectorTRD::AcceptTrack(const AliESDtrack* trc, int trtype) const
-{
-  // test if detector had seed this track
-  if (!CheckFlags(trc, trtype))
-    return false;
-  if (trc->GetTRDntracklets() < mNPointsSel[trtype])
-    return false;
-  return true;
 }
 
-//__________________________________________
 //____________________________________________
 void AlignableDetectorTRD::Print(const Option_t* opt) const
 {
   // print info
   AlignableDetector::Print(opt);
-  printf("Extra error for RC tracklets: Y:%e Z:%e\n", fExtraErrRC[0], fExtraErrRC[1]);
+  printf("Extra error for RC tracklets: Y:%e Z:%e\n", mExtraErrRC[0], mExtraErrRC[1]);
 }
 
+//____________________________________________
 const char* AlignableDetectorTRD::getCalibDOFName(int i) const
 {
   // return calibration DOF name
-  return i < kNCalibParams ? fgkCalibDOFName[i] : 0;
+  return i < NCalibParams ? CalibDOFName[i] : nullptr;
 }
 
 //______________________________________________________
@@ -138,7 +118,20 @@ void AlignableDetectorTRD::writePedeInfo(FILE* parOut, const Option_t* opt) cons
   for (int ip = 0; ip < getNCalibDOFs(); ip++) {
     int cmt = isCondDOF(ip) ? kOff : kOn;
     fprintf(parOut, "%s %9d %+e %+e\t%s %s p%d\n", comment[cmt], getParLab(ip),
-            getParVal(ip), getParErr(ip), comment[kOnOn], isFreeDOF(ip) ? "  " : "FX", ip);
+            -getParVal(ip), getParErr(ip), comment[kOnOn], isFreeDOF(ip) ? "  " : "FX", ip);
+  }
+  //
+}
+
+//______________________________________________________
+void AlignableDetectorTRD::writeLabeledPedeResults(FILE* parOut) const
+{
+  //
+  AlignableDetector::writeLabeledPedeResults(parOut);
+  //
+  for (int ip = 0; ip < getNCalibDOFs(); ip++) {
+    fprintf(parOut, "%9d %+e %+e\t! calib param %d of %s %s %s\n", getParLab(ip), -getParVal(ip), getParErr(ip), ip, GetName(),
+            isFreeDOF(ip) ? "   " : "FXU", o2::align::utils::isZeroAbs(getParVal(ip)) ? "FXP" : "   ");
   }
   //
 }
@@ -149,11 +142,11 @@ double AlignableDetectorTRD::getCalibDOFVal(int id) const
   // return preset value of calibration dof
   double val = 0;
   switch (id) {
-    case kCalibNRCCorrDzDtgl:
-      val = GetNonRCCorrDzDtgl();
+    case CalibNRCCorrDzDtgl:
+      val = getNonRCCorrDzDtgl();
       break;
-    case kCalibDVT:
-      val = GetCorrDVT();
+    case CalibDVT:
+      val = getCorrDVT();
       break;
     default:
       break;
@@ -168,5 +161,106 @@ double AlignableDetectorTRD::getCalibDOFValWithCal(int id) const
   return getCalibDOFVal(id) + getParVal(id);
 }
 
+//____________________________________________
+int AlignableDetectorTRD::processPoints(GIndex gid, int npntCut, bool inv)
+{
+  // Extract the points corresponding to this detector, recalibrate/realign them to the
+  // level of the "starting point" for the alignment/calibration session.
+  // If inv==true, the track propagates in direction of decreasing tracking X
+  // (i.e. upper leg of cosmic track)
+  //
+  const auto& algConf = AlignConfig::Instance();
+  const auto recoData = mController->getRecoContainer();
+  const auto& trk = recoData->getTrack(gid);
+  if (trk.getNtracklets() < npntCut) {
+    return -1;
+  }
+  auto propagator = o2::base::Propagator::Instance(); // float version!
+  static bool firstCall = true;
+  if (firstCall) {
+    o2::gpu::GPUO2InterfaceConfiguration config;
+    config.ReadConfigurableParam(config);
+    mRecoParam.init(propagator->getNominalBz(), &config.configReconstruction);
+    firstCall = false;
+  }
+  const auto* transformer = mController->getTRDTransformer();
+  auto algTrack = mController->getAlgTrack();
+  const auto trackletsRaw = recoData->getTRDTracklets();
+  bool fail = false;
+  int nPntIni = algTrack->getNPoints(), npoints = 0;
+  o2::track::TrackPar trkParam = trk.getOuterParam(); // we refit outer param inward to get tracklet coordinates accounting for tilt
+  for (int il = o2::trd::constants::NLAYER; il--;) {
+    if (trk.getTrackletIndex(il) == -1) {
+      continue;
+    }
+    int trkltId = trk.getTrackletIndex(il);
+    const auto& trackletRaw = trackletsRaw[trkltId];
+    const auto trackletCalibLoc = transformer->transformTracklet(trackletRaw, false); // calibrated tracket in local frame !!!
+    int trkltDet = trackletRaw.getDetector();
+    auto* sensor = (AlignableSensorTRD*)getSensor(trkltDet);
+    if (sensor->isDummy()) {
+      LOGP(error, "Dummy sensor {} is referred by a track", trkltDet);
+      fail = true;
+      continue;
+    }
+    double locXYZ[3] = {trackletCalibLoc.getX(), trackletCalibLoc.getY(), trackletCalibLoc.getZ()}, locXYZC[3], traXYZ[3];
+    const auto& matAlg = sensor->getMatrixClAlg(); // local alignment matrix
+    matAlg.LocalToMaster(locXYZ, locXYZC);         // aligned point in the local frame
+    const auto& mat = sensor->getMatrixT2L();      // RS FIXME check if correct
+    mat.MasterToLocal(locXYZC, traXYZ);
+    int trkltSec = sensor->getSector();       // trkltDet / (o2::trd::constants::NLAYER * o2::trd::constants::NSTACK);
+    float alpSens = sensor->getAlpTracking(); // o2::math_utils::sector2Angle(trkltSec);
+    if (trkltSec != o2::math_utils::angle2Sector(trkParam.getAlpha()) ||
+        !trkParam.rotateParam(alpSens) ||
+        !propagator->propagateTo(trkParam, traXYZ[0], propagator->getNominalBz(), o2::base::Propagator::MAX_SIN_PHI, 10., o2::base::Propagator::MatCorrType::USEMatCorrNONE)) { // we don't need high precision here
+      fail = true;
+      break;
+    }
+    const o2::trd::PadPlane* pad = o2::trd::Geometry::instance()->getPadPlane(trkltDet);
+    float tilt = std::tan(TMath::DegToRad() * pad->getTiltingAngle()); // tilt is signed! and returned in degrees
+    float tiltCorrUp = tilt * (traXYZ[2] - trkParam.getZ());
+    float zPosCorrUp = trkParam.getZ() + getNonRCCorrDzDtglWithCal() * trkParam.getTgl(); // + mRecoParam.getZCorrCoeffNRC() * trkParam.getTgl();
+    float padLength = pad->getRowSize(trackletRaw.getPadRow());
+    if (std::fabs(traXYZ[2] - trkParam.getZ()) < padLength) { // RS do we need this?
+      tiltCorrUp = 0.f;
+    }
+    std::array trkltPosUp{float(traXYZ[1] - tiltCorrUp), zPosCorrUp};
+    std::array trkltCovUp;
+    mRecoParam.recalcTrkltCov(tilt, trkParam.getSnp(), pad->getRowSize(trackletRaw.getPadRow()), trkltCovUp);
+    // Correction for DVT, equivalent to shift in X at which Y is evaluated: dY = tg_phi * dvt
+    {
+      auto dvt = getCorrDVTWithCal();
+      if (std::abs(dvt) > utils::AlmostZeroF) {
+        auto snp = trkParam.getSnp();
+        auto slpY = snp / std::sqrt((1.f - snp) * (1.f + snp));
+        trkltPosUp[0] += slpY * dvt;
+      }
+    }
+    auto& pnt = algTrack->addDetectorPoint();
+    const auto* sysE = sensor->getAddError(); // additional syst error
+    pnt.setYZErrTracking(trkltCovUp[0] + sysE[0] * sysE[0], trkltCovUp[1], trkltCovUp[2] + sysE[1] * sysE[1]);
+    if (getUseErrorParam()) { // errors will be calculated just before using the point in the fit, using track info
+      pnt.setNeedUpdateFromTrack();
+    }
+    pnt.setXYZTracking(traXYZ[0], trkltPosUp[0], trkltPosUp[1]);
+    pnt.setSensor(sensor);
+    pnt.setAlphaSens(alpSens);
+    pnt.setXSens(sensor->getXTracking());
+    pnt.setDetID(mDetID);
+    pnt.setSID(sensor->getSID());
+    pnt.setContainsMeasurement();
+    pnt.setInvDir(inv);
+    pnt.init();
+    npoints++;
+  }
+  if (fail) { // reset points to original start
+    algTrack->suppressLastPoints(npoints);
+    npoints = 0;
+  }
+  mNPoints += npoints;
+
+  return npoints;
+}
+
 } // namespace align
 } // namespace o2
diff --git a/Detectors/Align/src/AlignableSensor.cxx b/Detectors/Align/src/AlignableSensor.cxx
index 388a9b8493e48..91658eba0a196 100644
--- a/Detectors/Align/src/AlignableSensor.cxx
+++ b/Detectors/Align/src/AlignableSensor.cxx
@@ -47,7 +47,7 @@ void AlignableSensor::dPosTraDParGeomLOC(const AlignmentPoint* pnt, double* deri
   // Jacobian of position in sensor tracking frame (tra) vs sensor LOCAL frame
   // parameters in TGeoHMatrix convention.
   // Result is stored in array deriv as linearized matrix 6x3
-  const double kDelta[kNDOFGeom] = {0.1, 0.1, 0.1, 0.5, 0.5, 0.5};
+  const double kDelta[kNDOFGeom] = {0.1, 0.1, 0.1, 0.5 * DegToRad(), 0.5 * DegToRad(), 0.5 * DegToRad()}; // changed angles to radians
   double delta[kNDOFGeom], pos0[3], pos1[3], pos2[3], pos3[3];
   TGeoHMatrix matMod;
   //
@@ -94,7 +94,7 @@ void AlignableSensor::dPosTraDParGeomLOC(const AlignmentPoint* pnt, double* deri
   // Jacobian of position in sensor tracking frame (tra) vs parent volume LOCAL frame parameters.
   // NO check of parentship is done!
   // Result is stored in array deriv as linearized matrix 6x3
-  const double kDelta[kNDOFGeom] = {0.1, 0.1, 0.1, 0.5, 0.5, 0.5};
+  const double kDelta[kNDOFGeom] = {0.1, 0.1, 0.1, 0.5 * DegToRad(), 0.5 * DegToRad(), 0.5 * DegToRad()}; // changed angles to radians
   double delta[kNDOFGeom], pos0[3], pos1[3], pos2[3], pos3[3];
   TGeoHMatrix matMod;
   // this is the matrix for transition from sensor to parent volume local frames: LOC=matRel*loc
@@ -107,7 +107,7 @@ void AlignableSensor::dPosTraDParGeomLOC(const AlignmentPoint* pnt, double* deri
   //
   for (int ip = kNDOFGeom; ip--;) {
     //
-    if (!isFreeDOF(ip)) {
+    if (!parent->isFreeDOF(ip)) { // RSCHANGE: was isFreeDOF(ip)
       continue;
     }
     //
@@ -145,7 +145,7 @@ void AlignableSensor::dPosTraDParGeomTRA(const AlignmentPoint* pnt, double* deri
   // tra' = tau*tra
   //
   // Result is stored in array deriv as linearized matrix 6x3
-  const double kDelta[kNDOFGeom] = {0.1, 0.1, 0.1, 0.5, 0.5, 0.5};
+  const double kDelta[kNDOFGeom] = {0.1, 0.1, 0.1, 0.5 * DegToRad(), 0.5 * DegToRad(), 0.5 * DegToRad()}; // changed angles to radians
   double delta[kNDOFGeom], pos0[3], pos1[3], pos2[3], pos3[3];
   TGeoHMatrix matMod;
   //
@@ -188,12 +188,12 @@ void AlignableSensor::dPosTraDParGeomTRA(const AlignmentPoint* pnt, double* deri
 //_________________________________________________________
 void AlignableSensor::dPosTraDParGeomTRA(const AlignmentPoint* pnt, double* deriv, const AlignableVolume* parent) const
 {
-  // Jacobian of position in sensor tracking frame (tra) vs sensor TRACKING
+  // Jacobian of position in sensor tracking frame (tra) vs parent TRACKING
   // frame parameters in TGeoHMatrix convention, i.e. the modified parameter is
   // tra' = tau*tra
   //
   // Result is stored in array deriv as linearized matrix 6x3
-  const double kDelta[kNDOFGeom] = {0.1, 0.1, 0.1, 0.5, 0.5, 0.5};
+  const double kDelta[kNDOFGeom] = {0.1, 0.1, 0.1, 0.5 * DegToRad(), 0.5 * DegToRad(), 0.5 * DegToRad()}; // changed angles to radians
   double delta[kNDOFGeom], pos0[3], pos1[3], pos2[3], pos3[3];
   TGeoHMatrix matMod;
   //
@@ -218,7 +218,7 @@ void AlignableSensor::dPosTraDParGeomTRA(const AlignmentPoint* pnt, double* deri
   //
   for (int ip = kNDOFGeom; ip--;) {
     //
-    if (!isFreeDOF(ip)) {
+    if (!parent->isFreeDOF(ip)) { // RSCHANGE: was isFreeDOF(ip)
       continue;
     }
     //
@@ -405,10 +405,9 @@ void AlignableSensor::dPosTraDParCalib(const AlignmentPoint* pnt, double* deriv,
 }
 
 //______________________________________________________
-int AlignableSensor::finalizeStat(DOFStatistics& st)
+int AlignableSensor::finalizeStat()
 {
   // finalize statistics on processed points
-  fillDOFStat(st);
   return mNProcPoints;
 }
 
diff --git a/Detectors/Align/src/AlignableSensorITS.cxx b/Detectors/Align/src/AlignableSensorITS.cxx
index 3a906eb8bff4c..b4480a25bc740 100644
--- a/Detectors/Align/src/AlignableSensorITS.cxx
+++ b/Detectors/Align/src/AlignableSensorITS.cxx
@@ -19,7 +19,6 @@
 #include "Framework/Logger.h"
 #include "Align/AlignmentPoint.h"
 #include "Align/AlignableDetector.h"
-#include "ITSBase/GeometryTGeo.h"
 
 ClassImp(o2::align::AlignableSensorITS);
 
@@ -43,17 +42,64 @@ void AlignableSensorITS::prepareMatrixT2L()
 {
   // extract geometry T2L matrix
   TGeoHMatrix t2l;
-  float x, alp;
-  auto geom = o2::its::GeometryTGeo::Instance();
-  geom->getSensorXAlphaRefPlane(getVolID(), x, alp);
+  const auto& l2g = getMatrixL2GIdeal();
+  double locA[3] = {-100., 0., 0.}, locB[3] = {100., 0., 0.}, gloA[3], gloB[3];
+  l2g.LocalToMaster(locA, gloA);
+  l2g.LocalToMaster(locB, gloB);
+  double dx = gloB[0] - gloA[0], dy = gloB[1] - gloA[1];
+  double t = (gloB[0] * dx + gloB[1] * dy) / (dx * dx + dy * dy);
+  double xp = gloB[0] - dx * t, yp = gloB[1] - dy * t;
+  mX = std::sqrt(xp * xp + yp * yp);
+  float alp = std::atan2(yp, xp);
+  o2::math_utils::bringTo02Pi(alp);
   mAlp = alp;
-  mX = 0;
+  /* // this would proved x, alpha accounting for the corrections, we need ideal ones ?
+     float x, alp;
+     auto geom = o2::its::GeometryTGeo::Instance();
+     geom->getSensorXAlphaRefPlane(getVolID(), x, alp);
+     mAlp = alp;
+     mX = x;
+  */
   t2l.RotateZ(mAlp * RadToDeg()); // rotate in direction of normal to the sensor plane
-  const TGeoHMatrix* matL2G = base::GeometryManager::getMatrix(mDet->getDetID(), getSID());
-  const TGeoHMatrix& matL2Gi = matL2G->Inverse();
-  t2l.MultiplyLeft(&matL2Gi);
+  const TGeoHMatrix l2gi = l2g.Inverse();
+  t2l.MultiplyLeft(&l2gi);
   setMatrixT2L(t2l);
 }
 
+void AlignableSensorITS::prepareMatrixL2G(bool reco)
+{
+  // Note that for ITS2 it is NOT the same as GeometryManager::getMatrix() (i.e. that of the alignable volule)
+  // since we need the matrix of epitaxial layer and not the whole chip
+  auto geom = o2::its::GeometryTGeo::Instance();
+  const auto* m = geom->extractMatrixSensor(getVolID());
+  if (!m) {
+    LOGP(fatal, "Failed on :GeometryTGeo::extractMatrixSensor({})", getVolID());
+  }
+  reco ? setMatrixL2GReco(*m) : setMatrixL2G(*m);
+}
+
+void AlignableSensorITS::prepareMatrixL2GIdeal()
+{
+  // Note that for ITS2 it is NOT the same as GeometryManager::getOriginalMatrix (i.e. that of the alignable volule)
+  // since we need the matrix of epitaxial layer and not the whole chip
+  auto geom = o2::its::GeometryTGeo::Instance();
+  TGeoHMatrix mtmp;
+  if (!base::GeometryManager::getOriginalMatrix(getSymName(), mtmp)) { // this is chip ideal matrix, not that of the epitaxial layer
+    LOG(fatal) << "Failed to find ideal L2G matrix for " << getSymName();
+  }
+  // we have to apply to it the difference between the aligner epitaxial layer matrix and that of the chip
+  const auto* malgSens = geom->extractMatrixSensor(getVolID());
+  if (!malgSens) {
+    LOGP(fatal, "Failed on :GeometryTGeo::extractMatrixSensor({})", getVolID());
+  }
+  const auto* malgChip = geom->getMatrix(getVolID());
+  // correct chip original matrix by the difference between aligneg chip and sensor matrices
+  // Sens_ideal = Chip_ideal * Chip_aligned^-1 * Sens_aligned
+  auto chAlignInv = malgChip->Inverse();
+  chAlignInv.Multiply(*malgSens);
+  mtmp.Multiply(chAlignInv);
+  setMatrixL2GIdeal(mtmp);
+}
+
 } // namespace align
 } // namespace o2
diff --git a/Detectors/Align/src/AlignableSensorTOF.cxx b/Detectors/Align/src/AlignableSensorTOF.cxx
index 8c0af4ea0aeaa..eb154382dec37 100644
--- a/Detectors/Align/src/AlignableSensorTOF.cxx
+++ b/Detectors/Align/src/AlignableSensorTOF.cxx
@@ -33,38 +33,24 @@ namespace align
 {
 
 //_________________________________________________________
-AlignableSensorTOF::AlignableSensorTOF(const char* name, int vid, int iid, int isec)
-  : AlignableSensor(name, vid, iid), fSector(isec)
+AlignableSensorTOF::AlignableSensorTOF(const char* name, int vid, int iid, int isec, Controller* ctr) : AlignableSensor(name, vid, iid, ctr), mSector(isec)
 {
   // def c-tor
 }
 
-//_________________________________________________________
-AlignableSensorTOF::~AlignableSensorTOF()
-{
-  // d-tor
-}
-
 //____________________________________________
 void AlignableSensorTOF::prepareMatrixT2L()
 {
   // extract from geometry T2L matrix
-  double alp = sector2Alpha(fSector);
+  double alp = math_utils::detail::sector2Angle(mSector);
+  mAlp = alp;
+  TGeoHMatrix t2l;
   double loc[3] = {0, 0, 0}, glo[3];
   getMatrixL2GIdeal().LocalToMaster(loc, glo);
-  double x = Sqrt(glo[0] * glo[0] + glo[1] * glo[1]);
-  TGeoHMatrix t2l;
-  t2l.SetDx(x);
+  mX = Sqrt(glo[0] * glo[0] + glo[1] * glo[1]);
   t2l.RotateZ(alp * RadToDeg());
-  const TGeoHMatrix& l2gi = getMatrixL2GIdeal().Inverse();
+  const TGeoHMatrix l2gi = getMatrixL2GIdeal().Inverse();
   t2l.MultiplyLeft(&l2gi);
-  /*
-  const TGeoHMatrix* t2l = AliGeomManager::GetTracking2LocalMatrix(getVolID());
-  if (!t2l) {
-    Print("long");
-    AliFatalF("Failed to find T2L matrix for VID:%d %s",getVolID(),getSymName());
-  }
-  */
   setMatrixT2L(t2l);
   //
 }
diff --git a/Detectors/Align/src/AlignableSensorTPC.cxx b/Detectors/Align/src/AlignableSensorTPC.cxx
index 87212ed9a2761..87b3923633f77 100644
--- a/Detectors/Align/src/AlignableSensorTPC.cxx
+++ b/Detectors/Align/src/AlignableSensorTPC.cxx
@@ -10,62 +10,54 @@
 // or submit itself to any jurisdiction.
 
 /// @file   AlignableSensorTPC.h
-/// @author ruben.shahoyan@cern.ch, michael.lettrich@cern.ch
-/// @since  2021-02-01
-/// @brief  TPC sensor (chamber)
+/// @author ruben.shahoyan@cern.ch
+/// @brief  TPC sensor
 
 #include "Align/AlignableSensorTPC.h"
+#include "Align/AlignableDetectorTPC.h"
 #include "Align/utils.h"
 #include "Framework/Logger.h"
-//#include "AliTrackPointArray.h"
-//#include "AliESDtrack.h"
 #include "Align/AlignmentPoint.h"
-#include "Align/AlignableDetector.h"
-
-ClassImp(o2::align::AlignableSensorTPC)
-
-  using namespace o2::align::utils;
-using namespace TMath;
 
 namespace o2
 {
 namespace align
 {
+using namespace o2::align::utils;
+using namespace TMath;
 
 //_________________________________________________________
-AlignableSensorTPC::AlignableSensorTPC(const char* name, int vid, int iid, int isec)
-  : AlignableSensor(name, vid, iid), fSector(isec)
+AlignableSensorTPC::AlignableSensorTPC(const char* name, int vid, int iid, int isec, Controller* ctr) : AlignableSensor(name, vid, iid, ctr), mSector(isec)
 {
   // def c-tor
 }
 
-//_________________________________________________________
-AlignableSensorTPC::~AlignableSensorTPC()
+//____________________________________________
+void AlignableSensorTPC::prepareMatrixL2G(bool reco)
+{
+  double alp = math_utils::detail::sector2Angle(mSector % 18);
+  TGeoHMatrix m;
+  m.RotateZ(alp * RadToDeg());
+  reco ? setMatrixL2GReco(m) : setMatrixL2G(m);
+}
+
+//____________________________________________
+void AlignableSensorTPC::prepareMatrixL2GIdeal()
 {
-  // d-tor
+  double alp = math_utils::detail::sector2Angle(mSector % 18);
+  TGeoHMatrix m;
+  m.RotateZ(alp * RadToDeg());
+  setMatrixL2GIdeal(m);
 }
 
 //____________________________________________
 void AlignableSensorTPC::prepareMatrixT2L()
 {
-  // extract from geometry T2L matrix
-  double alp = sector2Alpha(fSector);
-  double loc[3] = {0, 0, 0}, glo[3];
-  getMatrixL2GIdeal().LocalToMaster(loc, glo);
-  double x = Sqrt(glo[0] * glo[0] + glo[1] * glo[1]);
-  TGeoHMatrix t2l;
-  t2l.SetDx(x);
-  t2l.RotateZ(alp * RadToDeg());
-  const TGeoHMatrix& l2gi = getMatrixL2GIdeal().Inverse();
-  t2l.MultiplyLeft(&l2gi);
-  /*
-  const TGeoHMatrix* t2l = AliGeomManager::GetTracking2LocalMatrix(getVolID());
-  if (!t2l) {
-    Print("long");
-    AliFatalF("Failed to find T2L matrix for VID:%d %s",getVolID(),getSymName());
-  }
-  */
-  setMatrixT2L(t2l);
+  // local and tracking matrices are the same
+  double alp = math_utils::detail::sector2Angle(mSector % 18);
+  mAlp = alp;
+  TGeoHMatrix m;
+  setMatrixT2L(m);
   //
 }
 
diff --git a/Detectors/Align/src/AlignableSensorTRD.cxx b/Detectors/Align/src/AlignableSensorTRD.cxx
index 9181d4bb97ccd..aa52567a36fdf 100644
--- a/Detectors/Align/src/AlignableSensorTRD.cxx
+++ b/Detectors/Align/src/AlignableSensorTRD.cxx
@@ -15,59 +15,90 @@
 /// @brief  TRD sensor
 
 #include "Align/AlignableSensorTRD.h"
-//#include "AliTRDgeometry.h"
 #include "Align/AlignableDetectorTRD.h"
 #include "Align/utils.h"
 #include "Framework/Logger.h"
 #include "Align/AlignmentPoint.h"
-//#include "AliTrackPointArray.h"
-//#include "AliESDtrack.h"
-//#include "AliTrackerBase.h"
-
-ClassImp(o2::align::AlignableSensorTRD)
-
-  using namespace o2::align::utils;
-using namespace TMath;
+#include "DetectorsBase/GeometryManager.h"
 
 namespace o2
 {
 namespace align
 {
+using namespace o2::align::utils;
+using namespace TMath;
 
 //_________________________________________________________
-AlignableSensorTRD::AlignableSensorTRD(const char* name, int vid, int iid, int isec)
-  : AlignableSensor(name, vid, iid), fSector(isec)
+AlignableSensorTRD::AlignableSensorTRD(const char* name, int vid, int iid, int isec, Controller* ctr) : AlignableSensor(name, vid, iid, ctr), mSector(isec)
 {
   // def c-tor
 }
 
-//_________________________________________________________
-AlignableSensorTRD::~AlignableSensorTRD()
+//____________________________________________
+void AlignableSensorTRD::prepareMatrixClAlg()
+{
+  // prepare alignment matrix in the pseudo-LOCAL frame of TRD (account that the chamber has extra X,Y rotations
+  TGeoHMatrix ma = getMatrixL2GIdeal().Inverse();
+  ma *= getMatrixL2G();
+  setMatrixClAlg(ma);
+  //
+}
+
+//____________________________________________
+void AlignableSensorTRD::prepareMatrixClAlgReco()
 {
-  // d-tor
+  // prepare alignment matrix in the pseudo-LOCAL frame of TRD (account that the chamber has extra X,Y rotations
+  TGeoHMatrix ma = getMatrixL2GIdeal().Inverse();
+  ma *= getMatrixL2G();
+  setMatrixClAlgReco(ma);
+  //
+}
+
+//____________________________________________
+void AlignableSensorTRD::prepareMatrixL2GIdeal()
+{
+  TGeoHMatrix Rxy;
+  Rxy.RotateX(-90);
+  Rxy.RotateY(-90);
+  TGeoHMatrix mtmp;
+  if (!base::GeometryManager::getOriginalMatrix(getSymName(), mtmp)) {
+    LOG(fatal) << "Failed to find ideal L2G matrix for " << getSymName();
+  }
+  mtmp *= Rxy;
+  setMatrixL2GIdeal(mtmp);
+}
+
+//____________________________________________
+void AlignableSensorTRD::prepareMatrixL2G(bool reco)
+{
+  TGeoHMatrix Rxy;
+  Rxy.RotateX(-90);
+  Rxy.RotateY(-90);
+  const char* path = getSymName();
+  const TGeoHMatrix* l2g = nullptr;
+  if (!gGeoManager->GetAlignableEntry(path) || !(l2g = base::GeometryManager::getMatrix(path))) {
+    LOGP(fatal, "Failed to find L2G matrix for {}alignable {} -> {}", gGeoManager->GetAlignableEntry(path) ? "" : "non-", path, (void*)l2g);
+  }
+  TGeoHMatrix mtmp = *l2g;
+  mtmp *= Rxy;
+  reco ? setMatrixL2GReco(mtmp) : setMatrixL2G(mtmp);
 }
 
 //____________________________________________
 void AlignableSensorTRD::prepareMatrixT2L()
 {
   // extract from geometry T2L matrix
-  double alp = sector2Alpha(fSector);
-  double loc[3] = {0, 0, 0}, glo[3];
-  getMatrixL2GIdeal().LocalToMaster(loc, glo);
-  double x = Sqrt(glo[0] * glo[0] + glo[1] * glo[1]);
-  TGeoHMatrix t2l;
-  t2l.SetDx(x);
-  t2l.RotateZ(alp * RadToDeg());
-  const TGeoHMatrix& l2gi = getMatrixL2GIdeal().Inverse();
-  t2l.MultiplyLeft(&l2gi);
-  /*
-  const TGeoHMatrix* t2l = AliGeomManager::GetTracking2LocalMatrix(getVolID());
-  if (!t2l) {
-    Print("long");
-    AliFatalF("Failed to find T2L matrix for VID:%d %s",getVolID(),getSymName());
-  }
-  */
+  double alp = math_utils::detail::sector2Angle(mSector);
+  mAlp = alp;
+  TGeoHMatrix Rs;
+  Rs.RotateZ(-alp * TMath::RadToDeg());
+  TGeoHMatrix m0 = getMatrixL2GIdeal();
+  m0.MultiplyLeft(Rs);
+  TGeoHMatrix t2l = m0.Inverse();
   setMatrixT2L(t2l);
+  double loc[3] = {0, 0, 0}, glo[3];
+  t2l.MasterToLocal(loc, glo);
+  mX = glo[0];
   //
 }
 
@@ -82,29 +113,28 @@ void AlignableSensorTRD::dPosTraDParCalib(const AlignmentPoint* pnt, double* der
   if (!parent) { // TRD detector global calibration
     //
     switch (calibID) {
-      case AlignableDetectorTRD::kCalibNRCCorrDzDtgl: { // correction for Non-Crossing tracklets Z,Y shift: Z -> Z + calib*tgl, Y -> Y + calib*tgl*tilt*sign(tilt);
-        double sgYZ = pnt->getYZErrTracking()[1];       // makes sense only for nonRC tracklets
-        if (Abs(sgYZ) > 0.01) {
+      case AlignableDetectorTRD::CalibNRCCorrDzDtgl: // correction for Non-Crossing tracklets Z,Y shift: Z -> Z + calib*tgl, Y -> Y + calib*tgl*tilt*sign(tilt);
+      {
+        double sgYZ = pnt->getYZErrTracking()[1]; // makes sense only for nonRC tracklets
+        if (std::abs(sgYZ) > 0.01) {
           const double kTilt = 2. * TMath::DegToRad();
           deriv[2] = pnt->getTrParamWSA()[AlignmentPoint::kParTgl];
           deriv[1] = deriv[2] * Sign(kTilt, sgYZ);
         }
         break;
       }
-        //
-      case AlignableDetectorTRD::kCalibDVT: { // correction for bias in VdriftT
+      case AlignableDetectorTRD::CalibDVT: // correction for bias in VdriftT
+      {
         // error in VdriftT equivalent to shift in X at which Y measurement is evaluated
         // Y -> Y + dVdriftT * tg_phi, where tg_phi is the slope of the track in YX plane
-        double snp = pnt->getTrParamWSA(AlignmentPoint::kParSnp), slpY = snp / Sqrt((1 - snp) * (1 + snp));
+        double snp = pnt->getTrParamWSA(AlignmentPoint::kParSnp), slpY = snp / std::sqrt((1.f - snp) * (1.f + snp));
         deriv[1] = slpY;
         break;
       }
-
       default:
         break;
-    };
+    }
   }
-  //
 }
 
 } // namespace align
diff --git a/Detectors/Align/src/AlignableVolume.cxx b/Detectors/Align/src/AlignableVolume.cxx
index e69cc3f3b6bfc..73d888dc1559b 100644
--- a/Detectors/Align/src/AlignableVolume.cxx
+++ b/Detectors/Align/src/AlignableVolume.cxx
@@ -42,10 +42,6 @@
   G_0*delta_0 ... G^-1_{j-2}*G_{j-1}*delta_{j-1}*G^-1_{j-1}*G_j*delta_j =
   Delta_0 * Delta_{1} ... Delta_{j-1}*Delta_{j}... * G_j
 
-  -> Delta_j = G'_{j-1} * G^-1_{j-1} * G_j * G'^-1_j
-  where G and G' are modified and original L2G matrices
-
-
   From this by induction one gets relation between local and global deltas:
 
   Delta_j = Z_j * delta_j * Z^-1_j
@@ -98,8 +94,8 @@
 
 #include "Align/Controller.h"
 #include "Align/AlignableVolume.h"
-#include "Align/DOFStatistics.h"
 #include "Align/GeometricalConstraint.h"
+#include "Align/AlignConfig.h"
 #include "DetectorsCommonDataFormats/AlignParam.h"
 #include "DetectorsBase/GeometryManager.h"
 #include "Align/utils.h"
@@ -111,6 +107,7 @@
 #include 
 #include 
 #include 
+#include 
 
 ClassImp(o2::align::AlignableVolume);
 
@@ -136,7 +133,7 @@ AlignableVolume::AlignableVolume(const char* symname, int iid, Controller* ctr)
   if (!ctr) {
     LOG(fatal) << "Controller has to be provided :" << symname;
   }
-  setVolID(0); // volumes have no VID, unless it is sensor
+  setVolID(-1); // volumes have no VID, unless it is sensor
   setNDOFs(kNDOFGeom);
   setFreeDOFPattern(sDefGeomFree);
 }
@@ -153,7 +150,7 @@ void AlignableVolume::delta2Matrix(TGeoHMatrix& deltaM, const double* delta) con
 {
   // prepare delta matrix for the volume from its
   // local delta vector (AliAlignObj convension): dx,dy,dz,,theta,psi,phi
-  const double *tr = &delta[0], *rt = &delta[3]; // translation(cm) and rotation(degree)
+  const double *tr = &delta[0], *rt = &delta[3]; // translation(cm) and rotation(radians)
 
   //    AliAlignObjParams tempAlignObj;
   //    tempAlignObj.SetRotation(rt[0], rt[1], rt[2]);
@@ -294,6 +291,9 @@ void AlignableVolume::prepareMatrixL2G(bool reco)
 {
   // extract from geometry L2G matrix, depending on reco flag, set it as a reco-time
   // or current alignment matrix
+  if (isDummyEnvelope() || isDummy()) {
+    return; // unit matrix
+  }
   const char* path = getSymName();
   if (gGeoManager->GetAlignableEntry(path)) {
     const TGeoHMatrix* l2g = base::GeometryManager::getMatrix(path);
@@ -356,7 +356,7 @@ void AlignableVolume::prepareMatrixT2L()
   }
   //
   mAlp = TMath::ATan2(tot[1], tot[0]);
-  utils::bringToPiPM(mAlp);
+  math_utils::detail::bringToPMPi(mAlp);
   //
   mX = TMath::Sqrt(tot[0] * tot[0] + tot[1] * tot[1]);
   //
@@ -417,7 +417,7 @@ void AlignableVolume::calcFree(bool condFix)
   mNDOFsFree = mNDOFGeomFree = 0;
   for (int i = 0; i < mNDOFs; i++) {
     if (!isFreeDOF(i)) {
-      if (condFix) {
+      if (condFix && varsSet()) {
         setParErr(i, -999);
       }
       continue;
@@ -458,15 +458,14 @@ bool AlignableVolume::isCondDOF(int i) const
 }
 
 //______________________________________________________
-int AlignableVolume::finalizeStat(DOFStatistics& st)
+int AlignableVolume::finalizeStat()
 {
   // finalize statistics on processed points
   mNProcPoints = 0;
   for (int ich = getNChildren(); ich--;) {
     AlignableVolume* child = getChild(ich);
-    mNProcPoints += child->finalizeStat(st);
+    mNProcPoints += child->finalizeStat();
   }
-  fillDOFStat(st);
   return mNProcPoints;
 }
 
@@ -479,6 +478,7 @@ void AlignableVolume::writePedeInfo(FILE* parOut, const Option_t* opt) const
          kOnOn };
   const char* comment[3] = {"  ", "! ", "!!"};
   const char* kKeyParam = "parameter";
+  const char* kKeyMeas = "measurement";
   TString opts = opt;
   opts.ToLower();
   bool showDef = opts.Contains("d"); // show free DOF even if not preconditioned
@@ -508,8 +508,8 @@ void AlignableVolume::writePedeInfo(FILE* parOut, const Option_t* opt) const
   }
   //
   if (nCond || showDef || showFix || showNam) {
-    fprintf(parOut, "%s%s %s\t\tDOF/Free: %d/%d (%s) %s\n", comment[cmt], kKeyParam, comment[kOnOn],
-            getNDOFs(), getNDOFsFree(), sFrameName[mVarFrame], GetName());
+    fprintf(parOut, "%s%s %s\t\tDOF/Free: %d/%d (%s) %s id : %d | Stat: %d\n", comment[cmt], kKeyParam, comment[kOnOn],
+            getNDOFs(), getNDOFsFree(), sFrameName[mVarFrame], GetName(), getVolID(), getNProcessedPoints());
   }
   //
   if (nCond || showDef || showFix) {
@@ -528,7 +528,13 @@ void AlignableVolume::writePedeInfo(FILE* parOut, const Option_t* opt) const
       } // free-unconditioned: print commented if asked
       //
       fprintf(parOut, "%s %9d %+e %+e\t%s %s p%d\n", comment[cmt], getParLab(i),
-              getParVal(i), getParErr(i), comment[kOnOn], isFreeDOF(i) ? "  " : "FX", i);
+              -getParVal(i), getParErr(i), comment[kOnOn], isFreeDOF(i) ? "  " : "FX", i);
+    }
+    // do we consider some DOFs of this volume as measured
+    for (int i = 0; i < mNDOFs; i++) {
+      cmt = isMeasuredDOF(i) ? kOff : kOn;
+      fprintf(parOut, "%s%s %+e %+e\n", comment[cmt], kKeyMeas, -getParVal(i), getParErr(i));
+      fprintf(parOut, "%s %d 1.0\n", comment[cmt], getParLab(i));
     }
     fprintf(parOut, "\n");
   }
@@ -541,18 +547,38 @@ void AlignableVolume::writePedeInfo(FILE* parOut, const Option_t* opt) const
   //
 }
 
+//______________________________________________________
+void AlignableVolume::writeLabeledPedeResults(FILE* parOut) const
+{
+  // write parameters with labels
+  for (int i = 0; i < mNDOFs; i++) {
+    fprintf(parOut, "%9d %+e %+e\t! %s %d:%s vol:%d %s %s\n", getParLab(i), -getParVal(i), getParErr(i), GetName(), i, sDOFName[i], getVolID(),
+            isFreeDOF(i) ? "   " : "FXU", isZeroAbs(getParVal(i)) ? "FXP" : "   ");
+  }
+  // children volume
+  int nch = getNChildren();
+  //
+  for (int ich = 0; ich < nch; ich++) {
+    getChild(ich)->writeLabeledPedeResults(parOut);
+  }
+  //
+}
+
 //_________________________________________________________________
-void AlignableVolume::createGloDeltaMatrix(TGeoHMatrix& deltaM) const
+bool AlignableVolume::createGloDeltaMatrix(TGeoHMatrix& deltaM) const
 {
   // Create global matrix deltaM from global array containing corrections.
   // This deltaM does not account for eventual prealignment
   // Volume knows if its variation was done in TRA or LOC frame
   //
-  createLocDeltaMatrix(deltaM);
-  const TGeoHMatrix& l2g = getMatrixL2G();
-  const TGeoHMatrix& l2gi = l2g.Inverse();
-  deltaM.Multiply(&l2gi);
-  deltaM.MultiplyLeft(&l2g);
+  if (createLocDeltaMatrix(deltaM)) { // do multiplications only if the matrix is non-trivial
+    const TGeoHMatrix& l2g = getMatrixL2G();
+    const TGeoHMatrix l2gi = l2g.Inverse();
+    deltaM.Multiply(&l2gi);
+    deltaM.MultiplyLeft(&l2g);
+    return true;
+  }
+  return false;
   //
 }
 /*
@@ -656,30 +682,34 @@ void AlignableVolume::createPreLocDeltaMatrix(TGeoHMatrix& deltaM) const
 }
 
 //_________________________________________________________________
-void AlignableVolume::createLocDeltaMatrix(TGeoHMatrix& deltaM) const
+bool AlignableVolume::createLocDeltaMatrix(TGeoHMatrix& deltaM) const
 {
   // Create local matrix deltaM from global array containing corrections.
   // This deltaM does not account for eventual prealignment
   // Volume knows if its variation was done in TRA or LOC frame
   auto pars = getParVals();
-  double corr[kNDOFGeom];
+  double corr[kNDOFGeom] = {0.};
+  int nonZero = 0;
   for (int i = kNDOFGeom; i--;) {
-    corr[i] = pars[i];
+    if (pars[i] != 0.) {
+      nonZero++;
+      corr[i] = pars[i];
+    }
   } // we need doubles
   delta2Matrix(deltaM, corr);
-  if (isFrameTRA()) { // we need corrections in local frame!
+  if (isFrameTRA() && nonZero) { // we need corrections in local frame!
     // l' = T2L * delta_t * t = T2L * delta_t * T2L^-1 * l = delta_l * l
     // -> delta_l = T2L * delta_t * T2L^-1
     const TGeoHMatrix& t2l = getMatrixT2L();
-    const TGeoHMatrix& t2li = t2l.Inverse();
+    const TGeoHMatrix t2li = t2l.Inverse();
     deltaM.Multiply(&t2li);
     deltaM.MultiplyLeft(&t2l);
   }
-  //
+  return nonZero;
 }
 
 //_________________________________________________________________
-void AlignableVolume::createAlignmenMatrix(TGeoHMatrix& alg) const
+void AlignableVolume::createAlignmenMatrix(TGeoHMatrix& alg, const TGeoHMatrix* envelopeDelta) const
 {
   // create final alignment matrix, accounting for eventual prealignment
   //
@@ -693,10 +723,22 @@ void AlignableVolume::createAlignmenMatrix(TGeoHMatrix& alg) const
   // but this creates precision problem.
   // Therefore we use explicitly cached Deltas from prealignment object.
   //
-  createGloDeltaMatrix(alg);
-  //
+  // If (parent) envelopeDelta is provided, it is simply added on top of its proper global delta matrix
+
   const AlignableVolume* par = getParent();
-  if (par) {
+  bool nonTrivial = createGloDeltaMatrix(alg);
+  if (envelopeDelta) {
+    if (nonTrivial) {
+      alg.MultiplyLeft(envelopeDelta);
+    } else {
+      alg = *envelopeDelta;
+    }
+    nonTrivial = true;
+  }
+  // Account parent matrices only if the alg matrix is non-trivial.
+  // Also, if parent is dummy envelope, it is already accounted via envelopeDelta
+  if (nonTrivial && (par && !par->isDummyEnvelope())) {
+    // if envelopeDelta is provided, then there should be
     TGeoHMatrix dchain;
     while (par) {
       dchain.MultiplyLeft(&par->getGlobalDeltaRef());
@@ -748,18 +790,35 @@ void AlignableVolume::createAlignmenMatrix(TGeoHMatrix& alg) const
 */
 
 //_________________________________________________________________
-void AlignableVolume::createAlignmentObjects(std::vector& arr) const
+void AlignableVolume::createAlignmentObjects(std::vector& arr, const TGeoHMatrix* envelopeDelta) const
 {
   // add to supplied array alignment object for itself and children
+  if (isDummy()) {
+    LOGP(info, "Skipping alignment object creation for dummy volume {}", GetName());
+    return;
+  }
   TGeoHMatrix algM;
-  createAlignmenMatrix(algM);
-  //  new (parr[parr.GetEntriesFast()]) AliAlignObjParams(GetName(), getVolID(), algM, true);
-  const double* translation = algM.GetTranslation();
-  const double* rotation = algM.GetRotationMatrix();
-  arr.emplace_back(getSymName(), getVolID(), translation[0], translation[1], translation[2], rotation[0], rotation[1], rotation[2], true);
+  bool nonTrivial = false;
+  if (!isDummyEnvelope()) {
+    createAlignmenMatrix(algM, envelopeDelta);
+    arr.emplace_back(getSymName(), getVolID(), algM, true).rectify(AlignConfig::Instance().alignParamZero);
+    envelopeDelta = nullptr;
+  } else {
+    nonTrivial = createGloDeltaMatrix(algM);
+    if (envelopeDelta) {
+      if (nonTrivial) {
+        algM.MultiplyLeft(envelopeDelta);
+      } else {
+        algM = *envelopeDelta;
+      }
+      nonTrivial = true;
+    }
+    envelopeDelta = &algM;
+  }
+
   int nch = getNChildren();
   for (int ich = 0; ich < nch; ich++) {
-    getChild(ich)->createAlignmentObjects(arr);
+    getChild(ich)->createAlignmentObjects(arr, nonTrivial ? envelopeDelta : nullptr);
   }
 }
 
@@ -840,46 +899,37 @@ const char* AlignableVolume::getDOFName(int i) const
   return getGeomDOFName(i);
 }
 
-//______________________________________________________
-void AlignableVolume::fillDOFStat(DOFStatistics& h) const
-{
-  // fill statistics info hist
-  int ndf = getNDOFs();
-  int dof0 = getFirstParGloID();
-  int stat = getNProcessedPoints();
-  for (int idf = 0; idf < ndf; idf++) {
-    int dof = idf + dof0;
-    h.addStat(dof, stat);
-  }
-}
-
 //________________________________________
-void AlignableVolume::addAutoConstraints(TObjArray* constrArr)
+void AlignableVolume::addAutoConstraints()
 {
   // adds automatic constraints
   int nch = getNChildren();
   //
   if (hasChildrenConstraint()) {
-    GeometricalConstraint* constr = new GeometricalConstraint();
-    constr->setConstrainPattern(mConstrChild);
-    constr->setParent(this);
+    auto& cstr = getController()->getConstraints().emplace_back();
+    cstr.setConstrainPattern(mConstrChild);
+    cstr.setParent(this);
     for (int ich = nch; ich--;) {
-      AlignableVolume* child = getChild(ich);
+      auto child = getChild(ich);
       if (child->getExcludeFromParentConstraint()) {
         continue;
       }
-      constr->addChild(child);
+      cstr.addChild(child);
     }
-    if (constr->getNChildren()) {
-      constrArr->Add(constr);
-    } else {
-      delete constr;
+    if (!cstr.getNChildren()) {
+      getController()->getConstraints().pop_back(); // destroy
     }
   }
   for (int ich = 0; ich < nch; ich++) {
-    getChild(ich)->addAutoConstraints(constrArr);
+    getChild(ich)->addAutoConstraints();
   }
 }
 
+//________________________________________
+bool AlignableVolume::isNameMatching(const std::string& regexStr) const
+{
+  return (!regexStr.empty() && std::regex_match(getSymName(), std::regex{regexStr}));
+}
+
 } // namespace align
 } // namespace o2
diff --git a/Detectors/Align/src/AlignmentPoint.cxx b/Detectors/Align/src/AlignmentPoint.cxx
index 72b41e49d6820..fb6e6863ca465 100644
--- a/Detectors/Align/src/AlignmentPoint.cxx
+++ b/Detectors/Align/src/AlignmentPoint.cxx
@@ -38,7 +38,7 @@ void AlignmentPoint::init()
   if (!isZeroPos(mErrYZTracking[0] + mErrYZTracking[2])) {
     //
     // is there a correlation?
-    if (smallerAbs(mErrYZTracking[1] * mErrYZTracking[1] / (mErrYZTracking[0] * mErrYZTracking[2]), kCorrToler)) {
+    if (math_utils::detail::abs(mErrYZTracking[1] * mErrYZTracking[1] / (mErrYZTracking[0] * mErrYZTracking[2])) < kCorrToler) {
       mCosDiagErr = 1.;
       mSinDiagErr = 0.;
       mErrDiag[0] = mErrYZTracking[0];
@@ -184,7 +184,7 @@ void AlignmentPoint::dumpCoordinates() const
   print3d(xyz); // track after mat corr
 
   printf("%+.4f ", mAlphaSens);
-  printf("%+.4e ", getXPoint());
+  printf("%+.4e ", getXTracking());
   printf("%+.4e ", getYTracking());
   printf("%+.4e ", getZTracking());
   //
@@ -216,8 +216,8 @@ bool AlignmentPoint::isAfter(const AlignmentPoint& pnt) const
   // 1) for tracks from collision: range in decreasing tracking X
   // 2) for cosmic tracks: upper leg (pnt->isInvDir()==true) ranged in increasing X
   //                       lower leg - in decreasing X
-  double x = getXPoint();
-  double xp = pnt.getXPoint();
+  double x = getXTracking();
+  double xp = pnt.getXTracking();
   if (!isInvDir()) {        // track propagates from low to large X via this point
     if (!pnt.isInvDir()) {  // via this one also
       return x > xp ? -1 : 1;
@@ -240,7 +240,7 @@ void AlignmentPoint::getXYZGlo(double r[3]) const
   // position in lab frame
   double cs = TMath::Cos(mAlphaSens);
   double sn = TMath::Sin(mAlphaSens);
-  double x = getXPoint();
+  double x = getXTracking();
   r[0] = x * cs - getYTracking() * sn;
   r[1] = x * sn + getYTracking() * cs;
   r[2] = getZTracking();
@@ -260,7 +260,7 @@ double AlignmentPoint::getPhiGlo() const
 int AlignmentPoint::getAliceSector() const
 {
   // get global sector ID corresponding to this point phi
-  return phi2Sector(getPhiGlo());
+  return math_utils::detail::angle2Sector(getPhiGlo());
 }
 
 //__________________________________________________________________
@@ -366,7 +366,7 @@ void AlignmentPoint::getTrWSA(trackParam_t& etp) const
   params_t tmp;
   std::copy(std::begin(mTrParamWSA), std::end(mTrParamWSA), std::begin(tmp));
 
-  etp.set(getXPoint(), getAlphaSens(), tmp, covDum);
+  etp.set(getXTracking(), getAlphaSens(), tmp, covDum);
 }
 
 //__________________________________________________________________
@@ -382,7 +382,7 @@ void AlignmentPoint::getTrWSB(trackParam_t& etp) const
   params_t tmp;
   std::copy(std::begin(mTrParamWSB), std::end(mTrParamWSB), std::begin(tmp));
 
-  etp.set(getXPoint(), getAlphaSens(), tmp, covDum);
+  etp.set(getXTracking(), getAlphaSens(), tmp, covDum);
 }
 
 } // namespace align
diff --git a/Detectors/Align/src/AlignmentTrack.cxx b/Detectors/Align/src/AlignmentTrack.cxx
index 387bd3e85949c..644ee07c64984 100644
--- a/Detectors/Align/src/AlignmentTrack.cxx
+++ b/Detectors/Align/src/AlignmentTrack.cxx
@@ -25,7 +25,10 @@
 #include 
 #include 
 #include 
+#include "MathUtils/SymMatrixSolver.h"
+#include "MathUtils/Utils.h"
 
+#define DEBUG 4
 using namespace o2::align::utils;
 using namespace o2::base;
 using namespace TMath;
@@ -127,19 +130,20 @@ bool AlignmentTrack::calcResidDeriv(double* params)
   //
   int np = getNPoints();
   //
+  const auto& algConf = AlignConfig::Instance();
   // collision track or cosmic lower leg
   if (!calcResidDeriv(params, mNeedInv[0], getInnerPointID(), 0)) {
-#if DEBUG > 3
-    LOG(warn) << "Failed on derivatives calculation 0";
-#endif
+    if (algConf.verbose > 2) {
+      LOG(warn) << "Failed on derivatives calculation 0";
+    }
     return false;
   }
   //
   if (isCosmic()) { // cosmic upper leg
     if (!calcResidDeriv(params, mNeedInv[1], getInnerPointID() + 1, np - 1)) {
-#if DEBUG > 3
-      LOG(warn) << "Failed on derivatives calculation 0";
-#endif
+      if (algConf.verbose > 2) {
+        LOG(warn) << "Failed on derivatives calculation 0";
+      }
     }
   }
   //
@@ -163,7 +167,8 @@ bool AlignmentTrack::calcResidDeriv(double* extendedParams, bool invert, int pFr
   // The derivatives are calculated using Richardson extrapolation
   // (like http://root.cern.ch/root/html/ROOT__Math__RichardsonDerivator.html)
   //
-  trackParam_t probD[kNRDClones]; // use this to vary supplied param for derivative calculation
+  const auto& algConf = AlignConfig::Instance();
+  trackPar_t probD[kNRDClones]; // use this to vary supplied param for derivative calculation
   double varDelta[kRichardsonN];
   const int kInvElem[kNKinParBON] = {-1, 1, 1, -1, -1};
   //
@@ -177,18 +182,20 @@ bool AlignmentTrack::calcResidDeriv(double* extendedParams, bool invert, int pFr
     delta[kParQ2Pt] = kDelta[kParQ2Pt] * Abs(getQ2Pt());
   }
   //
-  int pinc;
+  int pinc, signELoss = 0; // RS Validate for cosmic. Propagation is done from inner point to outer one:
+  // energy loss is applied for collision tracks and cosmic lower leg, compensated for cosmic upper leg
   if (pTo > pFrom) { // fit in points decreasing order: cosmics upper leg
     pTo++;
     pinc = 1;
+    signELoss = 1; // eloss is corrected
   } else { // fit in points increasing order: collision track or cosmics lower leg
     pTo--;
     pinc = -1;
+    signELoss = -1; // eloss is applied
   }
 
   // 1) derivative wrt trackParam_t parameters
   for (int ipar = mNLocExtPar; ipar--;) {
-
     setParams(probD, kNRDClones, getX(), getAlpha(), extendedParams, true);
     if (invert) {
       for (int ic = kNRDClones; ic--;) {
@@ -206,15 +213,13 @@ bool AlignmentTrack::calcResidDeriv(double* extendedParams, bool invert, int pFr
     // propagate varied tracks to each point
     for (int ip = pFrom; ip != pTo; ip += pinc) { // points are ordered against track direction
       AlignmentPoint* pnt = getPoint(ip);
-      if (!propagateParamToPoint(probD, kNRDClones, pnt)) {
+      // we propagate w/o mat. corrections, they will be accounted in applyMatCorr
+      if (!propagateParamToPoint(probD, kNRDClones, pnt, algConf.maxStep, algConf.maxSnp, MatCorrType::USEMatCorrNONE, signELoss)) {
         return false;
       }
-      //      if (pnt->containsMaterial()) { // apply material corrections
       if (!applyMatCorr(probD, kNRDClones, extendedParams, pnt)) {
         return false;
       }
-      //      }
-      //
       if (pnt->containsMeasurement()) {
         int offsDer = ip * mNLocPar + ipar;
         richardsonDeriv(probD, varDelta, pnt, mDResDLoc[0][offsDer], mDResDLoc[1][offsDer]); // calculate derivatives
@@ -225,18 +230,15 @@ bool AlignmentTrack::calcResidDeriv(double* extendedParams, bool invert, int pFr
       }
     } // loop over points
   }   // loop over ExtTrackParam parameters
-  //
   // 2) now vary material effect related parameters: MS and eventually ELoss
-  //
   for (int ip = pFrom; ip != pTo; ip += pinc) { // points are ordered against track direction
     AlignmentPoint* pnt = getPoint(ip);
-    //
     // global derivatives at this point
     if (pnt->containsMeasurement() && !calcResidDerivGlo(pnt)) {
-#if DEBUG > 3
-      AliWarningF("Failed on global derivatives calculation at point %d", ip);
-      pnt->print("meas");
-#endif
+      if (algConf.verbose > 2) {
+        LOGF(warn, "Failed on global derivatives calculation at point %d", ip);
+        pnt->print(AlignmentPoint::kMeasurementBit);
+      }
       return false;
     }
     //
@@ -250,7 +252,11 @@ bool AlignmentTrack::calcResidDeriv(double* extendedParams, bool invert, int pFr
     // while the variation should be done for parameters in the frame where the vector
     // of material corrections has diagonal cov. matrix -> rotate the delta to this frame
     double deltaMatD[kNKinParBON];
-    pnt->diagMatCorr(delta, deltaMatD);
+    //    pnt->diagMatCorr(delta, deltaMatD);
+    for (int ipar = 0; ipar < nParFreeI; ipar++) {
+      double d = pnt->getMatCorrCov()[ipar];
+      deltaMatD[ipar] = d > 0. ? Sqrt(d) * 5 : delta[ipar];
+    }
     //
     //    printf("Vary %d [%+.3e %+.3e %+.3e %+.3e] ",ip,deltaMatD[0],deltaMatD[1],deltaMatD[2],deltaMatD[3]); pnt->print();
 
@@ -262,7 +268,7 @@ bool AlignmentTrack::calcResidDeriv(double* extendedParams, bool invert, int pFr
       // We will vary the tracks starting from the original parameters propagated to given point
       // and stored there (before applying material corrections for this point)
       //
-      setParams(probD, kNRDClones, pnt->getXPoint(), pnt->getAlphaSens(), pnt->getTrParamWSB(), false);
+      setParams(probD, kNRDClones, pnt->getXTracking(), pnt->getAlphaSens(), pnt->getTrParamWSB(), false);
       // no need for eventual track inversion here: if needed, this is already done in ParamWSB
       //
       int offsIP = offsI + ipar; // parameter entry in the extendedParams array
@@ -299,7 +305,7 @@ bool AlignmentTrack::calcResidDeriv(double* extendedParams, bool invert, int pFr
 
         // printf("  DerFor:%d ",jp); pntJ->print();
 
-        if (!propagateParamToPoint(probD, kNRDClones, pntJ)) {
+        if (!propagateParamToPoint(probD, kNRDClones, pntJ, algConf.maxStep, algConf.maxSnp, MatCorrType::USEMatCorrNONE, signELoss)) {
           return false;
         }
         //
@@ -332,7 +338,7 @@ bool AlignmentTrack::calcResidDerivGlo(AlignmentPoint* pnt)
   const AlignableVolume* vol = sens;
   // precalculated track parameters
   double snp = pnt->getTrParamWSA(kParSnp), tgl = pnt->getTrParamWSA(kParTgl);
-  // precalculate track slopes to account tracking X veriation
+  // precalculate track slopes to account tracking X variation
   // these are coeffs to translate deltaX of the point to deltaY and deltaZ of track
   double cspi = 1. / Sqrt((1 - snp) * (1 + snp)), slpY = snp * cspi, slpZ = tgl * cspi;
   //
@@ -415,19 +421,20 @@ bool AlignmentTrack::calcResiduals(const double* extendedParams)
   mChi2 = 0;
   mNDF = 0;
   //
+  const auto& algConf = AlignConfig::Instance();
   // collision track or cosmic lower leg
   if (!calcResiduals(extendedParams, mNeedInv[0], getInnerPointID(), 0)) {
-#if DEBUG > 3
-    LOG(warn) << "Failed on residuals calculation 0";
-#endif
+    if (algConf.verbose > 2) {
+      LOG(warn) << "Failed on residuals calculation 0";
+    }
     return false;
   }
   //
   if (isCosmic()) { // cosmic upper leg
     if (!calcResiduals(extendedParams, mNeedInv[1], getInnerPointID() + 1, np - 1)) {
-#if DEBUG > 3
-      LOG(warn) << "Failed on residuals calculation 1";
-#endif
+      if (algConf.verbose > 2) {
+        LOG(warn) << "Failed on residuals calculation 1";
+      }
       return false;
     }
   }
@@ -455,31 +462,32 @@ bool AlignmentTrack::calcResiduals(const double* extendedParams, bool invert, in
   if (invert) {
     probe.invert();
   }
-  int pinc;
+  int pinc, signELoss = 0; // RS Validate for cosmic. Propagation is done from inner point to outer one:
+  // energy loss is applied for collision tracks and cosmic lower leg, compensated for cosmic upper leg
   if (pTo > pFrom) { // fit in points decreasing order: cosmics upper leg
     pTo++;
     pinc = 1;
+    signELoss = 1; // eloss is corrected
   } else { // fit in points increasing order: collision track or cosmics lower leg
     pTo--;
     pinc = -1;
+    signELoss = -1; // eloss is applied
   }
   //
-  for (int ip = pFrom; ip != pTo; ip += pinc) { // points are ordered against track direction
+  const auto& algConf = AlignConfig::Instance();
+  for (int ip = pFrom; ip != pTo; ip += pinc) { // points are ordered against track direction:
     AlignmentPoint* pnt = getPoint(ip);
-    if (!propagateParamToPoint(probe, pnt)) {
+    if (!propagateParamToPoint(probe, pnt, algConf.maxStep, algConf.maxSnp, MatCorrType::USEMatCorrNONE, signELoss)) {
       return false;
     }
     //
     // store the current track kinematics at the point BEFORE applying eventual material
     // corrections. This kinematics will be later varied around supplied parameters (in the calcResidDeriv)
     pnt->setTrParamWSB(probe.getParams());
-    //
     // account for materials
-    //    if (pnt->ContainsMaterial()) { // apply material corrections
     if (!applyMatCorr(probe, extendedParams, pnt)) {
       return false;
     }
-    //    }
     pnt->setTrParamWSA(probe.getParams());
     //
     if (pnt->containsMeasurement()) { // need to calculate residuals in the frame where errors are orthogonal
@@ -492,7 +500,7 @@ bool AlignmentTrack::calcResiduals(const double* extendedParams, bool invert, in
     if (pnt->containsMaterial()) {
       // material degrees of freedom do not contribute to NDF since they are constrained by 0 expectation
       int nCorrPar = pnt->getNMatPar();
-      float* corCov = pnt->getMatCorrCov();                                 // correction diagonalized covariance
+      const float* corCov = pnt->getMatCorrCov(); // correction diagonalized covariance
       auto offs = pnt->getMaxLocVarID() - nCorrPar;
       for (int i = 0; i < nCorrPar; i++) {
         mChi2 += mLocPar[offs + i] * mLocPar[offs + i] / corCov[i];
@@ -503,18 +511,19 @@ bool AlignmentTrack::calcResiduals(const double* extendedParams, bool invert, in
 }
 
 //______________________________________________________
-bool AlignmentTrack::propagateParamToPoint(trackParam_t* tr, int nTr, const AlignmentPoint* pnt, double maxStep, double maxSnp, MatCorrType mt)
+bool AlignmentTrack::propagateParamToPoint(trackPar_t* tr, int nTr, const AlignmentPoint* pnt, double maxStep, double maxSnp, MatCorrType mt, int signCorr)
 {
   // Propagate set of tracks to the point  (only parameters, no error matrix)
   // VECTORIZE this
   //
+  const auto& algConf = AlignConfig::Instance();
   for (int itr = nTr; itr--;) {
-    if (!propagateParamToPoint(tr[itr], pnt, maxStep)) {
-#if DEBUG > 3
-      LOG(fatal) << "Failed on clone " << itr << " propagation ";
-      tr[itr].print();
-      pnt->print("meas mat");
-#endif
+    if (!propagateParamToPoint(tr[itr], pnt, maxStep, maxSnp, mt, signCorr)) {
+      if (algConf.verbose > 2) {
+        LOG(error) << "Failed on clone " << itr << " propagation ";
+        tr[itr].printParam();
+        pnt->print(AlignmentPoint::kMeasurementBit | AlignmentPoint::kMaterialBit);
+      }
       return false;
     }
   }
@@ -522,43 +531,42 @@ bool AlignmentTrack::propagateParamToPoint(trackParam_t* tr, int nTr, const Alig
 }
 
 //______________________________________________________
-bool AlignmentTrack::propagateParamToPoint(trackParam_t& tr, const AlignmentPoint* pnt, double maxStep, double maxSnp, MatCorrType mt)
+bool AlignmentTrack::propagateParamToPoint(trackPar_t& tr, const AlignmentPoint* pnt, double maxStep, double maxSnp, MatCorrType mt, int signCorr)
 {
   // propagate tracks to the point (only parameters, no error matrix)
-  return propagate(tr, pnt, maxStep, maxSnp, mt, nullptr);
+  return propagate(tr, pnt, maxStep, maxSnp, mt, nullptr, signCorr);
 }
 
 //______________________________________________________
-bool AlignmentTrack::propagateToPoint(trackParam_t& tr, const AlignmentPoint* pnt, double maxStep, double maxSnp, MatCorrType mt, track::TrackLTIntegral* tLT)
+bool AlignmentTrack::propagateToPoint(trackParam_t& tr, trackPar_t* linRef, const AlignmentPoint* pnt, double maxStep, double maxSnp, MatCorrType mt, track::TrackLTIntegral* tLT, int signCorr)
 {
   // propagate tracks to the point. If matCor is true, then material corrections will be applied.
   // if matPar pointer is provided, it will be filled by total x2x0 and signed xrho
-  return propagate(tr, pnt, maxStep, maxSnp, mt, tLT);
+  return propagate(tr, linRef, pnt, maxStep, maxSnp, mt, tLT, signCorr);
 }
 
-bool AlignmentTrack::propagate(trackParam_t& track, const AlignmentPoint* pnt, double maxStep, double maxSnp, MatCorrType mt, track::TrackLTIntegral* tLT)
+bool AlignmentTrack::propagate(trackParam_t& track, trackPar_t* linRef, const AlignmentPoint* pnt, double maxStep, double maxSnp, MatCorrType mt, track::TrackLTIntegral* tLT, int signCorr)
 {
-  if (!track.rotate(pnt->getAlphaSens())) {
-#if DEBUG > 3
-    LOG(error) << "Failed to rotate to alpha=" << pnt->getAlphaSens();
-    tr.print();
-    pnt->Print();
-#endif
-    return false;
+  if (signCorr == 0) { // auto
+    // calculate the sign of the energy loss correction and ensure the upper leg of cosmics is calculated correctly.
+    double dx = pnt->getXTracking() - track.getX();
+    int dir = dx > 0.f ? 1 : -1;
+    signCorr = pnt->isInvDir() ? dir : -dir; // propagation along the track direction should have signCorr=-1
   }
-  // calculate the sign of the energy loss correction and ensure the upper leg of cosmics is calculated correctly.
-  const int signCorr = [this, &pnt, &track, maxStep] {
-    const double dx = maxStep - track.getX();
-    const int dir = dx > 0.f ? 1 : -1;
-    if (pnt->isInvDir()) {
-      // upper leg of a cosmic -> inward facing track
-      return dir;
-    } else {
-      // outward facing track
-      return -dir;
-    }
-  }();
-  return PropagatorD::Instance()->propagateTo(track, pnt->getXPoint(), pnt->getUseBzOnly(), maxSnp, maxStep, mt, tLT, signCorr);
+  // do propagation in at least 2 step to reveal eventual effect of MS on the position
+  return PropagatorD::Instance()->propagateToAlphaX(track, linRef, pnt->getAlphaSens(), pnt->getXTracking(), pnt->getUseBzOnly(), maxSnp, maxStep, 2, mt, tLT, signCorr);
+}
+
+bool AlignmentTrack::propagate(trackPar_t& track, const AlignmentPoint* pnt, double maxStep, double maxSnp, MatCorrType mt, track::TrackLTIntegral* tLT, int signCorr)
+{
+  if (signCorr == 0) { // auto
+    // calculate the sign of the energy loss correction and ensure the upper leg of cosmics is calculated correctly.
+    double dx = pnt->getXTracking() - track.getX();
+    int dir = dx > 0.f ? 1 : -1;
+    signCorr = pnt->isInvDir() ? dir : -dir; // propagation along the track direction should have signCorr=-1
+  }
+  // do propagation in at least 2 step to reveal eventual effect of MS on the position
+  return PropagatorD::Instance()->propagateToAlphaX(track, pnt->getAlphaSens(), pnt->getXTracking(), pnt->getUseBzOnly(), maxSnp, maxStep, 2, mt, tLT, signCorr);
 }
 
 /*
@@ -607,7 +615,7 @@ bool AlignmentTrack::ApplyMS(trackParam_t& trPar, double tms,double pms)
 */
 
 //______________________________________________________
-bool AlignmentTrack::applyMatCorr(trackParam_t& trPar, const double* corrPar, const AlignmentPoint* pnt)
+bool AlignmentTrack::applyMatCorr(trackPar_t& trPar, const double* corrPar, const AlignmentPoint* pnt)
 {
   // Modify track param (e.g. trackParam_t) in the tracking frame
   // by delta accounting for material effects
@@ -615,7 +623,7 @@ bool AlignmentTrack::applyMatCorr(trackParam_t& trPar, const double* corrPar, co
   // DIAGONALIZING ITS  COVARIANCE MATRIX!
   // transform parameters from the frame diagonalizing the errors to track frame
   double corr[kNKinParBON] = {0};
-  if (pnt->containsMaterial()) { // are there free params from meterials?
+  if (pnt->containsMaterial()) { // are there free params from materials?
     int nCorrPar = pnt->getNMatPar();
     const double* corrDiag = &corrPar[pnt->getMaxLocVarID() - nCorrPar]; // material corrections for this point start here
     pnt->unDiagMatCorr(corrDiag, corr);                                  // this is to account for MS and RANDOM Eloss (if varied)
@@ -634,24 +642,23 @@ bool AlignmentTrack::applyMatCorr(trackParam_t& trPar, const double* corrPar, co
 }
 
 //______________________________________________________
-bool AlignmentTrack::applyMatCorr(trackParam_t& trPar, const double* corr)
+bool AlignmentTrack::applyMatCorr(trackPar_t& trPar, const double* corr)
 {
   // Modify track param (e.g. trackParam_t) in the tracking frame
   // by delta accounting for material effects
   // Note: corr contains delta to track frame, NOT in diagonalized one
-  const double kMaxSnp = 0.95;
-
   const double snp = trPar.getSnp() + corr[kParSnp];
-  if (Abs(snp) > kMaxSnp) {
-#if DEBUG > 3
-    LOG(error) << "Snp is too large: " << snp;
-    printf("DeltaPar: ");
-    for (int i = 0; i < kNKinParBON; i++) {
-      printf("%+.3e ", corr[i]);
+  const auto& algConf = AlignConfig::Instance();
+  if (Abs(snp) > algConf.maxSnp) {
+    if (algConf.verbose > 2) {
+      LOG(error) << "Snp is too large: " << snp;
+      printf("DeltaPar: ");
+      for (int i = 0; i < kNKinParBON; i++) {
+        printf("%+.3e ", corr[i]);
+      }
+      printf("\n");
+      trPar.printParam();
     }
-    printf("\n");
-    trPar.print();
-#endif
     return false;
   }
 
@@ -661,15 +668,16 @@ bool AlignmentTrack::applyMatCorr(trackParam_t& trPar, const double* corr)
 }
 
 //______________________________________________________
-bool AlignmentTrack::applyMatCorr(trackParam_t* trSet, int ntr, const double* corrDiag, const AlignmentPoint* pnt)
+bool AlignmentTrack::applyMatCorr(trackPar_t* trSet, int ntr, const double* corrDiag, const AlignmentPoint* pnt)
 {
   // Modify set of track params (e.g. trackParam_t) in the tracking frame
   // by delta accounting for material effects
   // Note: corrDiag contain delta to track parameters rotated by the matrix DIAGONALIZING ITS
   // COVARIANCE MATRIX
   // transform parameters from the frame diagonalizing the errors to track frame
+  const auto& algConf = AlignConfig::Instance();
   double corr[kNKinParBON] = {0};
-  if (pnt->containsMaterial()) { // are there free params from meterials?
+  if (pnt->containsMaterial()) { // are there free params from materials?
     int nCorrPar = pnt->getNMatPar();
     const double* corrDiagP = &corrDiag[pnt->getMaxLocVarID() - nCorrPar]; // material corrections for this point start here
     pnt->unDiagMatCorr(corrDiagP, corr);
@@ -685,10 +693,10 @@ bool AlignmentTrack::applyMatCorr(trackParam_t* trSet, int ntr, const double* co
   //
   for (int itr = ntr; itr--;) {
     if (!applyMatCorr(trSet[itr], corr)) {
-#if DEBUG > 3
-      LOG(error) << "Failed on clone %d materials" << itr;
-      trSet[itr].print();
-#endif
+      if (algConf.verbose > 2) {
+        LOGP(error, "Failed on clone {} materials", itr);
+        trSet[itr].printParam();
+      }
       return false;
     }
   }
@@ -736,7 +744,7 @@ double AlignmentTrack::richardsonExtrap(const double* val, int ord)
 }
 
 //______________________________________________
-void AlignmentTrack::richardsonDeriv(const trackParam_t* trSet, const double* delta, const AlignmentPoint* pnt, double& derY, double& derZ)
+void AlignmentTrack::richardsonDeriv(const trackPar_t* trSet, const double* delta, const AlignmentPoint* pnt, double& derY, double& derZ)
 {
   // Calculate Richardson derivatives for diagonalized Y and Z from a set of kRichardsonN pairs
   // of tracks with same parameter of i-th pair varied by +-delta[i]
@@ -751,6 +759,9 @@ void AlignmentTrack::richardsonDeriv(const trackParam_t* trSet, const double* de
   }
   derY = richardsonExtrap(derRichY, kRichardsonOrd); // dY/dPar
   derZ = richardsonExtrap(derRichZ, kRichardsonOrd); // dZ/dPar
+  if (TMath::IsNaN(derY) || TMath::IsNaN(derZ)) {
+    LOGP(error, "NAN encounterd: DerY {}  : DerZ {}", derY, derZ);
+  }
   //
 }
 
@@ -788,7 +799,7 @@ void AlignmentTrack::Print(Option_t* opt) const
     for (int ip = 0; ip < getNPoints(); ip++) {
       printf("#%3d ", ip);
       auto* pnt = getPoint(ip);
-      pnt->print(0); // RS FIXME OPT
+      pnt->print(AlignmentPoint::kMeasurementBit); // RS FIXME OPT
       //
       if (res && pnt->containsMeasurement()) {
         printf("  Residuals  : %+.3e %+.3e -> Pulls: %+7.2f %+7.2f\n",
@@ -809,8 +820,8 @@ void AlignmentTrack::Print(Option_t* opt) const
         }
         printf("\n");
         printf("  Corr.Pull:  ");
-        float* corCov = pnt->getMatCorrCov(); // correction covariance
-        //float *corExp = pnt->getMatCorrExp(); // correction expectation
+        const float* corCov = pnt->getMatCorrCov(); // correction covariance
+        // const float *corExp = pnt->getMatCorrExp(); // correction expectation
         for (int i = 0; i < nCorrPar; i++) {
           printf("%+.3e ", (mLocPar[i + pnt->getMaxLocVarID() - nCorrPar] /* - corExp[i]*/) / Sqrt(corCov[i]));
         }
@@ -849,10 +860,11 @@ bool AlignmentTrack::iniFit()
 {
   // perform initial fit of the track
   //
-  trackParam_t trc = *this;
-  //
+  const auto& algConf = AlignConfig::Instance();
   if (!getFieldON()) { // for field-off data impose nominal momentum
+    setQ2Pt(isCosmic() ? 1. / algConf.defPTB0Cosm : 1. / algConf.defPTB0Coll);
   }
+  trackParam_t trc(*(trackParam_t*)this);
   mChi2 = mChi2CosmUp = mChi2CosmDn = 0;
   //
   // the points are ranged from outer to inner for collision tracks,
@@ -860,31 +872,29 @@ bool AlignmentTrack::iniFit()
   //
   // the fit will always start from the outgoing track in inward direction
   if (!fitLeg(trc, 0, getInnerPointID(), mNeedInv[0])) {
-#if DEBUG > 3
-    LOG(warn) << "Failed fitLeg 0";
-    trc.print();
-#endif
+    if (algConf.verbose > 2) {
+      LOG(warn) << "Failed fitLeg 0";
+      trc.print();
+    }
     return false; // collision track or cosmic lower leg
   }
   //
   //  printf("Lower leg: %d %d\n",0,getInnerPointID()); trc.print();
   //
-  const auto& cnf = AlignConfig::Instance();
-  ;
   if (isCosmic()) {
     mChi2CosmDn = mChi2;
     trackParam_t trcU = trc;
     if (!fitLeg(trcU, getNPoints() - 1, getInnerPointID() + 1, mNeedInv[1])) { //fit upper leg of cosmic track
-#if DEBUG > 3
-      LOG(warn) << "Failed fitLeg 0";
-      trc.print();
-#endif
+      if (algConf.verbose > 2) {
+        LOG(warn) << "Failed fitLeg 1";
+        trc.print();
+      }
       return false; // collision track or cosmic lower leg
     }
     //
     // propagate to reference point, which is the inner point of lower leg
     const AlignmentPoint* refP = getPoint(getInnerPointID());
-    if (!propagateToPoint(trcU, refP, cnf.maxStep, cnf.maxSnp, cnf.matCorType)) {
+    if (!propagateToPoint(trcU, nullptr, refP, algConf.maxStep, algConf.maxSnp, MatCorrType(algConf.matCorType), nullptr, -1)) { // moving along the track: energy is lost
       return false;
     }
     //
@@ -899,7 +909,6 @@ bool AlignmentTrack::iniFit()
   copyFrom(&trc);
   //
   mChi2Ini = mChi2;
-
   return true;
 }
 
@@ -915,13 +924,27 @@ bool AlignmentTrack::combineTracks(trackParam_t& trcL, const trackParam_t& trcU)
   // CL' = CL - K*CL
   // vL' = vL + K(vU-vL)
   //
+  const auto& algConf = AlignConfig::Instance();
   if (Abs(trcL.getX() - trcU.getX()) > TinyDist || Abs(trcL.getAlpha() - trcU.getAlpha()) > TinyDist) {
-    LOG(error) << "Tracks must be defined at same reference X and Alpha";
-    trcL.print();
-    trcU.print();
+    if (algConf.verbose > 2) {
+      LOG(error) << "Tracks must be defined at same reference X and Alpha";
+      trcL.print();
+      trcU.print();
+    }
     return false;
   }
   //
+  LOGP(debug, "CosmDn: {}", trcL.asString());
+  LOGP(debug, "CosmUp: {}", trcU.asString());
+  float dsnp = trcL.getSnp() - trcU.getSnp(), dTgl = trcL.getTgl() - trcU.getTgl();
+  if (std::abs(dsnp) > algConf.cosmMaxDSnp || std::abs(dTgl) > algConf.cosmMaxDTgl) {
+    if (algConf.verbose > 2) {
+      LOGP(error, "Rejecting track with dSnp={} dTgl={}", dsnp, dTgl);
+      LOGP(error, "CosmDn: {}", trcL.asString());
+      LOGP(error, "CosmUp: {}", trcU.asString());
+    }
+    return false;
+  }
   //  const covMat_t& covU = trcU.getCov();
   //  const covMat_t& covL = trcL.getCov();
   //
@@ -943,10 +966,10 @@ bool AlignmentTrack::combineTracks(trackParam_t& trcL, const trackParam_t& trcU)
   }
   matCLplCU.Invert(); // S^-1 = (Cl + Cu)^-1
   if (!matCLplCU.IsValid()) {
-#if DEBUG > 3
-    LOG(error) << "Failed to invert summed cov.matrix of cosmic track";
-    matCLplCU.print();
-#endif
+    if (algConf.verbose > 2) {
+      LOG(error) << "Failed to invert summed cov.matrix of cosmic track";
+      matCLplCU.Print();
+    }
     return false; // inversion failed
   }
   TMatrixD matK(matCL, TMatrixD::kMult, matCLplCU); // gain K = Cl*(Cl+Cu)^-1
@@ -968,7 +991,8 @@ bool AlignmentTrack::combineTracks(trackParam_t& trcL, const trackParam_t& trcU)
   }
   mChi2 += chi2;
   //
-  //  printf("Combined: Chi2Tot:%.2f ChiUp:%.2f ChiDn:%.2f ChiCmb:%.2f\n",mChi2,mChi2CosmUp,mChi2CosmDn, chi2);
+  LOGP(debug, "CosmCB: {}", trcL.asString());
+  LOGP(debug, "Combined: Chi2Tot:{} ChiUp:{} ChiDn:{} ChiCmb:{} DSnp:{} DTgl:{} Alp:{}", mChi2, mChi2CosmUp, mChi2CosmDn, chi2, dsnp, dTgl, trcL.getAlpha());
 
   return true;
 }
@@ -980,7 +1004,7 @@ bool AlignmentTrack::fitLeg(trackParam_t& trc, int pFrom, int pTo, bool& inv)
   // the fit will always start from the outgoing track in inward direction (i.e. if cosmics - bottom leg)
   const int kMinNStep = 3;
   const double kErrSpace = 50.;
-  const double kErrAng = 0.7;
+  const double kErrAng = 0.8;
   const double kErrRelPtI = 1.;
   const covMat_t kIniErr{// initial error
                          kErrSpace * kErrSpace,
@@ -989,56 +1013,53 @@ bool AlignmentTrack::fitLeg(trackParam_t& trc, int pFrom, int pTo, bool& inv)
                          0, 0, 0, kErrAng * kErrAng,
                          0, 0, 0, 0, kErrRelPtI * kErrRelPtI};
   //
+  static int count = 0;
+  const auto& algConf = AlignConfig::Instance();
+  if (algConf.verbose > 2) {
+    LOGP(info, "FIT COUNT {}", count++);
+  }
   // prepare seed at outer point
-  const auto& cnf = AlignConfig::Instance();
-  ;
   AlignmentPoint* p0 = getPoint(pFrom);
   double phi = trc.getPhi(), alp = p0->getAlphaSens();
-  bringTo02Pi(phi);
-  bringTo02Pi(alp);
-  double dphi = deltaPhiSmall(phi, alp); // abs delta angle
+  math_utils::detail::bringTo02Pi(phi);
+  math_utils::detail::bringTo02Pi(alp);
+  double dphi = math_utils::detail::deltaPhiSmall(phi, alp); // abs delta angle
   if (dphi > Pi() / 2.) {                // need to invert the track to new frame
     inv = true;
-    //    printf("Fit in %d %d Delta: %.3f -> Inverting for\n",pFrom,pTo,dphi);
-    //    p0->print("meas");
-    //    printf("BeforeInv "); trc.print();
     trc.invert();
-    //    printf("After Inv "); trc.print();
-  }
-  if (!trc.rotateParam(p0->getAlphaSens())) {
-#if DEBUG > 3
-    AliWarningF("Failed on rotateParam to %f", p0->getAlphaSens());
-    trc.print();
-#endif
-    return false;
   }
-  if (!propagateParamToPoint(trc, p0, cnf.maxStep)) {
-    //  if (!propagateToPoint(trc,p0,5,30,true)) {
-    //trc.PropagateParamOnlyTo(p0->getXPoint()+kOverShootX,AliTrackerBase::GetBz())) {
-#if DEBUG > 3
-    AliWarningF("Failed on PropagateParamOnlyTo to %f", p0->getXPoint() + kOverShootX);
-    trc.print();
-#endif
+  // Fit is done from outward to inward: normally against the track direction (hence e.loss is compensated) unless inversion is requested (cosmic upper leg)
+  if (!propagateParamToPoint(trc, p0, algConf.maxStep, algConf.maxSnp, MatCorrType(algConf.matCorType), -1)) {
+    if (algConf.verbose > 2) {
+      LOGF(warn, "Failed on PropagateParamOnlyTo to %f", p0->getXTracking());
+      trc.print();
+    }
     return false;
   }
+  trackPar_t linRef(trc), *linRefP = algConf.useLinRef ? &linRef : nullptr;
   trc.setCov(kIniErr);
-  trc.setCov(trc.getQ2Pt() * trc.getQ2Pt(), 4, 4); // lowest diagonal element (Q2Pt2)
+  trc.setCov(16 * trc.getQ2Pt() * trc.getQ2Pt(), 4, 4); // lowest diagonal element (Q2Pt2)
   //
-  int pinc;
+  int pinc, signELoss = 0; // RS Validate for cosmic. Propagation is done from inner point to outer one:
+  // energy loss is applied for collision tracks and cosmic lower leg, compensated for cosmic upper leg
   if (pTo > pFrom) { // fit in points increasing order: collision track or cosmics lower leg
     pTo++;
     pinc = 1;
+    signELoss = 1; // e.loss is compensated
   } else { // fit in points decreasing order: cosmics upper leg
     pTo--;
     pinc = -1;
+    signELoss = -1; // e.loass is applied
   }
   //
+  int pntCnt = 0;
   for (int ip = pFrom; ip != pTo; ip += pinc) { // inward fit from outer point
     AlignmentPoint* pnt = getPoint(ip);
-    //
-    //    printf("*** fitLeg %d (%d %d)\n",ip,pFrom,pTo);
-    //    printf("Before propagate: "); trc.print();
-    if (!propagateToPoint(trc, pnt, cnf.maxStep, cnf.maxSnp, cnf.matCorType)) {
+    if (!propagateToPoint(trc, linRefP, pnt, algConf.maxStep, algConf.maxSnp, MatCorrType(algConf.matCorType), nullptr, signELoss)) { // against track direction : e.loss is compensated
+      if (algConf.verbose > 2) {
+        LOGF(warn, "Failed on propagateToPoint %d (%d : %d) %f", ip, pFrom, pTo, pnt->getXTracking());
+        trc.print();
+      }
       return false;
     }
     if (pnt->containsMeasurement()) {
@@ -1048,25 +1069,24 @@ bool AlignmentTrack::fitLeg(trackParam_t& trc, int pFrom, int pTo, bool& inv)
       const double* yz = pnt->getYZTracking();
       const double* errYZ = pnt->getYZErrTracking();
       double chi = trc.getPredictedChi2(yz, errYZ);
-      //printf("***>> fitleg-> Y: %+e %+e / Z: %+e %+e -> Chi2: %e | %+e %+e\n",yz[0],trc.GetY(),yz[1],trc.GetZ(),chi,
-      //  trc.Phi(),trc.GetAlpha());
-      //      printf("Before update at %e %e\n",yz[0],yz[1]); trc.print();
       if (!trc.update(yz, errYZ)) {
-#if DEBUG > 3
-        AliWarningF("Failed on Update %f,%f {%f,%f,%f}", yz[0], yz[1], errYZ[0], errYZ[1], errYZ[2]);
-        trc.print();
-#endif
+        if (algConf.verbose > 2) {
+          LOGF(warn, "Failed on Update %f,%f {%f,%f,%f}", yz[0], yz[1], errYZ[0], errYZ[1], errYZ[2]);
+          trc.print();
+        }
         return false;
       }
       mChi2 += chi;
-      //      printf("After update: (%f) -> %f\n",chi,mChi2); trc.print();
+      pntCnt++;
     }
   }
   //
   if (inv) {
-    //    printf("Before inverting back "); trc.print();
     trc.invert();
   }
+  if (algConf.verbose > 2) {
+    LOGP(info, "Fitted leg with {} points, chi2: {}", pntCnt, mChi2);
+  }
   //
   return true;
 }
@@ -1079,7 +1099,7 @@ bool AlignmentTrack::residKalman()
   //
   bool inv = false;
   const int kMinNStep = 3;
-  const double kErrSpace = 50.;
+  const double kErrSpace = 100.;
   const double kErrAng = 0.7;
   const double kErrRelPtI = 1.;
   const covMat_t kIniErr = {// initial error
@@ -1091,58 +1111,48 @@ bool AlignmentTrack::residKalman()
   //  const double kOverShootX = 5;
   //
   trackParam_t trc = *this;
-  const auto& cnf = AlignConfig::Instance();
-  ;
-  //
+  const auto& algConf = AlignConfig::Instance();
   int pID = 0, nPnt = getNPoints();
   AlignmentPoint* pnt = nullptr;
   // get 1st measured point
-  while (pID < nPnt && !(pnt = getPoint(pID))->containsMeasurement()) {
+  while (pID < nPnt && !(pnt = getPoint(pID))->containsMeasurement()) { // points are ordered from outward to inward
     pID++;
   }
   if (!pnt) {
     return false;
   }
   double phi = trc.getPhi(), alp = pnt->getAlphaSens();
-  bringTo02Pi(phi);
-  bringTo02Pi(alp);
-  double dphi = deltaPhiSmall(phi, alp);
+  math_utils::detail::bringTo02Pi(phi);
+  math_utils::detail::bringTo02Pi(alp);
+  double dphi = math_utils::detail::deltaPhiSmall(phi, alp);
   if (dphi > Pi() / 2.) { // need to invert the track to new frame
     inv = true;
     trc.invert();
   }
-  // prepare track seed at 1st valid point
-  if (!trc.rotateParam(pnt->getAlphaSens())) {
-#if DEBUG > 3
-    AliWarningF("Failed on rotateParam to %f", pnt->getAlphaSens());
-    trc.print();
-#endif
-    return false;
-  }
-  if (!propagateParamToPoint(trc, pnt, cnf.maxStep)) {
-    //if (!trc.PropagateParamOnlyTo(pnt->getXPoint()+kOverShootX,AliTrackerBase::GetBz())) {
-#if DEBUG > 3
-    AliWarningF("Failed on PropagateParamOnlyTo to %f", pnt->getXPoint() + kOverShootX);
-    trc.print();
-#endif
+  // prepare track seed at 1st outermost valid point
+  if (!propagateParamToPoint(trc, pnt, algConf.maxStep, algConf.maxSnp, MatCorrType::USEMatCorrNONE, 1)) {
+    if (algConf.verbose > 2) {
+      LOG(warn) << "Failed on propagateParamToPoint";
+      trc.print();
+      pnt->print(AlignmentPoint::kMeasurementBit);
+    }
     return false;
   }
   //
   trc.setCov(kIniErr);
-  const double inwardQ2Pt2 = trc.getCovarElem(4, 4) * trc.getQ2Pt() * trc.getQ2Pt();
+  const double inwardQ2Pt2 = 4 * trc.getCovarElem(4, 4) * trc.getQ2Pt() * trc.getQ2Pt();
   trc.setCov(inwardQ2Pt2, 4, 4); // lowest diagonal element (Q2Pt2)
   //
   double chifwd = 0, chibwd = 0;
   // inward fit
-  for (int ip = 0; ip < nPnt; ip++) {
+  int signELoss = 1; // normally inward fit goes against track direction: eloss is compensated. // RS Check for cosmic
+  for (int ip = pID; ip < nPnt; ip++) {
     pnt = getPoint(ip);
     if (pnt->isInvDir() != inv) { // crossing point where the track should be inverted?
       trc.invert();
       inv = !inv;
     }
-    //    printf("*** ResidKalm %d (%d %d)\n",ip,0,nPnt);
-    //    printf("Before propagate: "); trc.print();
-    if (!propagateToPoint(trc, pnt, cnf.maxStep, cnf.maxSnp, cnf.matCorType)) {
+    if (!propagateToPoint(trc, nullptr, pnt, algConf.maxStep, algConf.maxSnp, MatCorrType(algConf.matCorType), nullptr, signELoss)) {
       return false;
     }
     if (!pnt->containsMeasurement()) {
@@ -1158,33 +1168,30 @@ bool AlignmentTrack::residKalman()
     ws[3] = trc.getSigmaZY();
     ws[4] = trc.getSigmaZ2();
     double chi = trc.getPredictedChi2(yz, errYZ);
-    //    printf(">> INV%d (%9d): %+.2e %+.2e | %+.2e %+.2e %+.2e %+.2e %+.2e | %.2e %d \n",ip,pnt->getSensor()->getInternalID(),yz[0],yz[1], ws[0],ws[1],ws[2],ws[3],ws[4],chi,inv);
-    //    printf(">>Bef ");trc.print();
-    // printf("KLM Before update at %e %e\n",yz[0],yz[1]); trc.print();
     if (!trc.update(yz, errYZ)) {
-#if DEBUG > 3
-      AliWarningF("Failed on Inward Update %f,%f {%f,%f,%f}", yz[0], yz[1], errYZ[0], errYZ[1], errYZ[2]);
-      trc.print();
-#endif
+      if (algConf.verbose > 2) {
+        LOGF(warn, "Failed on Inward Update %f,%f {%f,%f,%f}", yz[0], yz[1], errYZ[0], errYZ[1], errYZ[2]);
+        trc.print();
+      }
       return false;
     }
     //    printf(">>Aft ");trc.print();
     chifwd += chi;
-    //printf("KLM After update: (%f) -> %f\n",chi,chifwd);   trc.print();
+    //    printf("KLM After update: (%f) -> %f\n",chi,chifwd);   trc.print();
   }
   //
   // outward fit
   trc.setCov(kIniErr);
-  const double outwardQ2Pt2 = trc.getCovarElem(4, 4) * trc.getQ2Pt() * trc.getQ2Pt();
+  const double outwardQ2Pt2 = 4 * trc.getCovarElem(4, 4) * trc.getQ2Pt() * trc.getQ2Pt();
   trc.setCov(outwardQ2Pt2, 4, 4); // lowest diagonal element (Q2Pt2)
-
+  signELoss = -1;                 // normally outward fit goes along track direction: eloss is applied. // RS Check for cosmic
   for (int ip = nPnt; ip--;) {
     pnt = getPoint(ip);
     if (pnt->isInvDir() != inv) { // crossing point where the track should be inverted?
       trc.invert();
       inv = !inv;
     }
-    if (!propagateToPoint(trc, pnt, cnf.maxStep, cnf.maxSnp, cnf.matCorType)) {
+    if (!propagateToPoint(trc, nullptr, pnt, algConf.maxStep, algConf.maxSnp, MatCorrType(algConf.matCorType), nullptr, signELoss)) { // we are going along track direction, e.loss is applied
       return false;
     }
     if (!pnt->containsMeasurement()) {
@@ -1203,10 +1210,10 @@ bool AlignmentTrack::residKalman()
     //    printf("<< OUT%d (%9d): %+.2e %+.2e | %+.2e %+.2e %+.2e %+.2e %+.2e | %.2e %d \n",ip,pnt->getSensor()->getInternalID(),yz[0],yz[1], ws[0],ws[1],ws[2],ws[3],ws[4],chi,inv);
     //    printf("< 3
-      AliWarningF("Failed on Outward Update %f,%f {%f,%f,%f}", yz[0], yz[1], errYZ[0], errYZ[1], errYZ[2]);
-      trc.print();
-#endif
+      if (algConf.verbose > 2) {
+        LOGF(warn, "Failed on Outward Update %f,%f {%f,%f,%f}", yz[0], yz[1], errYZ[0], errYZ[1], errYZ[2]);
+        trc.print();
+      }
       return false;
     }
     chibwd += chi;
@@ -1257,16 +1264,15 @@ bool AlignmentTrack::processMaterials()
 {
   // attach material effect info to alignment points
   trackParam_t trc = *this;
-
-  // collision track or cosmic lower leg: move along track direction from last (middle for cosmic lower leg)
-  // point (inner) to 1st one (outer)
+  const auto& algConf = AlignConfig::Instance();
+  // collision track or cosmic lower leg: move along track direction from last (middle for cosmic lower leg) point (inner) to 1st one (outer)
   if (mNeedInv[0]) {
     trc.invert();
   } // track needs to be inverted ? (should be for upper leg)
   if (!processMaterials(trc, getInnerPointID(), 0)) {
-#if DEBUG > 3
-    LOG(error) << "Failed to process materials for leg along the track";
-#endif
+    if (algConf.verbose > 2) {
+      LOG(error) << "Failed to process materials for leg along the track";
+    }
     return false;
   }
   if (isCosmic()) {
@@ -1318,16 +1324,21 @@ bool AlignmentTrack::processMaterials(trackParam_t& trc, int pFrom, int pTo)
   double dpar[5] = {0};
   covMat_t dcov{0};
   matTL.setTimeNotNeeded();
-  const auto& cnf = AlignConfig::Instance();
-  ;
+  const auto& algConf = AlignConfig::Instance();
+  // calculate min X/X0 to treat the point as having materials
+  double minX2X0 = algConf.minScatteringAngleToAccount / 0.014 * trc.getPt();
+  minX2X0 *= minX2X0;
+
   //
-  int pinc;
-  if (pTo > pFrom) { // fit in points decreasing order: cosmics upper leg
+  int pinc, signELoss = 0;
+  if (pTo > pFrom) { // fit inward: cosmics upper leg
     pTo++;
     pinc = 1;
-  } else { // fit in points increasing order: collision track or cosmics lower leg
+    signELoss = 1; // against the track direction, eloss is applied (RS Check for cosmic)
+  } else {         // fit outward collision track or cosmics lower leg: along the track direction:
     pTo--;
     pinc = -1;
+    signELoss = -1; // eloss is applied
   }
   //
   for (int ip = pFrom; ip != pTo; ip += pinc) { // points are ordered against track direction
@@ -1335,34 +1346,31 @@ bool AlignmentTrack::processMaterials(trackParam_t& trc, int pFrom, int pTo)
     trc.setCov(kErrTiny); // assign tiny errors to both tracks
     tr0 = trc;
     //
+    matTL.clearFast();
     //    printf("-> ProcMat %d (%d->%d)\n",ip,pFrom,pTo);
-    if (!propagateToPoint(trc, pnt, cnf.maxStep, cnf.maxSnp, cnf.matCorType, &matTL)) { // with material corrections
-#if DEBUG > 3
-      LOG(error) << "Failed to take track to point" << ip << " (dir: " << pFrom << "->" pTo << ") with mat.corr.";
-      trc.print();
-      pnt->print("meas");
-#endif
+    if (!propagateToPoint(trc, nullptr, pnt, algConf.maxStep, algConf.maxSnp, MatCorrType(algConf.matCorType), &matTL, signELoss)) { // with material corrections
+      if (algConf.verbose > 2) {
+        LOG(error) << "Failed to take track to point" << ip << " (dir: " << pFrom << "->" << pTo << ") with mat.corr.";
+        trc.print();
+        pnt->print(AlignmentPoint::kMeasurementBit);
+      }
       return false;
     }
     //
     // is there enough material to consider the point as a scatterer?
-    pnt->setContainsMaterial(matTL.getX2X0() * Abs(trc.getQ2Pt()) > cnf.minX2X0Pt2Account);
-    //
-    //    printf("-> ProcMat000 %d (%d->%d)\n",ip,pFrom,pTo);
-    if (!propagateToPoint(tr0, pnt, cnf.maxStep, cnf.maxSnp, MatCorrType::USEMatCorrNONE)) { // no material corrections
-#if DEBUG > 3
-      LOG(error) << "Failed to take track to point" << ip << " (dir: " << pFrom << "->" pTo << ") with mat.corr.";
-      tr0.print();
-      pnt->print("meas");
-#endif
+    bool hasMaterial = matTL.getX2X0() > minX2X0;
+    if (!propagateToPoint(tr0, nullptr, pnt, algConf.maxStep, algConf.maxSnp, MatCorrType::USEMatCorrNONE, nullptr, signELoss)) { // no material corrections
+      if (algConf.verbose > 2) {
+        LOG(error) << "Failed to take track to point" << ip << " (dir: " << pFrom << "->" << pTo << ") with mat.corr.";
+        tr0.print();
+        pnt->print(AlignmentPoint::kMeasurementBit);
+      }
       return false;
     }
     // the difference between the params, covariance of tracks with and  w/o material accounting gives
     // parameters and covariance of material correction. For params ONLY ELoss effect is relevant
-    const covMat_t& cov0 = tr0.getCov();
-    double* par0 = (double*)tr0.getParams();
-    const covMat_t& cov1 = trc.getCov();
-    double* par1 = (double*)trc.getParams();
+    double *par0 = (double*)tr0.getParams(), *par1 = (double*)trc.getParams();
+    const covMat_t &cov0 = tr0.getCov(), &cov1 = trc.getCov();
     for (int l = 15; l--;) {
       dcov[l] = cov1[l] - cov0[l];
     }
@@ -1372,45 +1380,63 @@ bool AlignmentTrack::processMaterials(trackParam_t& trc, int pFrom, int pTo)
     pnt->setMatCorrExp(dpar);
     //dpar[kParQ2Pt] = par1[kParQ2Pt] - par0[kParQ2Pt]; // only e-loss expectation is non-0
     //
-    if (pnt->containsMaterial()) {
-      //
+    if (hasMaterial) {
       // MP2 handles only scalar residuals hence correlated matrix of material effect need to be diagonalized
       bool eLossFree = pnt->getELossVaried();
       int nParFree = eLossFree ? kNKinParBON : kNKinParBOFF;
       TMatrixDSym matCov(nParFree);
       for (int i = nParFree; i--;) {
         for (int j = i + 1; j--;) {
-          matCov(i, j) = matCov(j, i) = dcov[j + ((i * (i + 1)) >> 1)];
+          auto err2 = dcov[j + ((i * (i + 1)) >> 1)];
+          if (i == j && err2 < 1e-20) { // meaninglessly small diagonal error
+            LOGP(warn, "Material correction {}-th error too small: {}, declare no material despite x/X0={}", i, err2, matTL.getX2X0());
+            hasMaterial = false;
+            break;
+          }
+          matCov(i, j) = matCov(j, i) = err2;
         }
       }
-      //
-      TMatrixDSymEigen matDiag(matCov); // find eigenvectors
-      const TMatrixD& matEVec = matDiag.GetEigenVectors();
-      if (!matEVec.IsValid()) {
-#if DEBUG > 3
-        LOG(error) << "Failed to diagonalize covariance of material correction";
-        matCov.print();
-        return false;
-#endif
-      }
-      pnt->setMatCovDiagonalizationMatrix(matEVec); // store diagonalization matrix
-      pnt->setMatCovDiag(matDiag.GetEigenValues()); // store E.Values: diagonalized cov.matrix
-      if (!eLossFree) {
-        pnt->setMatCovDiagElem(kParQ2Pt, dcov[14]);
+      if (hasMaterial) {
+        TMatrixDSymEigen matDiag(matCov); // find eigenvectors
+        const TMatrixD& matEVec = matDiag.GetEigenVectors();
+        if (!matEVec.IsValid()) {
+          if (algConf.verbose > 2) {
+            LOG(error) << "Failed to diagonalize covariance of material correction";
+            matCov.Print();
+            return false;
+          }
+        }
+        for (int ix = 0; ix < matDiag.GetEigenValues().GetNrows(); ix++) {
+          if (matDiag.GetEigenValues()[ix] < 1e-18) {
+            LOG(warn) << "Too small " << ix << "-th eignevalue " << matDiag.GetEigenValues()[ix] << " for cov.matrix !!!";
+            LOG(warn) << "Track with mat.corr: " << trc.asString();
+            LOG(warn) << "Track w/o  mat.corr: " << tr0.asString();
+            matCov.Print();
+            matDiag.GetEigenValues().Print();
+            hasMaterial = false;
+            break;
+          }
+        }
+        if (hasMaterial) {
+          pnt->setMatCovDiagonalizationMatrix(matEVec); // store diagonalization matrix
+          pnt->setMatCovDiag(matDiag.GetEigenValues()); // store E.Values: diagonalized cov.matrix
+          if (!eLossFree) {
+            pnt->setMatCovDiagElem(kParQ2Pt, dcov[14]);
+          }
+          pnt->setX2X0(matTL.getX2X0());
+          pnt->setXTimesRho(matTL.getXRho());
+        }
       }
-      //
-      pnt->setX2X0(matTL.getX2X0());
-      pnt->setXTimesRho(matTL.getXRho());
-      //
+      pnt->setContainsMaterial(hasMaterial);
     }
     if (pnt->containsMeasurement()) { // update track to have best possible kinematics
       const double* yz = pnt->getYZTracking();
       const double* errYZ = pnt->getYZErrTracking();
       if (!trc.update(yz, errYZ)) {
-#if DEBUG > 3
-        AliWarningF("Failed on Update %f,%f {%f,%f,%f}", yz[0], yz[1], errYZ[0], errYZ[1], errYZ[2]);
-        trc.print();
-#endif
+        if (algConf.verbose > 2) {
+          LOGF(warn, "Failed on Update %f,%f {%f,%f,%f}", yz[0], yz[1], errYZ[0], errYZ[1], errYZ[2]);
+          trc.print();
+        }
         return false;
       }
       //
@@ -1427,7 +1453,7 @@ void AlignmentTrack::sortPoints()
   // sort points in order against track direction: innermost point is last
   // for collision tracks.
   // For 2-leg cosmic tracks: 1st points of outgoing (lower) leg are added from large to
-  // small radii, then the points of incomint (upper) leg are added in increasing R direction
+  // small radii, then the points of incoming (upper) leg are added in increasing R direction
   //
   // The mInnerPointID will mark the id of the innermost point, i.e. the last one for collision-like
   // tracks and in case of cosmics - the point of lower leg with smallest R
@@ -1438,7 +1464,7 @@ void AlignmentTrack::sortPoints()
   }
 
   auto isAfter = [](const AlignmentPoint* a, const AlignmentPoint* b) {
-    auto xa = a->getXPoint(), xb = b->getXPoint();
+    auto xa = a->getXTracking(), xb = b->getXTracking();
     if (!a->isInvDir()) {   // lower leg: track propagates from low to large X via this point
       if (!b->isInvDir()) { // this one also
         return xa > xb;
@@ -1476,6 +1502,15 @@ void AlignmentTrack::setLocPars(const double* pars)
   memcpy(mLocPar.data(), pars, mNLocPar * sizeof(double));
 }
 
+//______________________________________________
+void AlignmentTrack::addLocPars(const double* pars)
+{
+  // store loc par corrections
+  for (int i = 0; i < mNLocPar; i++) {
+    mLocPar[i] += pars[i];
+  }
+}
+
 //______________________________________________
 void AlignmentTrack::checkExpandDerGloBuffer(unsigned int minSize)
 {
@@ -1487,5 +1522,66 @@ void AlignmentTrack::checkExpandDerGloBuffer(unsigned int minSize)
   }
 }
 
+//____________________________________________
+bool AlignmentTrack::testLocalSolution()
+{
+  // test track local solution
+  int npnt = getNPoints();
+  double mat[mNLocPar][mNLocPar], rhs[mNLocPar];
+  std::memset(mat, 0, sizeof(double) * mNLocPar * mNLocPar);
+  std::memset(rhs, 0, sizeof(double) * mNLocPar);
+  for (int ip = npnt; ip--;) {
+    if (mPoints[ip]->containsMeasurement()) {
+      for (int idim = 2; idim--;) {                       // each point has 2 position residuals
+        auto sg2inv = 1. / mPoints[ip]->getErrDiag(idim); // inv. error
+        auto deriv = getDResDLoc(idim, ip);               // array of Dresidual/Dparams
+        for (int parI = 0; parI < mNLocPar; parI++) {
+          rhs[parI] -= deriv[parI] * mResid[idim][ip] * sg2inv;
+          for (int parJ = parI; parJ < mNLocPar; parJ++) {
+            mat[parI][parJ] += deriv[parI] * deriv[parJ] * sg2inv;
+          }
+        }
+      } // loop over 2 orthogonal measurements at the point
+    }   // derivarives at measured points
+    // if the point contains material, consider its expected kinks, eloss as measurements
+    if (mPoints[ip]->containsMaterial()) { // at least 4 parameters: 2 spatial + 2 angular kinks with 0 expectaction
+      int npm = mPoints[ip]->getNMatPar();
+      // const float* expMatCorr = mPoints[ip]->getMatCorrExp(); // expected correction (diagonalized) // RS??
+      const auto expMatCov = mPoints[ip]->getMatCorrCov(); // its error
+      int offs = mPoints[ip]->getMaxLocVarID() - npm;
+      for (int ipar = 0; ipar < npm; ipar++) {
+        int parI = offs + ipar;
+        // expected
+        // rhs[parI] -= expMatCorr[ipar]/expMatCov[ipar]; // consider expectation as measurement // RS??
+        mat[parI][parI] += 1. / expMatCov[ipar]; // this measurement is orthogonal to all others
+      }
+    } // material effect descripotion params
+    //
+  }
+  o2::math_utils::SymMatrixSolver solver(mNLocPar, 1);
+  for (int i = 0; i < mNLocPar; i++) {
+    for (int j = i; j < mNLocPar; j++) {
+      solver.A(i, j) = mat[i][j];
+    }
+    solver.B(i, 0) = rhs[i];
+  }
+  solver.solve();
+  // increment current params by new solution
+  for (int i = 0; i < mNLocPar; i++) {
+    mLocPar[i] += solver.B(i, 0);
+  }
+  return calcResiduals();
+}
+
+//______________________________________________
+void AlignmentTrack::suppressLastPoints(int n)
+{
+  // remove last n points, must be called before sortPoints
+  while (n > 0) {
+    mDetPoints.pop_back();
+    n--;
+  }
+}
+
 } // namespace align
 } // namespace o2
diff --git a/Detectors/Align/src/Controller.cxx b/Detectors/Align/src/Controller.cxx
index 5f4d8af9f68d7..5cfbbf9f3a4ae 100644
--- a/Detectors/Align/src/Controller.cxx
+++ b/Detectors/Align/src/Controller.cxx
@@ -22,26 +22,17 @@
 #include "Align/AlignableDetector.h"
 #include "Align/AlignableVolume.h"
 #include "Align/AlignableDetectorITS.h"
-//#include "Align/AlignableDetectorTPC.h"
-//#include "Align/AlignableDetectorTRD.h"
-//#include "Align/AlignableDetectorTOF.h"
+#include "Align/AlignableDetectorTRD.h"
+#include "Align/AlignableDetectorTOF.h"
+#include "Align/AlignableDetectorTPC.h"
 #include "Align/EventVertex.h"
 #include "Align/ResidualsControllerFast.h"
 #include "Align/GeometricalConstraint.h"
-#include "Align/DOFStatistics.h"
 #include "ReconstructionDataFormats/VtxTrackIndex.h"
 #include "ReconstructionDataFormats/PrimaryVertex.h"
 #include "DataFormatsGlobalTracking/RecoContainer.h"
 #include "ReconstructionDataFormats/VtxTrackRef.h"
-//#include "AliTrackerBase.h"
-//#include "AliESDCosmicTrack.h"
-//#include "AliESDtrack.h"
-//#include "AliESDEvent.h"
-//#include "AliESDVertex.h"
-//#include "AliRecoParam.h"
-//#include "AliCDBRunRange.h"
-//#include "AliCDBManager.h"
-//#include "AliCDBEntry.h"
+#include "TRDBase/TrackletTransformer.h"
 #include "MathUtils/Utils.h"
 
 #include 
@@ -53,31 +44,42 @@
 #include 
 #include 
 #include 
+#include "GPUO2ExternalUser.h"
+#include "DataFormatsTPC/WorkflowHelper.h"
 #include 
 #include "CommonUtils/NameConf.h"
+#include "MathUtils/SymMatrixSolver.h"
 #include "DataFormatsParameters/GRPObject.h"
+#include "GPUParam.h"
+
+#include "SimulationDataFormat/MCUtils.h"
+#include "Steer/MCKinematicsReader.h"
+#include "CommonUtils/TreeStreamRedirector.h"
+#include 
 
 using namespace TMath;
 using namespace o2::align::utils;
 using namespace o2::dataformats;
 using namespace o2::globaltracking;
 
-using GIndex = o2::dataformats::VtxTrackIndex;
-
 namespace o2
 {
 namespace align
 {
+using GIndex = o2::dataformats::VtxTrackIndex;
+using PropagatorD = o2::base::PropagatorD;
+using MatCorrType = PropagatorD::MatCorrType;
 
 void Controller::ProcStat::print() const
 {
-  // TODO RS
-  //  const Char_t* Controller::sStatClName[Controller::kNStatCl] = {"Inp: ", "Acc: "};
-  //  const Char_t* Controller::sStatName[Controller::kMaxStat] =
-  //  {"runs", "Ev.Coll", "Ev.Cosm", "Trc.Coll", "Trc.Cosm"};
+  const auto& stat0 = data[kInput];
+  LOGP(info, "StatSeen: Vtx: {:10} Tracks: {:10} TracksWVtx: {:10}", stat0[kVertices], stat0[kTracks], stat0[kTracksWithVertex]);
+  const auto& stat1 = data[kAccepted];
+  LOGP(info, "StatAcc : Vtx: {:10} Tracks: {:10} TracksWVtx: {:10}", stat1[kVertices], stat1[kTracks], stat1[kTracksWithVertex]);
 }
 
 const Char_t* Controller::sMPDataExt = ".mille";
+const Char_t* Controller::sMPDataTxtExt = ".mille_txt";
 
 const Char_t* Controller::sDetectorName[Controller::kNDetectors] = {"ITS", "TPC", "TRD", "TOF", "HMPID"}; //RSREM
 
@@ -85,14 +87,11 @@ const Char_t* Controller::sDetectorName[Controller::kNDetectors] = {"ITS", "TPC"
 //                                                                 AliGeomManager::kMUON, AliGeomManager::kEMCAL}; TODO(milettri, shahoian): needs detector IDs previously stored in AliGeomManager
 const int Controller::sSkipLayers[Controller::kNLrSkip] = {0, 0, 0, 0}; // TODO(milettri, shahoian): needs AliGeomManager - remove this line after fix.
 
-
-const Char_t* Controller::sHStatName[Controller::kNHVars] = {
-  "Runs", "Ev.Inp", "Ev.VtxOK", "Tr.Inp", "Tr.2Fit", "Tr.2FitVC", "Tr.2PrMat", "Tr.2ResDer", "Tr.Stored", "Tr.Acc", "Tr.ContRes"};
-
 //________________________________________________________________
-Controller::Controller(DetID::mask_t detmask)
-  : mDetMask(detmask)
+Controller::Controller(DetID::mask_t detmask, GTrackID::mask_t trcmask, bool cosmic, bool useMC, int instID)
+  : mDetMask(detmask), mMPsrc(trcmask), mUseMC(useMC), mInstanceID(instID)
 {
+  setCosmic(cosmic);
   init();
 }
 
@@ -100,20 +99,9 @@ Controller::Controller(DetID::mask_t detmask)
 Controller::~Controller()
 {
   // d-tor
-  if (mMPRecFile) {
-    closeMPRecOutput();
-  }
-  if (mMille) {
-    closeMilleOutput();
-  }
-  if (mResidFile) {
-    closeResidOutput();
-  }
-  //
-  for (int i = 0; i < DetID::nDetectors; i++) {
-    delete mDetectors[i];
-  }
-  delete mHistoStat;
+  closeMPRecOutput();
+  closeMilleOutput();
+  closeResidOutput();
   //
 }
 
@@ -123,12 +111,42 @@ void Controller::init()
   if (mDetMask[DetID::ITS]) {
     addDetector(new AlignableDetectorITS(this));
   }
+  if (mDetMask[DetID::TRD]) {
+    addDetector(new AlignableDetectorTRD(this));
+  }
+  if (mDetMask[DetID::TPC]) {
+    addDetector(new AlignableDetectorTPC(this));
+  }
+  if (mDetMask[DetID::TOF]) {
+    addDetector(new AlignableDetectorTOF(this));
+  }
+  for (int src = GIndex::NSources; src--;) {
+    if (mMPsrc[src]) {
+      mTrackSources.push_back(src);
+    }
+  }
   mVtxSens = std::make_unique(this);
+  mVtxSens->setInternalID(1);
+  const auto& algConf = AlignConfig::Instance();
+  if (algConf.MPRecOutFraction > 0. || mInstanceID == 0) {
+    mMPRecOutFraction = std::abs(algConf.MPRecOutFraction);
+  }
+  if (algConf.controlFraction > 0. || mInstanceID == 0) {
+    mControlFraction = std::abs(algConf.controlFraction);
+  }
 }
 
 //________________________________________________________________
 void Controller::process()
 {
+  o2::steer::MCKinematicsReader mcReader;
+  if (mUseMC) {
+    if (!mcReader.initFromDigitContext("collisioncontext.root")) {
+      throw std::invalid_argument("initialization of MCKinematicsReader failed");
+    }
+  }
+  auto timerStart = std::chrono::system_clock::now();
+  int nVtx = 0, nVtxAcc = 0, nTrc = 0, nTrcAcc = 0;
   for (auto id = DetID::First; id <= DetID::Last; id++) {
     auto* det = getDetector(id);
     if (det) {
@@ -140,51 +158,122 @@ void Controller::process()
   auto primVerGIs = mRecoData->getPrimaryVertexMatchedTracks();
   const auto& algConf = AlignConfig::Instance();
   // process vertices with contributor tracks
-  std::unordered_map mAmbigTable;
+  std::unordered_map ambigTable;
   int nvRefs = primVer2TRefs.size();
-  bool fieldON = std::abs(o2::base::PropagatorD::Instance()->getNominalBz()) > 0.1;
+  bool fieldON = std::abs(PropagatorD::Instance()->getNominalBz()) > 0.1;
+
   for (int ivref = 0; ivref < nvRefs; ivref++) {
     const o2::dataformats::PrimaryVertex* vtx = (ivref < nvRefs - 1) ? &primVertices[ivref] : nullptr;
     bool useVertexConstrain = false;
     if (vtx) {
       auto nContrib = vtx->getNContributors();
+      // check cov matrix since data reconstructed with < 6797a257f5ab8ffaec32d56dddb0a321939bdf1c may have negative errors
+      if (vtx->getSigmaX2() < 0. || vtx->getSigmaY2() < 0. || vtx->getSigmaZ2() < 0.) {
+        continue;
+      }
       useVertexConstrain = nContrib >= algConf.vtxMinCont && nContrib <= algConf.vtxMaxCont;
+      mStat.data[ProcStat::kInput][ProcStat::kVertices]++;
     }
     auto& trackRef = primVer2TRefs[ivref];
-    LOGP(info, "processing vtref {} of {}, with {} tracks ", ivref, nvRefs, trackRef.getEntries());
-
-    for (int src = GIndex::NSources; src--;) {
+    if (algConf.verbose > 1) {
+      LOGP(info, "processing vtref {} of {} with {} tracks, {}", ivref, nvRefs, trackRef.getEntries(), vtx ? vtx->asString() : std::string{});
+    }
+    nVtx++;
+    bool newVtx = true;
+    for (int src : mTrackSources) {
       if ((GIndex::getSourceDetectorsMask(src) & mDetMask).none()) { // do we need this source?
         continue;
       }
       int start = trackRef.getFirstEntryOfSource(src), end = start + trackRef.getEntriesOfSource(src);
       for (int ti = start; ti < end; ti++) {
-        auto& trackIndex = primVerGIs[ti];
-        int npnt = 0;
-        LOGP(info, "processing track {} {} of vtref {}, use vertex: {}", ti, trackIndex.asString(), ivref, useVertexConstrain);
-        if (trackIndex.isAmbiguous() && mAmbigTable.find(trackIndex) != mAmbigTable.end()) { // was it already processed?
-          continue;
+        auto trackIndex = primVerGIs[ti];
+        mAlgTrack->setCurrentTrackID(trackIndex);
+        bool tpcIn = false;
+        if (trackIndex.isAmbiguous()) {
+          auto& ambSeen = ambigTable[trackIndex];
+          if (ambSeen) { // processed
+            continue;
+          }
+          ambSeen = true;
+        }
+        mStat.data[ProcStat::kInput][ProcStat::kTracks]++;
+        if (vtx) {
+          mStat.data[ProcStat::kInput][ProcStat::kTracksWithVertex]++;
         }
+        int npnt = 0;
         auto contributorsGID = mRecoData->getSingleDetectorRefs(trackIndex);
-        resetForNextTrack();
 
-        const auto& trcKin = mRecoData->getTrackParam(trackIndex); // RS FIXME : make sure this is a barrel track
+        std::string trComb;
+        for (int ig = 0; ig < GIndex::NSources; ig++) {
+          if (contributorsGID[ig].isIndexSet()) {
+            trComb += " " + contributorsGID[ig].asString();
+          }
+        }
+        if (algConf.verbose > 1) {
+          LOG(info) << "processing track " << trackIndex.asString() << " contributors: " << trComb;
+        }
+        resetForNextTrack();
+        nTrc++;
+        // RS const auto& trcOut = mRecoData->getTrackParamOut(trackIndex);
+        auto trcOut = mRecoData->getTrackParamOut(trackIndex);
+        const auto& trcIn = mRecoData->getTrackParam(trackIndex);
         // check detectors contributions
         AlignableDetector* det = nullptr;
-        if ((det = getDetector(DetID::ITS)) && contributorsGID[GIndex::ITS].isIndexSet()) {
-          npnt += det->processPoints(trackIndex, false); // collision tracks -> no inversion
+        int ndet = 0, npntDet = 0;
+
+        if ((det = getDetector(DetID::ITS))) {
+          if (contributorsGID[GIndex::ITS].isIndexSet() && (npntDet = det->processPoints(contributorsGID[GIndex::ITS], algConf.minITSClusters, false)) > 0) {
+            npnt += npntDet;
+            ndet++;
+          } else if (mAllowAfterburnerTracks && contributorsGID[GIndex::ITSAB].isIndexSet() && (npntDet = det->processPoints(contributorsGID[GIndex::ITSAB], 2, false)) > 0) {
+            npnt += npntDet;
+            ndet++;
+          } else {
+            continue;
+          }
+        }
+        if ((det = getDetector(DetID::TPC)) && contributorsGID[GIndex::TPC].isIndexSet()) {
+          float t0 = 0, t0err = 0;
+          mRecoData->getTrackTime(trackIndex, t0, t0err);
+          ((AlignableDetectorTPC*)det)->setTrackTimeStamp(t0);
+          npntDet = det->processPoints(contributorsGID[GIndex::TPC], algConf.minTPCClusters, false);
+          if (npntDet > 0) {
+            npnt += npntDet;
+            ndet++;
+            tpcIn = true;
+          }
+        }
+
+        if ((det = getDetector(DetID::TRD)) && contributorsGID[GIndex::TRD].isIndexSet() && (npntDet = det->processPoints(contributorsGID[GIndex::TRD], algConf.minTRDTracklets, false)) > 0) {
+          npnt += npntDet;
+          ndet++;
+        }
+        if ((det = getDetector(DetID::TOF)) && contributorsGID[GIndex::TOF].isIndexSet() && (npntDet = det->processPoints(contributorsGID[GIndex::TOF], algConf.minTOFClusters, false)) > 0) {
+          npnt += npntDet;
+          ndet++;
         }
         // other detectors
+        if (algConf.verbose > 1) {
+          LOGP(info, "processing track {} {} of vtref {}, Ndets:{}, Npoints: {}, use vertex: {} | Kin: {} Kout: {}", ti, trackIndex.asString(), ivref, ndet, npnt, useVertexConstrain && trackIndex.isPVContributor(), trcIn.asString(), trcOut.asString());
+        }
+        if (ndet < algConf.minDetectors || (tpcIn && ndet == 1)) { // we don't want TPC only track
+          continue;
+        }
         if (npnt < algConf.minPointTotal) {
+          if (algConf.verbose > 0) {
+            LOGP(info, "too few points {} < {}", npnt, algConf.minPointTotal);
+          }
           continue;
         }
-
-        mAlgTrack->setRefPoint(mRefPoint.get());
-        if (npnt && trackIndex.isPVContributor() && useVertexConstrain) {
-          addVertexConstraint(*vtx);
+        bool vtxCont = false;
+        if (trackIndex.isPVContributor() && useVertexConstrain) {
+          mAlgTrack->copyFrom(trcIn); // copy kinematices of inner track just for propagation to the vertex
+          if (addVertexConstraint(*vtx)) {
+            mAlgTrack->setRefPoint(mRefPoint.get()); // set vertex as a reference point
+            vtxCont = true;
+          }
         }
-        // RS FIXME Account stat.
-        mAlgTrack->copyFrom(trcKin); // copy kinematices
+        mAlgTrack->copyFrom(trcOut); // copy kinematices of outer track as the refit will be done inward
         mAlgTrack->setFieldON(fieldON);
         mAlgTrack->sortPoints();
 
@@ -194,19 +283,307 @@ void Controller::process()
           LOG(error) << "AliAlgTrack->GetInnerPointID() cannot be 0";
         }
         if (!mAlgTrack->iniFit()) {
+          if (algConf.verbose > 0) {
+            LOGP(warn, "iniFit failed");
+          }
           continue;
         }
+        // compare refitted and original track
+        if (mDebugOutputLevel) {
+          trackParam_t trcAlgRef(*mAlgTrack.get());
+          std::array dpar{};
+          std::array dcov{};
+          for (int i = 0; i < 5; i++) {
+            dpar[i] = trcIn.getParam(i);
+          }
+          for (int i = 0; i < 15; i++) {
+            dcov[i] = trcIn.getCov()[i];
+          }
+          trackParam_t trcOrig(trcIn.getX(), trcIn.getAlpha(), dpar, dcov, trcIn.getCharge());
+          if (PropagatorD::Instance()->propagateToAlphaX(trcOrig, trcAlgRef.getAlpha(), trcAlgRef.getX(), true)) {
+            (*mDBGOut) << "trcomp"
+                       << "orig=" << trcOrig << "fit=" << trcAlgRef << "\n";
+          }
+        }
+        // RS: this is to substitute the refitter track by MC truth, just for debugging
+        /*
+        if (mUseMC) {
+          auto lbl = mRecoData->getTrackMCLabel(trackIndex);
+          if (lbl.isValid()) {
+            o2::MCTrack mcTrack = *mcReader.getTrack(lbl);
+            std::array xyz{(float)mcTrack.GetStartVertexCoordinatesX(),(float)mcTrack.GetStartVertexCoordinatesY(),(float)mcTrack.GetStartVertexCoordinatesZ()},
+              pxyz{(float)mcTrack.GetStartVertexMomentumX(),(float)mcTrack.GetStartVertexMomentumY(),(float)mcTrack.GetStartVertexMomentumZ()};
+            std::array cv21{10., 0.,10., 0.,0.,10., 0.,0.,0.,1.,   0.,0.,0.,0.,1., 0.,0.,0.,0.,0.,1.};
+            trcOut.set(xyz, pxyz, cv21, trcOut.getSign(), false);
+            mAlgTrack->copyFrom(trcOut);
+          }
+        }
+        */
         if (!mAlgTrack->processMaterials()) {
+          if (algConf.verbose > 0) {
+            LOGP(warn, "processMaterials failed");
+          }
           continue;
         }
         mAlgTrack->defineDOFs();
         if (!mAlgTrack->calcResidDeriv()) {
+          if (algConf.verbose > 0) {
+            LOGP(warn, "calcResidDeriv failed");
+          }
           continue;
         }
-        mAlgTrack->Print("ldrcu");
+        if (mDebugOutputLevel && mAlgTrackDbg.setTrackParam(mAlgTrack.get())) {
+          mAlgTrackDbg.mGID = trackIndex;
+          (*mDBGOut) << "algtrack"
+                     << "runNumber=" << mTimingInfo.runNumber
+                     << "tfID=" << mTimingInfo.tfCounter
+                     << "orbit=" << mTimingInfo.firstTForbit
+                     << "bz=" << PropagatorD::Instance()->getNominalBz()
+                     << "t=" << mAlgTrackDbg << "\n";
+        }
+        if (mUseMC && mDebugOutputLevel > 1) {
+          auto lbl = mRecoData->getTrackMCLabel(trackIndex);
+          if (lbl.isValid()) {
+            std::vector pntX, pntY, pntZ, trcX, trcY, trcZ, prpX, prpY, prpZ, alpha, xsens, pntXTF, pntYTF, pntZTF, resY, resZ;
+            std::vector detid, volid;
+
+            o2::MCTrack mcTrack = *mcReader.getTrack(lbl);
+            trackParam_t recTrack{*mAlgTrack};
+            for (int ip = 0; ip < mAlgTrack->getNPoints(); ip++) {
+              double tmp[3], tmpg[3];
+              auto* pnt = mAlgTrack->getPoint(ip);
+              auto* sens = pnt->getSensor();
+              detid.emplace_back(pnt->getDetID());
+              volid.emplace_back(pnt->getVolID());
+              TGeoHMatrix t2g;
+              sens->getMatrixT2G(t2g);
+              t2g.LocalToMaster(pnt->getXYZTracking(), tmpg);
+              pntX.emplace_back(tmpg[0]);
+              pntY.emplace_back(tmpg[1]);
+              pntZ.emplace_back(tmpg[2]);
+              double xyz[3]{pnt->getXTracking(), pnt->getYTracking(), pnt->getZTracking()};
+              xyz[1] += mAlgTrack->getResidual(0, ip);
+              xyz[2] += mAlgTrack->getResidual(1, ip);
+              t2g.LocalToMaster(xyz, tmpg);
+              trcX.emplace_back(tmpg[0]);
+              trcY.emplace_back(tmpg[1]);
+              trcZ.emplace_back(tmpg[2]);
+
+              pntXTF.emplace_back(pnt->getXTracking());
+              pntYTF.emplace_back(pnt->getYTracking());
+              pntZTF.emplace_back(pnt->getZTracking());
+              resY.emplace_back(mAlgTrack->getResidual(0, ip));
+              resZ.emplace_back(mAlgTrack->getResidual(1, ip));
+
+              alpha.emplace_back(pnt->getAlphaSens());
+              xsens.emplace_back(pnt->getXSens());
+            }
+            (*mDBGOut) << "mccomp"
+                       << "mcTr=" << mcTrack << "recTr=" << recTrack << "gid=" << trackIndex << "lbl=" << lbl << "vtxConst=" << vtxCont
+                       << "pntX=" << pntX << "pntY=" << pntY << "pntZ=" << pntZ
+                       << "trcX=" << trcX << "trcY=" << trcY << "trcZ=" << trcZ
+                       << "alp=" << alpha << "xsens=" << xsens
+                       << "pntXTF=" << pntXTF << "pntYTF=" << pntYTF << "pntZTF=" << pntZTF
+                       << "resY=" << resY << "resZ=" << resZ
+                       << "detid=" << detid << "volid=" << volid << "\n";
+          }
+        }
+        mStat.data[ProcStat::kAccepted][ProcStat::kTracks]++;
+        if (vtxCont) {
+          mStat.data[ProcStat::kAccepted][ProcStat::kTracksWithVertex]++;
+        }
+        nTrcAcc++;
+        if (newVtx) {
+          newVtx = false;
+          mStat.data[ProcStat::kAccepted][ProcStat::kVertices]++;
+          nVtxAcc++;
+        }
+        storeProcessedTrack(trackIndex);
       }
     }
   }
+  auto timerEnd = std::chrono::system_clock::now();
+  std::chrono::duration duration = timerEnd - timerStart;
+  LOGP(info, "Processed TF {}: {} vertices ({} used), {} tracks ({} used) in {} ms", mNTF, nVtx, nVtxAcc, nTrc, nTrcAcc, duration.count());
+  mNTF++;
+}
+
+//________________________________________________________________
+void Controller::processCosmic()
+{
+  auto timerStart = std::chrono::system_clock::now();
+  const auto tracks = mRecoData->getCosmicTracks();
+  if (!tracks.size()) {
+    LOGP(info, "Skipping TF {}: No cosmic tracks", mNTF);
+    mNTF++;
+    return;
+  }
+  int nTrc = 0, nTrcAcc = 0;
+  for (auto id = DetID::First; id <= DetID::Last; id++) {
+    auto* det = getDetector(id);
+    if (det) {
+      det->prepareDetectorData(); // in case the detector needs to preprocess the RecoContainer data
+    }
+  }
+  const auto& algConf = AlignConfig::Instance();
+  bool fieldON = std::abs(PropagatorD::Instance()->getNominalBz()) > 0.1;
+  for (const auto& track : tracks) {
+    resetForNextTrack();
+    nTrc++;
+    mStat.data[ProcStat::kInput][ProcStat::kCosmic]++;
+    std::array contributorsGID[2] = {mRecoData->getSingleDetectorRefs(track.getRefBottom()), mRecoData->getSingleDetectorRefs(track.getRefTop())};
+    bool hasTRD = false, hasITS = false, hasTPC = false, hasTOF = false;
+    if (contributorsGID[0][GTrackID::TRD].isIndexSet() || contributorsGID[1][GTrackID::TRD].isIndexSet()) {
+      hasTRD = true;
+    }
+    if (contributorsGID[0][GTrackID::TOF].isIndexSet() || contributorsGID[1][GTrackID::TOF].isIndexSet()) {
+      hasTOF = true;
+    }
+    if (contributorsGID[0][GTrackID::TPC].isIndexSet() || contributorsGID[1][GTrackID::TPC].isIndexSet()) {
+      hasTPC = true;
+    }
+    if (contributorsGID[0][GTrackID::ITS].isIndexSet() || contributorsGID[1][GTrackID::ITS].isIndexSet()) {
+      hasITS = true;
+    }
+    // check detectors contributions
+    AlignableDetector* det = nullptr;
+    int ndet = 0, npnt = 0;
+    bool accTrack = true;
+    bool tpcIn = false;
+    if ((det = getDetector(DetID::ITS))) {
+      int npntDet = 0;
+      for (int ibt = 0; ibt < 2; ibt++) {
+        int npntDetBT = 0;
+        mAlgTrack->setCurrentTrackID(ibt ? track.getRefBottom() : track.getRefTop());
+        if (contributorsGID[ibt][GIndex::ITS].isIndexSet() && (npntDetBT = det->processPoints(contributorsGID[ibt][GIndex::ITS], algConf.minITSClustersCosmLeg, ibt)) < 0) {
+          accTrack = false;
+          break;
+        }
+        npntDet += npntDetBT;
+      }
+      if (!accTrack || npntDet < algConf.minITSClustersCosm) {
+        continue;
+      }
+      if (npntDet) {
+        ndet++;
+        npnt += npntDet;
+      }
+    }
+
+    if ((det = getDetector(DetID::TPC))) {
+      int npntDet = 0;
+      ((AlignableDetectorTPC*)det)->setTrackTimeStamp(track.getTimeMUS().getTimeStamp());
+      for (int ibt = 0; ibt < 2; ibt++) {
+        int npntDetBT = 0;
+        mAlgTrack->setCurrentTrackID(ibt ? track.getRefBottom() : track.getRefTop());
+        if (contributorsGID[ibt][GIndex::TPC].isIndexSet() && (npntDetBT = det->processPoints(contributorsGID[ibt][GIndex::TPC], algConf.minTPCClustersCosmLeg, ibt)) < 0) {
+          accTrack = false;
+          break;
+        }
+        npntDet += npntDetBT;
+      }
+      if (!accTrack || npntDet < algConf.minTPCClustersCosm) {
+        continue;
+      }
+      if (npntDet) {
+        ndet++;
+        npnt += npntDet;
+        tpcIn = true;
+      }
+    }
+
+    if ((det = getDetector(DetID::TRD))) {
+      int npntDet = 0;
+      for (int ibt = 0; ibt < 2; ibt++) {
+        int npntDetBT = 0;
+        mAlgTrack->setCurrentTrackID(ibt ? track.getRefBottom() : track.getRefTop());
+        if (contributorsGID[ibt][GIndex::TRD].isIndexSet() && (npntDetBT = det->processPoints(contributorsGID[ibt][GIndex::TRD], algConf.minTRDTrackletsCosmLeg, ibt)) < 0) {
+          accTrack = false;
+          break;
+        }
+        npntDet += npntDetBT;
+      }
+      if (!accTrack || npntDet < algConf.minTRDTrackletsCosm) {
+        continue;
+      }
+      if (npntDet) {
+        ndet++;
+        npnt += npntDet;
+      }
+    }
+
+    if ((det = getDetector(DetID::TOF))) {
+      int npntDet = 0;
+      for (int ibt = 0; ibt < 2; ibt++) {
+        int npntDetBT = 0;
+        mAlgTrack->setCurrentTrackID(ibt ? track.getRefBottom() : track.getRefTop());
+        if (contributorsGID[ibt][GIndex::TOF].isIndexSet() && (npntDetBT = det->processPoints(contributorsGID[ibt][GIndex::TOF], algConf.minTOFClustersCosmLeg, ibt)) < 0) {
+          accTrack = false;
+          break;
+        }
+        npntDet += npntDetBT;
+      }
+      if (!accTrack || npntDet < algConf.minTOFClustersCosm) {
+        continue;
+      }
+      if (npntDet) {
+        ndet++;
+        npnt += npntDet;
+      }
+    }
+    if (algConf.verbose > 1) {
+      LOGP(info, "processing cosmic track B-Leg:{} T-Leg:{}, Ndets:{}, Npoints: {}", track.getRefBottom().asString(), track.getRefTop().asString(), ndet, npnt);
+    }
+    if (ndet < algConf.minDetectorsCosm /* || (tpcIn && ndet == 1)*/) {
+      continue;
+    }
+    if (npnt < algConf.minPointTotalCosm) {
+      if (algConf.verbose > 0) {
+        LOGP(info, "too few points {} < {}", npnt, algConf.minPointTotalCosm);
+      }
+      continue;
+    }
+    mAlgTrack->setCosmic(true);
+    mAlgTrack->copyFrom(mRecoData->getTrackParamOut(track.getRefBottom())); // copy kinematices of outer track as the refit will be done inward
+    mAlgTrack->setFieldON(fieldON);
+    mAlgTrack->sortPoints();
+    if (!mAlgTrack->iniFit()) {
+      if (algConf.verbose > 0) {
+        LOGP(warn, "iniFit failed");
+      }
+      continue;
+    }
+    if (!mAlgTrack->processMaterials()) {
+      if (algConf.verbose > 0) {
+        LOGP(warn, "processMaterials failed");
+      }
+      continue;
+    }
+    mAlgTrack->defineDOFs();
+    if (!mAlgTrack->calcResidDeriv()) {
+      if (algConf.verbose > 0) {
+        LOGP(warn, "calcResidDeriv failed");
+      }
+      continue;
+    }
+    if (mDebugOutputLevel && mAlgTrackDbg.setTrackParam(mAlgTrack.get())) {
+      mAlgTrackDbg.mGID = track.getRefBottom();
+      mAlgTrackDbg.mGIDCosmUp = track.getRefTop();
+      (*mDBGOut) << "algtrack"
+                 << "runNumber=" << mTimingInfo.runNumber
+                 << "tfID=" << mTimingInfo.tfCounter
+                 << "orbit=" << mTimingInfo.firstTForbit
+                 << "bz=" << PropagatorD::Instance()->getNominalBz()
+                 << "t=" << mAlgTrackDbg << "\n";
+    }
+    storeProcessedTrack();
+    mStat.data[ProcStat::kAccepted][ProcStat::kCosmic]++;
+    nTrcAcc++;
+  }
+  auto timerEnd = std::chrono::system_clock::now();
+  std::chrono::duration duration = timerEnd - timerStart;
+  LOGP(info, "Processed cosmic TF {}: {} tracks ({} used) in {} ms", mNTF, nTrc, nTrcAcc, duration.count());
+  mNTF++;
 }
 
 //________________________________________________________________
@@ -224,8 +601,6 @@ void Controller::initDetectors()
   int dofCnt = 0;
   // special fake sensor for vertex constraint point
   // it has special T2L matrix adjusted for each track, no need to init it here
-  mVtxSens = std::make_unique(this);
-  mVtxSens->setInternalID(1);
   mVtxSens->prepareMatrixL2G();
   mVtxSens->prepareMatrixL2GIdeal();
   dofCnt += mVtxSens->getNDOFs();
@@ -315,13 +690,10 @@ void Controller::assignDOFs()
   mGloParVal.clear();
   mGloParErr.clear();
   mGloParLab.clear();
-  mOrderedLbl.clear();
   mLbl2ID.clear();
   mGloParVal.reserve(ndofTOT);
   mGloParErr.reserve(ndofTOT);
   mGloParLab.reserve(ndofTOT);
-  mOrderedLbl.reserve(ndofTOT);
-  mLbl2ID.reserve(ndofTOT);
 
   mVtxSens->assignDOFs();
 
@@ -336,14 +708,14 @@ void Controller::assignDOFs()
   if (ndfOld > 0 && ndfOld != mNDOFs) {
     LOG(error) << "Recalculated NDOFs=" << mNDOFs << " not equal to saved NDOFs=" << ndfOld;
   }
-  //
-  // build Lbl <-> parID table
-  /* FIXME RS TODO
-  Sort(mNDOFs, mGloParLab, mLbl2ID, false); // sort in increasing order
-  for (int i = mNDOFs; i--;) {
-    mOrderedLbl[i] = mGloParLab[mLbl2ID[i]];
+  // build Lbl -> parID table
+  for (int i = 0; i < ndofTOT; i++) {
+    int& id = mLbl2ID[mGloParLab[i]];
+    if (id != 0) {
+      LOGP(fatal, "parameters {} and {} share the same label {}", id - 1, i, mGloParLab[i]);
+    }
+    id = i + 1;
   }
-  */
   //
 }
 
@@ -351,7 +723,7 @@ void Controller::assignDOFs()
 void Controller::addDetector(AlignableDetector* det)
 {
   // add detector constructed externally to alignment framework
-  mDetectors[det->getDetID()] = det;
+  mDetectors[det->getDetID()].reset(det);
   mNDet++;
 }
 
@@ -384,392 +756,26 @@ bool Controller::checkDetectorPoints(const int* npsel) const
   return ndOK >= AlignConfig::Instance().minDetAcc[mTracksType];
 }
 
-//FIXME(milettri): needs AliESDtrack
-////_________________________________________________________
-//uint32_t Controller::AcceptTrack(const AliESDtrack* esdTr, bool strict) const
-//{
-//  // decide if the track should be processed
-//  AlignableDetector* det = 0;
-//  uint32_t detAcc = 0;
-//  if (mFieldOn && esdTr->Pt() < mPtMin[mTracksType]){
-//    return 0;}
-//  if (Abs(esdTr->Eta()) > mEtaMax[mTracksType]){
-//    return 0;}
-//  //
-//  for (auto id=DetID::First; id<=DetID::Last; id++) {
-//
-//    if (!(det = getDetector(id)) || det->isDisabled(mTracksType)){
-//      continue;}
-//    if (!det->AcceptTrack(esdTr, mTracksType)) {
-//      if (strict && det->isObligatory(mTracksType)){
-//        return 0;}
-//      else
-//        continue;
-//    }
-//    //
-//    detAcc |= 0x1 << idet;
-//  }
-//  if (numberOfBitsSet(detAcc) < mMinDetAcc[mTracksType]){
-//    return 0;}
-//  return detAcc;
-//  //
-//}
-
-//FIXME(milettri): needs AliESDtrack
-////_________________________________________________________
-//uint32_t Controller::AcceptTrackCosmic(const AliESDtrack* esdPairCosm[kNCosmLegs]) const
-//{
-//  // decide if the pair of tracks making cosmic track should be processed
-//  uint32_t detAcc = 0, detAccLeg;
-//  for (int i = kNCosmLegs; i--;) {
-//    detAccLeg = AcceptTrack(esdPairCosm[i], mCosmicSelStrict); // missing obligatory detectors in one leg might be allowed
-//    if (!detAccLeg){
-//      return 0;}
-//    detAcc |= detAccLeg;
-//  }
-//  if (mCosmicSelStrict){
-//    return detAcc;}
-//  //
-//  // for non-stric selection check convolution of detector presence
-//  if (!checkDetectorPattern(detAcc)){
-//    return 0;}
-//  return detAcc;
-//  //
-//}
-
-//FIXME(milettri): needs AliESDEvent
-////_________________________________________________________
-//void Controller::SetESDEvent(const AliESDEvent* ev)
-//{
-//  // attach event to analyse
-//  fESDEvent = ev;
-//  // setup magnetic field if needed
-//  if (fESDEvent &&
-//      (!TGeoGlobalMagField::Instance()->GetField() ||
-//       !smallerAbs(fESDEvent->GetMagneticField() - AliTrackerBase::GetBz(), 5e-4))) {
-//    fESDEvent->InitMagneticField();
-//  }
-//}
-
-//FIXME(milettri): needs AliESDEvent
-////_________________________________________________________
-//bool Controller::ProcessEvent(const AliESDEvent* esdEv)
-//{
-//  // process event
-//  const int kProcStatFreq = 100;
-//  static int evCount = 0;
-//  if (!(evCount % kProcStatFreq)) {
-//    ProcInfo_t procInfo;
-//    gSystem->GetProcInfo(&procInfo);
-//    LOG(info) << "ProcStat: CPUusr:" << int(procInfo.fCpuUser) << " CPUsys:" << int(procInfo.fCpuSys) << " RMem:" << int(procInfo.fMemResident / 1024) << " VMem:" << int(procInfo.fMemVirtual / 1024);
-//  }
-//  evCount++;
-//  //
-//  SetESDEvent(esdEv);
-//  //
-//  if (esdEv->getRunNumber() != getRunNumber()){
-//    SetRunNumber(esdEv->getRunNumber());
-//  }
-//  //
-//  setCosmic(esdEv->GetEventSpecie() == AliRecoParam::kCosmic ||
-//            (esdEv->GetNumberOfCosmicTracks() > 0 && !esdEv->GetPrimaryVertexTracks()->GetStatus()));
-//  //
-//  fillStatHisto(kEvInp);
-//  //
-//#if DEBUG > 2
-//  LOG << "Processing event " << esdEv->GetEventNumberInFile() << " of ev.specie " << esdEv->GetEventSpecie() << " -> Ntr: " << esdEv->GetNumberOfTracks() << " NtrCosm: " << esdEv->GetNumberOfCosmicTracks();
-//#endif
-//  //
-//  setFieldOn(Abs(esdEv->GetMagneticField()) > kAlmost0Field);
-//  if (!isCosmic() && !CheckSetVertex(esdEv->GetPrimaryVertexTracks())){
-//    return false;}
-//  fillStatHisto(kEvVtx);
-//  //
-//  int ntr = 0, accTr = 0;
-//  if (isCosmic()) {
-//    mStat[kInpStat][kEventCosm]++;
-//    ntr = esdEv->GetNumberOfCosmicTracks();
-//    fillStatHisto(kTrackInp, ntr);
-//    for (int itr = 0; itr < ntr; itr++) {
-//      accTr += ProcessTrack(esdEv->GetCosmicTrack(itr));
-//    }
-//    if (accTr){
-//      mStat[kAccStat][kEventCosm]++;}
-//  } else {
-//    mStat[kInpStat][kEventColl]++;
-//    ntr = esdEv->GetNumberOfTracks();
-//    fillStatHisto(kTrackInp, ntr);
-//    for (int itr = 0; itr < ntr; itr++) {
-//      //      int accTrOld = accTr;
-//      accTr += ProcessTrack(esdEv->GetTrack(itr));
-//      /*
-//      if (accTr>accTrOld && mCResid) {
-// int ndf = mCResid.getNPoints()*2-5;
-// if (mCResid.getChi2()/ndf>20 || !mCResid.getKalmanDone()
-//     || mCResid.getChi2K()/ndf>20) {
-//   printf("BAD FIT for %d\n",itr);
-// }
-//        mCResid.Print("er");
-//      }
-//      */
-//    }
-//    if (accTr){
-//      mStat[kAccStat][kEventColl]++;}
-//  }
-//  //
-//  fillStatHisto(kTrackAcc, accTr);
-//  //
-//  if (accTr) {
-//    LOG(info) << "Processed event " << esdEv->GetEventNumberInFile() << " of ev.specie " << esdEv->GetEventSpecie() << " -> Accepted: " << accTr << " of " << ntr << " tracks";
-//  }
-//  return true;
-//}
-
-//FIXME(milettri): needs AliESDtrack
 //_________________________________________________________
-//bool Controller::ProcessTrack(const AliESDtrack* esdTr)
-//{
-//  // process single track
-//  //
-//  mStat[kInpStat][kTrackColl]++;
-//  fESDTrack[0] = esdTr;
-//  fESDTrack[1] = 0;
-//  //
-//  int nPnt = 0;
-//  const AliESDfriendTrack* trF = esdTr->GetFriendTrack();
-//  if (!trF){
-//    return false;}
-//  const AliTrackPointArray* trPoints = trF->GetTrackPointArray();
-//  if (!trPoints || (nPnt = trPoints->GetNPoints()) < 1){
-//    return false;}
-//  //
-//  uint32_t detAcc = AcceptTrack(esdTr);
-//  if (!detAcc){
-//    return false;}
-//  //
-//  resetDetectors();
-//  mAlgTrack->Clear();
-//  //
-//  // process the track points for each detector,
-//  AlignableDetector* det = 0;
-//  for (auto id=DetID::First; id<=DetID::Last; id++) {
-//    if (!(detAcc & (0x1 << idet))){  // RS!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-//      continue;}
-//    det = getDetector(id);
-//    if (det->ProcessPoints(esdTr, mAlgTrack) < det->getNPointsSel(kColl)) {
-//      detAcc &= ~(0x1 << idet); // did not survive, suppress detector in the track
-//      if (det->isObligatory(kColl)){
-//        return false;}
-//    }
-//    if (numberOfBitsSet(detAcc) < mMinDetAcc[kColl]){
-//      return false;} // abandon track
-//  }
-//  //
-//  if (mAlgTrack->getNPoints() < getMinPoints()){
-//    return false;}
-//  // fill needed points (tracking frame) in the mAlgTrack
-//  mRefPoint->setContainsMeasurement(false);
-//  mRefPoint->setContainsMaterial(false);
-//  mAlgTrack->addPoint(mRefPoint); // reference point which the track will refer to
-//  //
-//  mAlgTrack->copyFrom(esdTr);
-//  if (!getFieldOn()){
-//    mAlgTrack->imposePtBOff(mDefPtBOff[utils::kColl]);}
-//  mAlgTrack->setFieldON(getFieldOn());
-//  mAlgTrack->sortPoints();
-//  //
-//  // at this stage the points are sorted from maxX to minX, the latter corresponding to
-//  // reference point (e.g. vertex) with X~0. The mAlgTrack->getInnerPointID() points on it,
-//  // hence mAlgTrack->getInnerPointID() is the 1st really measured point. We will set the
-//  // alpha of the reference point to alpha of the barrel sector corresponding to this
-//  // 1st measured point
-//  int pntMeas = mAlgTrack->getInnerPointID() - 1;
-//  if (pntMeas < 0) { // this should not happen
-//    mAlgTrack->Print("p meas");
-//    LOG(fatal) << "AlignmentTrack->getInnerPointID() cannot be 0";
-//  }
-//  // do we want to add the vertex as a measured point ?
-//  if (!addVertexConstraint()) { // no constrain, just reference point w/o measurement
-//    mRefPoint->setXYZTracking(0, 0, 0);
-//    mRefPoint->setAlphaSens(sector2Alpha(mAlgTrack->getPoint(pntMeas)->getAliceSector()));
-//  } else
-//    fillStatHisto(kTrackFitInpVC);
-//  //
-//  fillStatHisto(kTrackFitInp);
-//  if (!mAlgTrack->iniFit()){
-//    return false;}
-//  fillStatHisto(kTrackProcMatInp);
-//  if (!mAlgTrack->processMaterials()){
-//    return false;}
-//  mAlgTrack->defineDOFs();
-//  //
-//  fillStatHisto(kTrackResDerInp);
-//  if (!mAlgTrack->calcResidDeriv()){
-//    return false;}
-//  //
-//  if (!storeProcessedTrack(mMPOutType & ~kContR)){
-//    return false;} // store derivatives for MP
-//  //
-//  if (getProduceControlRes() &&                                   // need control residuals, ignore selection fraction if this is the
-//      (mMPOutType == kContR || gRandom->Rndm() < mControlFrac)) { // output requested
-//    if (!testLocalSolution() || !storeProcessedTrack(kContR)){
-//      return false;}
-//  }
-//  //
-//  fillStatHisto(kTrackStore);
-//  //
-//  mStat[kAccStat][kTrackColl]++;
-//  //
-//  return true;
-//}
-
-//FIXME(milettri): needs AliESDVertex
-////_________________________________________________________
-//bool Controller::CheckSetVertex(const AliESDVertex* vtx)
-//{
-//  // vertex selection/constraint check
-//  if (!vtx) {
-//    fVertex = 0;
-//    return true;
-//  }
-//  int ncont = vtx->GetNContributors();
-//  if (mVtxMinCont > 0 && mVtxMinCont > ncont) {
-//#if DEBUG > 2
-//    LOG(info) << "Rejecting event with " << % d << " vertex contributors (min " << % d << " asked)", ncont, mVtxMinCont);
-//#endif
-//    return false;
-//  }
-//  if (mVtxMaxCont > 0 && ncont > mVtxMaxCont) {
-//#if DEBUG > 2
-//    LOG(info) << "Rejecting event with " << % d << " vertex contributors (max " << % d << " asked)", ncont, mVtxMaxCont);
-//#endif
-//    return false;
-//  }
-//  fVertex = (ncont >= mVtxMinContVC) ? vtx : 0; // use vertex as a constraint
-//  return true;
-//}
-
-//FIXME(milettri): needs AliESDCosmicTrack
-////_________________________________________________________
-//bool Controller::ProcessTrack(const AliESDCosmicTrack* cosmTr)
-//{
-//  // process single cosmic track
-//  //
-//  mStat[kInpStat][kTrackCosm]++;
-//  int nPnt = 0;
-//  fESDTrack[0] = 0;
-//  fESDTrack[1] = 0;
-//  //
-//  for (int leg = kNCosmLegs; leg--;) {
-//    const AliESDtrack* esdTr =
-//      fESDEvent->GetTrack(leg == kCosmLow ? cosmTr->GetESDLowerTrackIndex() : cosmTr->GetESDUpperTrackIndex());
-//    const AliESDfriendTrack* trF = esdTr->GetFriendTrack();
-//    if (!trF){
-//      return false;}
-//    const AliTrackPointArray* trPoints = trF->GetTrackPointArray();
-//    if (!trPoints || (nPnt += trPoints->GetNPoints()) < 1){
-//      return false;}
-//    //
-//    fESDTrack[leg] = esdTr;
-//  }
-//  //
-//  uint32_t detAcc = AcceptTrackCosmic(fESDTrack);
-//  if (!detAcc){
-//    return false;}
-//  //
-//  resetDetectors();
-//  mAlgTrack->Clear();
-//  mAlgTrack->setCosmic(true);
-//  //
-//  // process the track points for each detector,
-//  // fill needed points (tracking frame) in the mAlgTrack
-//  mRefPoint->setContainsMeasurement(false);
-//  mRefPoint->setContainsMaterial(false);
-//  mAlgTrack->addPoint(mRefPoint); // reference point which the track will refer to
-//  //
-//  AlignableDetector* det = 0;
-//  int npsel[kNDetectors] = {0};
-//  for (int nPleg = 0, leg = kNCosmLegs; leg--;) {
-//    for (auto id=DetID::First; id<=DetID::Last; id++) {
-//      if (!(detAcc & (0x1 << idet))){
-//        continue;}
-//      det = getDetector(id);
-//      //
-//      // upper leg points marked as the track going in inverse direction
-//      int np = det->ProcessPoints(fESDTrack[leg], mAlgTrack, leg == kCosmUp);
-//      if (np < det->getNPointsSel(Cosm) && mCosmicSelStrict &&
-//          det->isObligatory(Cosm))
-//        return false;
-//      npsel[id] += np;
-//      nPleg += np;
-//    }
-//    if (nPleg < getMinPoints()){
-//      return false;}
-//  }
-//  // last check on legs-combined patter
-//  if (!checkDetectorPoints(npsel)){
-//    return false;}
-//  //
-//  mAlgTrack->copyFrom(cosmTr);
-//  if (!getFieldOn()){
-//    mAlgTrack->imposePtBOff(mDefPtBOff[utils::Cosm]);}
-//  mAlgTrack->setFieldON(getFieldOn());
-//  mAlgTrack->sortPoints();
-//  //
-//  // at this stage the points are sorted from maxX to minX, the latter corresponding to
-//  // reference point (e.g. vertex) with X~0. The mAlgTrack->getInnerPointID() points on it,
-//  // hence mAlgTrack->getInnerPointID() is the 1st really measured point. We will set the
-//  // alpha of the reference point to alpha of the barrel sector corresponding to this
-//  // 1st measured point
-//  int pntMeas = mAlgTrack->getInnerPointID() - 1;
-//  if (pntMeas < 0) { // this should not happen
-//    mAlgTrack->Print("p meas");
-//    LOG(fatal) << "AlignmentTrack->getInnerPointID() cannot be 0";
-//  }
-//  mRefPoint->setAlphaSens(sector2Alpha(mAlgTrack->getPoint(pntMeas)->getAliceSector()));
-//  //
-//  fillStatHisto(kTrackFitInp);
-//  if (!mAlgTrack->iniFit()){
-//    return false;}
-//  //
-//  fillStatHisto(kTrackProcMatInp);
-//  if (!mAlgTrack->processMaterials()){
-//    return false;}
-//  mAlgTrack->defineDOFs();
-//  //
-//  fillStatHisto(kTrackResDerInp);
-//  if (!mAlgTrack->calcResidDeriv()){
-//    return false;}
-//  //
-//  if (!storeProcessedTrack(mMPOutType & ~kContR)){
-//    return false;} // store derivatives for MP
-//  //
-//  if (getProduceControlRes() &&                                   // need control residuals, ignore selection fraction if this is the
-//      (mMPOutType == kContR || gRandom->Rndm() < mControlFrac)) { // output requested
-//    if (!testLocalSolution() || !storeProcessedTrack(kContR)){
-//      return false;}
-//  }
-//  //
-//  fillStatHisto(kTrackStore);
-//  mStat[kAccStat][kTrackCosm]++;
-//  return true;
-//}
-
-//_________________________________________________________
-bool Controller::storeProcessedTrack(int what)
+bool Controller::storeProcessedTrack(o2::dataformats::GlobalTrackID tid)
 {
   // write alignment track
   bool res = true;
-  if ((what & kMille)) {
+  const auto& conf = AlignConfig::Instance();
+  if (conf.MilleOut) {
     res &= fillMilleData();
   }
-  if ((what & kMPRec)) {
-    res &= fillMPRecData();
+  float rnd = gRandom->Rndm();
+  if (mMPRecOutFraction > rnd) {
+    res &= fillMPRecData(tid);
   }
-  if ((what & kContR)) {
-    res &= fillControlData();
+  if ((mControlFraction > rnd) && mAlgTrack->testLocalSolution()) {
+    res &= fillControlData(tid);
   }
   //
+  if (!res) {
+    LOGP(error, "storeProcessedTrack failed");
+  }
   return res;
 }
 
@@ -778,20 +784,19 @@ bool Controller::fillMilleData()
 {
   // store MP2 data in Mille format
   if (!mMille) {
-    mMille = std::make_unique(fmt::format("{}{}", mMPDatFileName, sMPDataExt).c_str());
+    const auto& conf = AlignConfig::Instance();
+    mMilleFileName = fmt::format("{}_{:08d}_{:010d}{}", AlignConfig::Instance().mpDatFileName, mTimingInfo.runNumber, mTimingInfo.tfCounter, conf.MilleOutBin ? sMPDataExt : sMPDataTxtExt);
+    mMille = std::make_unique(mMilleFileName.c_str(), conf.MilleOutBin);
   }
-  //
   if (!mAlgTrack->getDerivDone()) {
     LOG(error) << "Track derivatives are not yet evaluated";
     return false;
   }
-  int np(mAlgTrack->getNPoints()), nDGloTot(0); // total number global derivatives stored
-  int nParETP(mAlgTrack->getNLocExtPar());      // numnber of local parameters for reference track param
-  int nVarLoc(mAlgTrack->getNLocPar());         // number of local degrees of freedom in the track
-  float *buffDL(nullptr), *buffDG(nullptr);     // faster acces arrays
-  int* buffI(nullptr);
+  int np = mAlgTrack->getNPoints(), nDGloTot = 0; // total number global derivatives stored
+  int nParETP = mAlgTrack->getNLocExtPar();       // numnber of local parameters for reference track param
+  int nVarLoc = mAlgTrack->getNLocPar();          // number of local degrees of freedom in the track
   //
-  const int* gloParID(mAlgTrack->getGloParID()); // IDs of global DOFs this track depends on
+  const int* gloParID = mAlgTrack->getGloParID(); // IDs of global DOFs this track depends on
   for (int ip = 0; ip < np; ip++) {
     AlignmentPoint* pnt = mAlgTrack->getPoint(ip);
     if (pnt->containsMeasurement()) {
@@ -800,227 +805,119 @@ bool Controller::fillMilleData()
       if (!pnt->isStatOK()) {
         pnt->incrementStat();
       }
-      // check buffer sizes
-      {
-        if (mMilleDBuffer.GetSize() < nVarLoc + nDGlo) {
-          mMilleDBuffer.Set(100 + nVarLoc + nDGlo);
-        }
-        if (mMilleIBuffer.GetSize() < nDGlo) {
-          mMilleIBuffer.Set(100 + nDGlo);
-        }
-        buffDL = mMilleDBuffer.GetArray(); // faster acces
-        buffDG = buffDL + nVarLoc;         // faster acces
-        buffI = mMilleIBuffer.GetArray();  // faster acces
-      }
+      int milleIBufferG[nDGlo];
+      float milleDBufferG[nDGlo];
+      float milleDBufferL[nVarLoc];
+      std::memset(milleIBufferG, 0, sizeof(int) * nDGlo);
+      std::memset(milleDBufferG, 0, sizeof(float) * nDGlo);
+      std::memset(milleDBufferL, 0, sizeof(float) * nVarLoc);
       // local der. array cannot be 0-suppressed by Mille construction, need to reset all to 0
-      //
-      for (int idim = 0; idim < 2; idim++) { // 2 dimensional orthogonal measurement
-        memset(buffDL, 0, nVarLoc * sizeof(float));
+      for (int idim = 0; idim < 2; idim++) {                    // 2 dimensional orthogonal measurement
         const double* deriv = mAlgTrack->getDResDLoc(idim, ip); // array of Dresidual/Dparams_loc
         // derivatives over reference track parameters
         for (int j = 0; j < nParETP; j++) {
-          buffDL[j] = (isZeroAbs(deriv[j])) ? 0 : deriv[j];
+          milleDBufferL[j] = isZeroAbs(deriv[j]) ? 0. : deriv[j];
         }
         //
         // point may depend on material variables within these limits
-        int lp0 = pnt->getMinLocVarID(), lp1 = pnt->getMaxLocVarID();
-        for (int j = lp0; j < lp1; j++) {
-          buffDL[j] = (isZeroAbs(deriv[j])) ? 0 : deriv[j];
+        for (int j = pnt->getMinLocVarID(); j < pnt->getMaxLocVarID(); j++) {
+          milleDBufferL[j] = isZeroAbs(deriv[j]) ? 0. : deriv[j];
         }
-        //
         // derivatives over global params: this array can be 0-suppressed, no need to reset
-        int nGlo(0);
+        int nGlo = 0;
         deriv = mAlgTrack->getDResDGlo(idim, gloOffs);
         const int* gloIDP(gloParID + gloOffs);
         for (int j = 0; j < nDGlo; j++) {
-          if (!isZeroAbs(deriv[j])) {
-            buffDG[nGlo] = deriv[j];                 // value of derivative
-            buffI[nGlo++] = getGloParLab(gloIDP[j]); // global DOF ID + 1 (Millepede needs positive labels)
-          }
+          milleDBufferG[nGlo] = isZeroAbs(deriv[j]) ? 0. : deriv[j]; // value of derivative
+          milleIBufferG[nGlo++] = getGloParLab(gloIDP[j]);           // global DOF ID + 1 (Millepede needs positive labels)
         }
-        mMille->mille(nVarLoc, buffDL, nGlo, buffDG, buffI,
-                      mAlgTrack->getResidual(idim, ip), Sqrt(pnt->getErrDiag(idim)));
+        mMille->mille(nVarLoc, milleDBufferL, nGlo, milleDBufferG, milleIBufferG, mAlgTrack->getResidual(idim, ip), Sqrt(pnt->getErrDiag(idim)));
         nDGloTot += nGlo;
-        //
       }
     }
-    if (pnt->containsMaterial()) { // material point can add 4 or 5 otrhogonal pseudo-measurements
-      memset(buffDL, 0, nVarLoc * sizeof(float));
+    if (pnt->containsMaterial()) {     // material point can add 4 or 5 otrhogonal pseudo-measurements
       int nmatpar = pnt->getNMatPar(); // residuals (correction expectation value)
       //      const float* expMatCorr = pnt->getMatCorrExp(); // expected corrections (diagonalized)
       const float* expMatCov = pnt->getMatCorrCov(); // their diagonalized error matrix
       int offs = pnt->getMaxLocVarID() - nmatpar;    // start of material variables
       // here all derivatives are 1 = dx/dx
+      float milleDBufferL[nVarLoc];
+      std::memset(milleDBufferL, 0, sizeof(float) * nVarLoc);
       for (int j = 0; j < nmatpar; j++) { // mat. "measurements" don't depend on global params
         int j1 = j + offs;
-        buffDL[j1] = 1.0; // only 1 non-0 derivative
-        //mMille->mille(nVarLoc,buffDL,0,buffDG,buffI,expMatCorr[j],Sqrt(expMatCov[j]));
+        milleDBufferL[j1] = 1.0; // only 1 non-0 derivative
+        // mMille->mille(nVarLoc,milleDBufferL,0, nullptr, nullptr, expMatCorr[j], Sqrt(expMatCov[j]));
         // expectation for MS effect is 0
-        mMille->mille(nVarLoc, buffDL, 0, buffDG, buffI, 0, Sqrt(expMatCov[j]));
-        buffDL[j1] = 0.0; // reset buffer
+        mMille->mille(nVarLoc, milleDBufferL, 0, nullptr, nullptr, 0, Sqrt(expMatCov[j]));
+        milleDBufferL[j1] = 0.0; // reset buffer
       }
     } // material "measurement"
   }   // loop over points
   //
   if (!nDGloTot) {
     LOG(info) << "Track does not depend on free global parameters, discard";
-    mMille->kill();
+    mMille->clear();
     return false;
   }
-  mMille->end(); // store the record
+  mMille->finalise(); // store the record
   return true;
 }
 
 //_________________________________________________________
-bool Controller::fillMPRecData()
+bool Controller::fillMPRecData(o2::dataformats::GlobalTrackID tid)
 {
-  LOG(fatal) << __PRETTY_FUNCTION__ << " is disabled";
-  //FIXME(milettri): needs AliESDEvent
-  //  // store MP2 in MPRecord format
-  //  if (!mMPRecord){
-  //    initMPRecOutput();}
-  //  //
-  //  mMPRecord->Clear();
-  //  if (!mMPRecord->fillTrack(mAlgTrack, mGloParLab)){
-  //    return false;}
-  //  mMPRecord->SetRun(mRunNumber);
-  //  mMPRecord->setTimeStamp(fESDEvent->GetTimeStamp());
-  //  uint32_t tID = 0xffff & uint(fESDTrack[0]->GetID());
-  //  if (isCosmic()){
-  //    tID |= (0xffff & uint(fESDTrack[1]->GetID())) << 16;}
-  //  mMPRecord->setTrackID(tID);
-  //  mMPRecTree->Fill();
+  // store MP2 in MPRecord format
+  if (!mMPRecFile) {
+    initMPRecOutput();
+  }
+  mMPRecord.clear();
+  if (!mMPRecord.fillTrack(*mAlgTrack.get(), mGloParLab)) {
+    return false;
+  }
+  mMPRecord.setRun(mRunNumber);
+  mMPRecord.setFirstTFOrbit(mTimingInfo.firstTForbit);
+  mMPRecord.setTrackID(tid);
+  mMPRecTree->Fill();
   return true;
 }
 
 //_________________________________________________________
-bool Controller::fillControlData()
+bool Controller::fillControlData(o2::dataformats::GlobalTrackID tid)
 {
-  LOG(fatal) << __PRETTY_FUNCTION__ << " is disabled";
-  //FIXME(milettri): needs AliESDEvent
-  //  // store control residuals
-  //  if (!mCResid){
-  //    initResidOutput();}
-  //  //
-  //  int nps, np = mAlgTrack->getNPoints();
-  //  nps = (!mRefPoint->containsMeasurement()) ? np - 1 : np; // ref point is dummy?
-  //  if (nps < 0){
-  //    return true;}
-  //  //
-  //  mCResid.Clear();
-  //  if (!mCResid.fillTrack(mAlgTrack, mDoKalmanResid)){
-  //    return false;}
-  //  mCResid.setRun(mRunNumber);
-  //  mCResid.setTimeStamp(fESDEvent->GetTimeStamp());
-  //  mCResid.setBz(fESDEvent->GetMagneticField());
-  //  uint32_t tID = 0xffff & uint(fESDTrack[0]->GetID());
-  //  if (isCosmic()){
-  //    tID |= (0xffff & uint(fESDTrack[1]->GetID())) << 16;}
-  //  mCResid.setTrackID(tID);
-  //  //
-  //  mResidTree->Fill();
-  //  fillStatHisto(kTrackControl);
-  //  //
+  // store control residuals
+  if (!mResidFile) {
+    initResidOutput();
+  }
+  int nps, np = mAlgTrack->getNPoints();
+  nps = (!mRefPoint->containsMeasurement()) ? np - 1 : np; // ref point is dummy?
+  if (nps < 0) {
+    return true;
+  }
+  mCResid.clear();
+  if (!mCResid.fillTrack(*mAlgTrack.get(), AlignConfig::Instance().KalmanResid)) {
+    return false;
+  }
+  mCResid.setRun(mRunNumber);
+  mCResid.setFirstTFOrbit(mTimingInfo.firstTForbit);
+  mCResid.setBz(o2::base::PropagatorD::Instance()->getNominalBz());
+  mCResid.setTrackID(tid);
+  // if (isCosmic()) {
+  //   mCResid.setInvTrackID(tid);
+  // }
+  mResidTree->Fill();
   return true;
 }
 
 //_________________________________________________________
-void Controller::setRunNumber(int run)
-{
-  if (run == mRunNumber) {
-    return;
-  } // nothing to do
-  //
-  acknowledgeNewRun(run);
-}
-
-//_________________________________________________________
-void Controller::acknowledgeNewRun(int run)
+void Controller::setTimingInfo(const o2::framework::TimingInfo& ti)
 {
-  LOG(warning) << __PRETTY_FUNCTION__ << " yet incomplete";
-
-  o2::base::GeometryManager::loadGeometry();
-  o2::base::PropagatorImpl::initFieldFromGRP();
-  std::unique_ptr grp{o2::parameters::GRPObject::loadFrom()};
-
-  //FIXME(milettri): needs AliESDEvent
-  //  // load needed info for new run
-  //  if (run == mRunNumber){
-  //    return;} // nothing to do
-  //  if (run > 0) {
-  //    mStat[kAccStat][kRun]++;
-  //  }
-  //  if (mRunNumber > 0){
-  //    fillStatHisto(kRunDone);}
-  //  mRunNumber = run;
-  //  LOG(info) << "Processing new run " << mRunNumber;
-  //  //
-  //  // setup magnetic field
-  //  if (fESDEvent &&
-  //      (!TGeoGlobalMagField::Instance()->GetField() ||
-  //       !smallerAbs(fESDEvent->GetMagneticField() - AliTrackerBase::GetBz(), 5e-4))) {
-  //    fESDEvent->InitMagneticField();
-  //  }
-  //  //
-  //  if (!mUseRecoOCDB) {
-  //    LOG(warning) << "Reco-time OCDB will NOT be preloaded";
-  //    return;
-  //  }
-  //  LoadRecoTimeOCDB();
-  //  //
-  //  for (auto id=DetID::First; id<=DetID::Last; id++) {
-  //    AlignableDetector* det = getDetector(id);
-  //    if (!det->isDisabled()){
-  //      det->acknowledgeNewRun(run);}
-  //  }
-  //  //
-  //  // bring to virgin state
-  //  // CleanOCDB();
-  //  //
-  //  // LoadRefOCDB(); //??? we need to get back reference OCDB ???
-  //  //
-  //  mStat[kInpStat][kRun]++;
-  //  //
+  mTimingInfo = ti;
+  LOGP(info, "TIMING {} {}", ti.runNumber, ti.creation);
+  if (ti.runNumber != mRunNumber) {
+    mRunNumber = ti.runNumber;
+  }
 }
 
-// FIXME(milettri): needs OCDB
-////_________________________________________________________
-//bool Controller::LoadRecoTimeOCDB()
-//{
-//  // Load OCDB paths used for the reconstruction of data being processed
-//  // In order to avoid unnecessary uploads, the objects are not actually
-//  // loaded/cached but just added as specific paths with version
-//  LOG(info) << "Preloading Reco-Time OCDB for run " << mRunNumber << " from ESD UserInfo list";
-//  //
-//  CleanOCDB();
-//  //
-//  if (!mRecoOCDBConf.IsNull() && !gSystem->AccessPathName(mRecoOCDBConf.c_str(), kFileExists)) {
-//    LOG(info) << "Executing reco-time OCDB setup macro " << mRecoOCDBConf.c_str();
-//    gROOT->ProcessLine(Form(".x %s(%d)", mRecoOCDBConf.c_str(), mRunNumber));
-//    if (AliCDBManager::Instance()->IsDefaultStorageSet()){
-//      return true;}
-//    LOG(fatal) << "macro " << mRecoOCDBConf.c_str() << " failed to configure reco-time OCDB";
-//  } else
-//    LOG(warning) << "No reco-time OCDB config macro" << mRecoOCDBConf.c_str() << "  is found, will use ESD:UserInfo";
-//  //
-//  if (!mESDTree){
-//    LOG(fatal) << "Cannot preload Reco-Time OCDB since the ESD tree is not set";}
-//  const TTree* tr = mESDTree; // go the the real ESD tree
-//  while (tr->GetTree() && tr->GetTree() != tr)
-//    tr = tr->GetTree();
-//  //
-//  const TList* userInfo = const_cast(tr)->GetUserInfo();
-//  TMap* cdbMap = (TMap*)userInfo->FindObject("cdbMap");
-//  TList* cdbList = (TList*)userInfo->FindObject("cdbList");
-//  //
-//  if (!cdbMap || !cdbList) {
-//    userInfo->Print();
-//    LOG(fatal) << "Failed to extract cdbMap and cdbList from UserInfo list";
-//  }
-//  //
-//  return PreloadOCDB(mRunNumber, cdbMap, cdbList);
-//}
-
 //____________________________________________
 void Controller::Print(const Option_t* opt) const
 {
@@ -1047,33 +944,11 @@ void Controller::Print(const Option_t* opt) const
   if (mRefRunNumber >= 0) {
     printf("(%d)", mRefRunNumber);
   }
-  //
-  printf("%-40s:\t%s\n", "Output OCDB path", mOutCDBPath.c_str());
-  printf("%-40s:\t%s/%s\n", "Output OCDB comment/responsible",
-         mOutCDBComment.c_str(), mOutCDBResponsible.c_str());
-  printf("%-40s:\t%6d:%6d\n", "Output OCDB run range", mOutCDBRunRange[0], mOutCDBRunRange[1]);
-  //
-  printf("%-40s:\t%s\n", "Filename for MillePede steering", mMPSteerFileName.c_str());
-  printf("%-40s:\t%s\n", "Filename for MillePede parameters", mMPParFileName.c_str());
-  printf("%-40s:\t%s\n", "Filename for MillePede constraints", mMPConFileName.c_str());
-  printf("%-40s:\t%s\n", "Filename for control residuals:", mResidFileName.c_str());
-  printf("%-40s:\t%.3f\n", "Fraction of control tracks", mControlFrac);
-  printf("MPData output :\t");
-  if (getProduceMPData()) {
-    printf("%s%s ", mMPDatFileName.c_str(), sMPDataExt);
-  }
-  if (getProduceMPRecord()) {
-    printf("%s%s ", mMPDatFileName.c_str(), ".root");
-  }
-  printf("\n");
+  AlignConfig::Instance().printKeyValues();
   //
   if (opts.Contains("stat")) {
     printStatistics();
   }
-
-  if (opts.Contains("conf")) {
-    AlignConfig::Instance().printKeyValues(true);
-  }
 }
 
 //________________________________________________________
@@ -1100,128 +975,73 @@ void Controller::resetForNextTrack()
 //____________________________________________
 bool Controller::testLocalSolution()
 {
-  LOG(fatal) << __PRETTY_FUNCTION__ << " is disabled";
-  //FIXME(milettri): needs AliSymMatrix
-  //  // test track local solution
-  //  TVectorD rhs;
-  //  AliSymMatrix* mat = BuildMatrix(rhs);
-  //  if (!mat){
-  //    return false;}
-  //  //  mat->Print("long data");
-  //  //  rhs.Print();
-  //  TVectorD vsl(rhs);
-  //  if (!mat->SolveChol(rhs, vsl, true)) {
-  //    delete mat;
-  //    return false;
-  //  }
-  //  //
-  //  /*
-  //  // print solution vector
-  //  int nlocpar = mAlgTrack->getNLocPar();
-  //  int nlocparETP = mAlgTrack->getNLocExtPar(); // parameters of external track param
-  //  printf("ETP Update: ");
-  //  for (int i=0;inlocparETP) printf("Mat.Corr. update:\n");
-  //  for (int ip=mAlgTrack->getNPoints();ip--;) {
-  //    AlignmentPoint* pnt = mAlgTrack->getPoint(ip);
-  //    int npm = pnt->getNMatPar();
-  //    const float* expMatCov  = pnt->getMatCorrCov(); // its error
-  //    int offs  = pnt->getMaxLocVarID() - npm;
-  //    for (int ipar=0;ipar sig:%+.3e -> pull: %+.2e\n",
-  //            ip,ipar,parI,vsl[parI],Sqrt((*mat)(parI,parI)), err,vsl[parI]/err);
-  //    }
-  //  }
-  //  */
-  //  //
-  //  // increment current params by new solution
-  //  rhs.SetElements(mAlgTrack->getLocPars());
-  //  vsl += rhs;
-  //  mAlgTrack->setLocPars(vsl.GetMatrixArray());
-  //  mAlgTrack->calcResiduals();
-  //  delete mat;
-  //  //
+  // test track local solution
+  int npnt = mAlgTrack->getNPoints(), nlocpar = mAlgTrack->getNLocPar();
+  double mat[nlocpar][nlocpar], rhs[nlocpar];
+  std::memset(mat, 0, sizeof(double) * nlocpar * nlocpar);
+  std::memset(rhs, 0, sizeof(double) * nlocpar);
+  for (int ip = npnt; ip--;) {
+    AlignmentPoint* pnt = mAlgTrack->getPoint(ip);
+    if (pnt->containsMeasurement()) {
+      for (int idim = 2; idim--;) {                                                           // each point has 2 position residuals
+        double resid = mAlgTrack->getResidual(idim, ip), sg2inv = 1. / pnt->getErrDiag(idim); // residual and its inv. error
+        auto deriv = mAlgTrack->getDResDLoc(idim, ip);                                        // array of Dresidual/Dparams
+        for (int parI = 0; parI < nlocpar; parI++) {
+          rhs[parI] -= deriv[parI] * resid * sg2inv;
+          for (int parJ = parI; parJ < nlocpar; parJ++) {
+            mat[parI][parJ] += deriv[parI] * deriv[parJ] * sg2inv;
+          }
+        }
+      } // loop over 2 orthogonal measurements at the point
+    }   // derivarives at measured points
+    // if the point contains material, consider its expected kinks, eloss as measurements
+    if (pnt->containsMaterial()) { // at least 4 parameters: 2 spatial + 2 angular kinks with 0 expectaction
+      int npm = pnt->getNMatPar();
+      // const float* expMatCorr = pnt->getMatCorrExp(); // expected correction (diagonalized) // RS??
+      const float* expMatCov = pnt->getMatCorrCov(); // its error
+      int offs = pnt->getMaxLocVarID() - npm;
+      for (int ipar = 0; ipar < npm; ipar++) {
+        int parI = offs + ipar;
+        // expected
+        // rhs[parI] -= expMatCorr[ipar]/expMatCov[ipar]; // consider expectation as measurement // RS??
+        mat[parI][parI] += 1. / expMatCov[ipar]; // this measurement is orthogonal to all others
+      }
+    } // material effect descripotion params
+    //
+  }
+  o2::math_utils::SymMatrixSolver solver(nlocpar, 1);
+  for (int i = 0; i < nlocpar; i++) {
+    for (int j = i; j < nlocpar; j++) {
+      solver.A(i, j) = mat[i][j];
+    }
+    solver.B(i, 0) = rhs[i];
+  }
+  solver.solve();
+  // increment current params by new solution
+  auto& pars = mAlgTrack->getLocParsV();
+  for (int i = 0; i < nlocpar; i++) {
+    pars[i] += solver.B(i, 0);
+  }
+  mAlgTrack->calcResiduals();
   return true;
 }
 
-// FIXME(milettri): needs AliSymMatrix
-////____________________________________________
-// AliSymMatrix* Controller::BuildMatrix(TVectorD& vec)
-//{
-//   // build matrix/vector for local track
-//   int npnt = mAlgTrack->getNPoints();
-//   int nlocpar = mAlgTrack->getNLocPar();
-//   //
-//   vec.ResizeTo(nlocpar);
-//   memset(vec.GetMatrixArray(), 0, nlocpar * sizeof(double));
-//   AliSymMatrix* matp = new AliSymMatrix(nlocpar);
-//   AliSymMatrix& mat = *matp;
-//   //
-//   for (int ip = npnt; ip--;) {
-//     AlignmentPoint* pnt = mAlgTrack->getPoint(ip);
-//     //
-//     if (pnt->containsMeasurement()) {
-//       //      pnt->Print("meas");
-//       for (int idim = 2; idim--;) {                       // each point has 2 position residuals
-//         double sigma2 = pnt->getErrDiag(idim);            // residual error
-//         double resid = mAlgTrack->getResidual(idim, ip);  // residual
-//         const double* deriv = mAlgTrack->getDResDLoc(idim, ip); // array of Dresidual/Dparams
-//         //
-//         double sg2inv = 1. / sigma2;
-//         for (int parI = nlocpar; parI--;) {
-//           vec[parI] -= deriv[parI] * resid * sg2inv;
-//           //  printf("%d %d %d %+e %+e %+e -> %+e\n",ip,idim,parI,sg2inv,deriv[parI],resid,vec[parI]);
-//           //   for (int parJ=nlocpar;parJ--;) {
-//           for (int parJ = parI + 1; parJ--;) {
-//             mat(parI, parJ) += deriv[parI] * deriv[parJ] * sg2inv;
-//           }
-//         }
-//       } // loop over 2 orthogonal measurements at the point
-//     }   // derivarives at measured points
-//     //
-//     // if the point contains material, consider its expected kinks, eloss
-//     // as measurements
-//     if (pnt->containsMaterial()) {
-//       // at least 4 parameters: 2 spatial + 2 angular kinks with 0 expectaction
-//       int npm = pnt->getNMatPar();
-//       // const float* expMatCorr = pnt->getMatCorrExp(); // expected correction (diagonalized)
-//       const float* expMatCov = pnt->getMatCorrCov(); // its error
-//       int offs = pnt->getMaxLocVarID() - npm;
-//       for (int ipar = 0; ipar < npm; ipar++) {
-//         int parI = offs + ipar;
-//         // expected
-//         // vec[parI] -= expMatCorr[ipar]/expMatCov[ipar]; // consider expectation as measurement
-//         mat(parI, parI) += 1. / expMatCov[ipar]; // this measurement is orthogonal to all others
-//                                                  //printf("Pnt:%3d MatVar:%d DOF %3d | ExpVal: %+e Cov: %+e\n",ip,ipar,parI, expMatCorr[ipar], expMatCov[ipar]);
-//       }
-//     } // material effect descripotion params
-//     //
-//   } // loop over track points
-//   //
-//   return matp;
-// }
-
 //____________________________________________
 void Controller::initMPRecOutput()
 {
   // prepare MP record output
-  mMPRecFile.reset(TFile::Open(fmt::format("{}{}", mMPDatFileName, ".root").c_str(), "recreate"));
+  mMPRecFile.reset(TFile::Open(fmt::format("{}_{:08d}_{:010d}{}", AlignConfig::Instance().mpDatFileName, mTimingInfo.runNumber, mTimingInfo.tfCounter, ".root").c_str(), "recreate"));
   mMPRecTree = std::make_unique("mpTree", "MPrecord Tree");
-  mMPRecTree->Branch("mprec", "Millepede2Record", &mMPRecordPtr);
-  //
+  mMPRecTree->Branch("mprec", "o2::align::Millepede2Record", &mMPRecordPtr);
 }
 
 //____________________________________________
 void Controller::initResidOutput()
 {
   // prepare residual output
-  mResidFile.reset(TFile::Open(mResidFileName.c_str(), "recreate"));
+  mResidFile.reset(TFile::Open(fmt::format("{}_{:08d}_{:010d}{}", AlignConfig::Instance().residFileName, mTimingInfo.runNumber, mTimingInfo.tfCounter, ".root").c_str(), "recreate"));
   mResidTree = std::make_unique("res", "Control Residuals");
-  mResidTree->Branch("t", "ResidualsController", &mCResidPtr);
-  //
+  mResidTree->Branch("t", "o2::align::ResidualsController", &mCResidPtr);
 }
 
 //____________________________________________
@@ -1231,7 +1051,7 @@ void Controller::closeMPRecOutput()
   if (!mMPRecFile) {
     return;
   }
-  LOG(info) << "Closing " << mMPRecFile->GetName();
+  LOGP(info, "Writing tree {} with {} entries to {}", mMPRecTree->GetName(), mMPRecTree->GetEntries(), mMPRecFile->GetName());
   mMPRecFile->cd();
   mMPRecTree->Write();
   mMPRecTree.reset();
@@ -1252,83 +1072,29 @@ void Controller::closeResidOutput()
   mResidTree.reset();
   mResidFile->Close();
   mResidFile.reset();
-  mCResid.Clear();
+  mCResid.clear();
 }
 
 //____________________________________________
 void Controller::closeMilleOutput()
 {
   // close output
+  bool compress = false;
   if (mMille) {
-    LOG(info) << "Closing " << mMPDatFileName.c_str() << sMPDataExt;
+    LOG(info) << "Closing " << mMilleFileName;
+    compress = AlignConfig::Instance().GZipMilleOut;
+  } else {
+    return;
   }
   mMille.reset();
-}
-
-//____________________________________________
-void Controller::setMPDatFileName(const char* name)
-{
-  // set output file name
-  mMPDatFileName = name;
-  if (mMPDatFileName.empty()) {
-    mMPDatFileName = "mpData";
-  }
-  //
-}
-
-//____________________________________________
-void Controller::setMPParFileName(const char* name)
-{
-  // set MP params output file name
-  mMPParFileName = name;
-  if (mMPParFileName.empty()) {
-    mMPParFileName = "mpParams.txt";
-  }
-  //
-}
-
-//____________________________________________
-void Controller::setMPConFileName(const char* name)
-{
-  // set MP constraints output file name
-  mMPConFileName = name;
-  if (mMPConFileName.empty()) {
-    mMPConFileName = "mpConstraints.txt";
-  }
-  //
-}
-
-//____________________________________________
-void Controller::setMPSteerFileName(const char* name)
-{
-  // set MP constraints output file name
-  mMPSteerFileName = name;
-  if (mMPSteerFileName.empty()) {
-    mMPSteerFileName = "mpConstraints.txt";
-  }
-  //
-}
-
-//____________________________________________
-void Controller::setResidFileName(const char* name)
-{
-  // set output file name
-  mResidFileName = name;
-  if (mResidFileName.empty()) {
-    mResidFileName = "mpControlRes.root";
-  }
-  //
-}
-
-//____________________________________________
-void Controller::setOutCDBPath(const char* name)
-{
-  // set output storage name
-  mOutCDBPath = name;
-  if (mOutCDBPath.empty()) {
-    mOutCDBPath = "local://outOCDB";
+  if (compress) {
+    std::string cmd = fmt::format("sh -c \"gzip {}\"", mMilleFileName);
+    LOG(info) << "Compressing: " << cmd;
+    const auto sysRet = gSystem->Exec(cmd.c_str());
+    if (sysRet != 0) {
+      LOGP(alarm, "non-zero exit code {} for cmd={}", sysRet, cmd);
+    }
   }
-  //
 }
 
 //____________________________________________
@@ -1353,18 +1119,18 @@ void Controller::setObligatoryDetector(DetID detID, int trtype, bool v)
 //____________________________________________
 bool Controller::addVertexConstraint(const o2::dataformats::PrimaryVertex& vtx)
 {
-  LOG(info) << "adding vtx constraint";
-  auto* prop = o2::base::PropagatorD::Instance();
+  auto* prop = PropagatorD::Instance();
   const auto& conf = AlignConfig::Instance();
   o2::dataformats::DCA dcaInfo;
   AlignmentTrack::trackParam_t trcDCA(*mAlgTrack.get());
-  if (prop->propagateToDCABxByBz(vtx, trcDCA, conf.maxStep, conf.matCorType, &dcaInfo)) {
+  if (!prop->propagateToDCABxByBz(vtx, trcDCA, conf.maxStep, MatCorrType(conf.matCorType), &dcaInfo)) {
     return false;
   }
   // RS FIXME do selections if needed
   mVtxSens->setAlpha(trcDCA.getAlpha());
+  const auto* addErr = mVtxSens->getAddError();
   double xyz[3] = {vtx.getX(), vtx.getY(), vtx.getZ()}, xyzT[3];
-  double c[3] = {0.5 * (vtx.getSigmaX2() + vtx.getSigmaY2()), 0., vtx.getSigmaZ2()};
+  double c[3] = {0.5 * (vtx.getSigmaX2() + vtx.getSigmaY2()) + addErr[0] * addErr[0], 0., vtx.getSigmaZ2() + addErr[1] * addErr[1]};
 
   mVtxSens->applyCorrection(xyz);
   mVtxSens->getMatrixT2L().MasterToLocal(xyz, xyzT);
@@ -1379,83 +1145,24 @@ bool Controller::addVertexConstraint(const o2::dataformats::PrimaryVertex& vtx)
   return true;
 }
 
-//FIXME(milettri): needs OCDB
-////______________________________________________________
-//void Controller::writeCalibrationResults() const
-//{
-//  // writes output calibration
-//  CleanOCDB();
-//  AliCDBManager::Instance()->SetDefaultStorage(mOutCDBPath.c_str());
-//  //
-//  AlignableDetector* det;
-//  for (auto id=DetID::First; id<=DetID::Last; id++) {
-//    if (!(det = getDetector(id)) || det->isDisabled()){
-//      continue;
-//    }
-//    det->writeCalibrationResults();
-//  }
-//  //
-//}
-
-//FIXME(milettri): needs OCDB
-////______________________________________________________
-//void Controller::SetOutCDBRunRange(int rmin, int rmax)
-//{
-//  // set output run range
-//  mOutCDBRunRange[0] = rmin >= 0 ? rmin : 0;
-//  mOutCDBRunRange[1] = rmax > mOutCDBRunRange[0] ? rmax : AliCDBRunRange::Infinity();
-//}
-
-//FIXME(milettri): needs OCDB
-////______________________________________________________
-//bool Controller::LoadRefOCDB()
-//{
-//  // setup OCDB whose objects will be used as a reference with respect to which the
-//  // alignment/calibration will prodice its corrections.
-//  // Detectors which need some reference calibration data must use this one
-//  //
-//  //
-//  LOG(info) << "Loading reference OCDB");
-//  CleanOCDB();
-//  AliCDBManager* man = AliCDBManager::Instance();
-//  //
-//  if (!mRefOCDBConf.IsNull() && !gSystem->AccessPathName(mRefOCDBConf.c_str(), kFileExists)) {
-//    LOG(info) << "Executing reference OCDB setup macro %s", mRefOCDBConf.c_str());
-//    if (mRefRunNumber > 0){
-//      gROOT->ProcessLine(Form(".x %s(%d)", mRefOCDBConf.c_str(), mRefRunNumber));}
-//    else
-//      gROOT->ProcessLine(Form(".x %s", mRefOCDBConf.c_str()));
-//  } else {
-//    LOG(warning) << "No reference OCDB config macro "<SetRaw(true);
-//    man->SetRun(AliCDBRunRange::Infinity());
-//  }
-//  //
-//  if (AliGeomManager::GetGeometry()) {
-//    LOG(info) << "Destroying current geometry before loading reference one");
-//    AliGeomManager::Destroy();
-//  }
-//  AliGeomManager::LoadGeometry("geometry.root");
-//  if (!AliGeomManager::GetGeometry()){
-//    LOG(fatal) << "Failed to load geometry, cannot run");}
-//  //
-//  TString detList = "";
-//  for (int i = 0; i < kNDetectors; i++) {
-//    detList += getDetNameByDetID(i);
-//    detList += " ";
-//  }
-//  AliGeomManager::ApplyAlignObjsFromCDB(detList.c_str());
-//  //
-//  mRefOCDBLoaded++;
-//  //
-//  return true;
-//}
+//______________________________________________________
+void Controller::writeCalibrationResults() const
+{
+  // writes output calibration
+  AlignableDetector* det;
+  for (auto id = DetID::First; id <= DetID::Last; id++) {
+    if (!(det = getDetector(id)) || det->isDisabled()) {
+      continue;
+    }
+    det->writeCalibrationResults();
+  }
+}
 
 //________________________________________________________
 AlignableDetector* Controller::getDetOfDOFID(int id) const
 {
   // return detector owning DOF with this ID
-  for (auto id = DetID::First; id <= DetID::Last; id++) {
+  for (auto idet = DetID::First; idet <= DetID::Last; idet++) {
     AlignableDetector* det = getDetector(id);
     if (det && det->ownsDOFID(id)) {
       return det;
@@ -1468,8 +1175,8 @@ AlignableDetector* Controller::getDetOfDOFID(int id) const
 AlignableVolume* Controller::getVolOfDOFID(int id) const
 {
   // return volume owning DOF with this ID
-  for (auto id = DetID::First; id <= DetID::Last; id++) {
-    AlignableDetector* det = getDetector(id);
+  for (auto idet = DetID::First; idet <= DetID::Last; idet++) {
+    AlignableDetector* det = getDetector(idet);
     if (det && det->ownsDOFID(id)) {
       return det->getVolOfDOFID(id);
     }
@@ -1481,17 +1188,17 @@ AlignableVolume* Controller::getVolOfDOFID(int id) const
 }
 
 //________________________________________________________
-void Controller::terminate(bool doStat)
+AlignableVolume* Controller::getVolOfLabel(int lbl) const
+{
+  // return volume owning DOF with this label
+  const auto& ent = mLbl2ID.find(lbl);
+  return ent == mLbl2ID.end() ? nullptr : getVolOfDOFID(ent->second);
+}
+
+//________________________________________________________
+void Controller::terminate()
 {
   // finalize processing
-  if (mRunNumber > 0) {
-    fillStatHisto(kRunDone);
-  }
-  if (doStat) {
-    if (mVtxSens) {
-      mVtxSens->fillDOFStat(mDOFStat);
-    }
-  }
   //
   for (auto id = DetID::First; id <= DetID::Last; id++) {
     if (getDetector(id)) {
@@ -1501,7 +1208,7 @@ void Controller::terminate(bool doStat)
   closeMPRecOutput();
   closeMilleOutput();
   closeResidOutput();
-  Print("stat");
+  //  Print("stat");
   //
 }
 
@@ -1526,6 +1233,27 @@ Char_t* Controller::getDOFLabelTxt(int idf) const
 
 //********************* interaction with PEDE **********************
 
+//______________________________________________________
+void Controller::writeLabeledPedeResults() const
+{
+  // attach labels to millepede.res-like file
+  FILE* parFl = fopen(AlignConfig::Instance().mpLabFileName.c_str(), "w+");
+  fprintf(parFl, "parameters\n");
+  if (mVtxSens) {
+    mVtxSens->writeLabeledPedeResults(parFl);
+  }
+
+  for (auto id = DetID::First; id <= DetID::Last; id++) {
+    AlignableDetector* det = getDetector(id);
+    if (!det || det->isDisabled()) {
+      continue;
+    }
+    det->writeLabeledPedeResults(parFl);
+    //
+  }
+  fclose(parFl);
+}
+
 //______________________________________________________
 void Controller::genPedeSteerFile(const Option_t* opt) const
 {
@@ -1535,22 +1263,22 @@ void Controller::genPedeSteerFile(const Option_t* opt) const
          kOn,
          kOnOn };
   const char* cmt[3] = {"  ", "! ", "!!"};
-  const char* kSolMeth[] = {"inversion", "diagonalization", "fullGMRES", "sparseGMRES", "cholesky", "HIP"};
+  const char* kSolMeth[] = {"inversion", "diagonalization", "fullGMRES", "sparseMINRES", "cholesky", "HIP"};
   const int kDefNIter = 3;     // default number of iterations to ask
   const float kDefDelta = 0.1; // def. delta to exit
   TString opts = opt;
   opts.ToLower();
   LOG(info) << "Generating MP2 templates:\n "
-            << "Steering   :\t" << mMPSteerFileName << "\n"
-            << "Parameters :\t" << mMPParFileName << "\n"
-            << "Constraints:\t" << mMPConFileName << "\n";
+            << "Steering   :\t" << AlignConfig::Instance().mpSteerFileName << "\n"
+            << "Parameters :\t" << AlignConfig::Instance().mpParFileName << "\n"
+            << "Constraints:\t" << AlignConfig::Instance().mpConFileName << "\n";
   //
-  FILE* parFl = fopen(mMPParFileName.c_str(), "w+");
-  FILE* strFl = fopen(mMPSteerFileName.c_str(), "w+");
+  FILE* parFl = fopen(AlignConfig::Instance().mpParFileName.c_str(), "w+");
+  FILE* strFl = fopen(AlignConfig::Instance().mpSteerFileName.c_str(), "w+");
   //
   // --- template of steering file
-  fprintf(strFl, "%-20s%s %s\n", mMPParFileName.c_str(), cmt[kOnOn], "parameters template");
-  fprintf(strFl, "%-20s%s %s\n", mMPConFileName.c_str(), cmt[kOnOn], "constraints template");
+  fprintf(strFl, "%-20s%s %s\n", AlignConfig::Instance().mpParFileName.c_str(), cmt[kOnOn], "parameters template");
+  fprintf(strFl, "%-20s%s %s\n", AlignConfig::Instance().mpConFileName.c_str(), cmt[kOnOn], "constraints template");
   //
   fprintf(strFl, "\n\n%s %s\n", cmt[kOnOn], "MUST uncomment 1 solving methods and tune it");
   //
@@ -1558,20 +1286,22 @@ void Controller::genPedeSteerFile(const Option_t* opt) const
   for (int i = 0; i < nm; i++) {
     fprintf(strFl, "%s%s %-20s %2d %.2f %s\n", cmt[kOn], "method", kSolMeth[i], kDefNIter, kDefDelta, cmt[kOnOn]);
   }
+  fprintf(strFl, "\n%sskipemptycons\n", cmt[kOff]);
+  fprintf(strFl, "\n%sthreads 20 1\n", cmt[kOff]);
   //
-  const float kDefChi2F0 = 20., kDefChi2F = 3.; // chi2 factors for 1st and following iterations
-  const float kDefDWFrac = 0.2;                 // cut outliers with downweighting above this factor
+  const float kDefChi2F0 = 10., kDefChi2F = 3.; // chi2 factors for 1st and following iterations
+  const float kDefDWFrac = 0.1;                 // cut outliers with downweighting above this factor
   const int kDefOutlierDW = 4;                  // start Cauchy function downweighting from iteration
   const int kDefEntries = 25;                   // min entries per DOF to allow its variation
   //
   fprintf(strFl, "\n\n%s %s\n", cmt[kOnOn], "Optional settings");
-  fprintf(strFl, "\n%s%-20s %.2f %.2f %s %s\n", cmt[kOn], "chisqcut", kDefChi2F0, kDefChi2F,
+  fprintf(strFl, "\n%s%-20s %.2f %.2f %s %s\n", cmt[kOff], "chisqcut", kDefChi2F0, kDefChi2F,
           cmt[kOnOn], "chi2 cut factors for 1st and next iterations");
-  fprintf(strFl, "%s%-20s %2d %s %s\n", cmt[kOn], "outlierdownweighting", kDefOutlierDW,
+  fprintf(strFl, "%s%-20s %2d %s %s\n", cmt[kOff], "outlierdownweighting", kDefOutlierDW,
           cmt[kOnOn], "iteration for outliers downweighting with Cauchi factor");
-  fprintf(strFl, "%s%-20s %.3f %s %s\n", cmt[kOn], "dwfractioncut", kDefDWFrac,
+  fprintf(strFl, "%s%-20s %.3f %s %s\n", cmt[kOff], "dwfractioncut", kDefDWFrac,
           cmt[kOnOn], "cut outliers with downweighting above this factor");
-  fprintf(strFl, "%s%-20s %2d %s %s\n", cmt[kOn], "entries", kDefEntries,
+  fprintf(strFl, "%s%-20s %2d %s %s\n", cmt[kOff], "entries", kDefEntries,
           cmt[kOnOn], "min entries per DOF to allow its variation");
   //
   fprintf(strFl, "\n\n\n%s%-20s %s %s\n\n\n", cmt[kOff], "CFiles", cmt[kOnOn], "put below *.mille files list");
@@ -1597,16 +1327,15 @@ void Controller::genPedeSteerFile(const Option_t* opt) const
 }
 
 //___________________________________________________________
-bool Controller::readParameters(const char* parfile, bool useErrors)
+bool Controller::readParameters(const std::string& parfile, bool useErrors)
 {
   // read parameters file (millepede output)
-  if (mNDOFs < 1 || mGloParVal.size() || mGloParErr.size()) {
-    LOG(error) << "Something is wrong in init: mNDOFs=" << mNDOFs << " N GloParVal=" << mGloParVal.size() << " N GloParErr=" << mGloParErr.size();
+  if (mNDOFs < 1) {
+    LOG(error) << "Something is wrong in init, no DOFs found: mNDOFs=" << mNDOFs << " N GloParVal=" << mGloParVal.size() << " N GloParErr=" << mGloParErr.size();
   }
   std::ifstream inpf(parfile);
   if (!inpf.good()) {
-    printf("Failed on input filename %s\n", parfile);
-    return false;
+    LOGP(fatal, "Failed on input filename {}", parfile);
   }
   mGloParVal.resize(mNDOFs);
   if (useErrors) {
@@ -1614,34 +1343,36 @@ bool Controller::readParameters(const char* parfile, bool useErrors)
   }
   int cnt = 0;
   TString fline;
-  fline.ReadLine(inpf);
-  fline = fline.Strip(TString::kBoth, ' ');
-  fline.ToLower();
-  if (!fline.BeginsWith("parameter")) {
-    LOG(error) << "First line is not parameter keyword: " << fline.Data();
-    return false;
+  while (fline.ReadLine(inpf)) {
+    fline = fline.Strip(TString::kBoth, ' ');
+    fline.ToLower();
+    if (fline.Length() == 0 || fline.BeginsWith("!") || fline.BeginsWith("*")) {
+      continue;
+    }
+    if (!fline.BeginsWith("parameter")) {
+      LOGP(fatal, "First line of {} is not parameter keyword: {}", parfile, fline.Data());
+    }
+    break;
   }
   double v0, v1, v2;
   int lab, asg = 0, asg0 = 0;
   while (fline.ReadLine(inpf)) {
     cnt++;
     fline = fline.Strip(TString::kBoth, ' ');
-    if (fline.BeginsWith("!") || fline.BeginsWith("*")) {
+    if (fline.BeginsWith("!") || fline.BeginsWith("*") || fline.BeginsWith("parameter")) { // parameter keyword may repeat
       continue;
     } // ignore comment
     int nr = sscanf(fline.Data(), "%d%lf%lf%lf", &lab, &v0, &v1, &v2);
     if (nr < 3) {
       LOG(error) << "Expected to read at least 3 numbers, got " << nr << ", this is NOT milleped output";
-      LOG(error) << "line (" << cnt << ") was:\n " << fline.Data();
-      return false;
+      LOG(fatal) << "line (" << cnt << ") was: " << fline.Data();
     }
     if (nr == 3) {
       asg0++;
     }
     int parID = label2ParID(lab);
     if (parID < 0 || parID >= mNDOFs) {
-      LOG(error) << "Invalid label " << lab << " at line " << cnt << " -> ParID=" << parID;
-      return false;
+      LOG(fatal) << "Invalid label " << lab << " at line " << cnt << " -> ParID=" << parID;
     }
     mGloParVal[parID] = -v0;
     if (useErrors) {
@@ -1667,34 +1398,33 @@ void Controller::checkConstraints(const char* params)
   //
   int ncon = getNConstraints();
   for (int icon = 0; icon < ncon; icon++) {
-    const GeometricalConstraint* con = getConstraint(icon);
-    con->checkConstraint();
+    getConstraint(icon).checkConstraint();
   }
   //
 }
 
 //___________________________________________________________
-void Controller::mPRec2Mille(const char* mprecfile, const char* millefile, bool bindata)
+void Controller::MPRec2Mille(const std::string& mprecfile, const std::string& millefile, bool bindata)
 {
   // converts MPRecord tree to millepede binary format
-  TFile* flmpr = TFile::Open(mprecfile);
+  TFile* flmpr = TFile::Open(mprecfile.c_str());
   if (!flmpr) {
-    LOG(error) << "Failed to open MPRecord file " << mprecfile;
+    LOG(fatal) << "Failed to open MPRecord file " << mprecfile;
     return;
   }
   TTree* mprTree = (TTree*)flmpr->Get("mpTree");
   if (!mprTree) {
-    LOG(error) << "No mpTree in xMPRecord file " << mprecfile;
+    LOG(fatal) << "No mpTree in xMPRecord file " << mprecfile;
     return;
   }
-  mPRec2Mille(mprTree, millefile, bindata);
+  MPRec2Mille(mprTree, millefile, bindata);
   delete mprTree;
   flmpr->Close();
   delete flmpr;
 }
 
 //___________________________________________________________
-void Controller::mPRec2Mille(TTree* mprTree, const char* millefile, bool bindata)
+void Controller::MPRec2Mille(TTree* mprTree, const std::string& millefile, bool bindata)
 {
   // converts MPRecord tree to millepede binary format
   //
@@ -1703,74 +1433,47 @@ void Controller::mPRec2Mille(TTree* mprTree, const char* millefile, bool bindata
     LOG(error) << "provided tree does not contain branch mprec";
     return;
   }
-  Millepede2Record* rec = new Millepede2Record();
+  Millepede2Record mrec, *rec = &mrec;
   br->SetAddress(&rec);
   int nent = mprTree->GetEntries();
-  TString mlname = millefile;
-  if (mlname.IsNull()) {
+  std::string mlname = millefile;
+  if (mlname.empty()) {
     mlname = "mpRec2mpData";
   }
-  if (!mlname.EndsWith(sMPDataExt)) {
+  if (!o2::utils::Str::endsWith(mlname, sMPDataExt)) {
     mlname += sMPDataExt;
   }
-  Mille* mille = new Mille(mlname, bindata);
-  TArrayF buffDLoc;
+  Mille mille(mlname, bindata);
+  std::vector buffLoc;
   for (int i = 0; i < nent; i++) {
     br->GetEntry(i);
     int nr = rec->getNResid(); // number of residual records
     int nloc = rec->getNVarLoc();
-    if (buffDLoc.GetSize() < nloc) {
-      buffDLoc.Set(nloc + 100);
-    }
-    float* buffLocV = buffDLoc.GetArray();
-    const float* recDGlo = rec->getArrGlo();
-    const float* recDLoc = rec->getArrLoc();
-    const short* recLabLoc = rec->getArrLabLoc();
-    const int* recLabGlo = rec->getArrLabGlo();
+    auto recDGlo = rec->getArrGlo();
+    auto recDLoc = rec->getArrLoc();
+    auto recLabLoc = rec->getArrLabLoc();
+    auto recLabGlo = rec->getArrLabGlo();
     //
     for (int ir = 0; ir < nr; ir++) {
-      memset(buffLocV, 0, nloc * sizeof(float));
+      buffLoc.clear();
+      buffLoc.resize(nloc);
       int ndglo = rec->getNDGlo(ir);
       int ndloc = rec->getNDLoc(ir);
       // fill 0-suppressed array from MPRecord to non-0-suppressed array of Mille
       for (int l = ndloc; l--;) {
-        buffLocV[recLabLoc[l]] = recDLoc[l];
+        buffLoc[recLabLoc[l]] = recDLoc[l];
       }
       //
-      mille->mille(nloc, buffLocV, ndglo, recDGlo, recLabGlo, rec->getResid(ir), rec->getResErr(ir));
+      mille.mille(nloc, buffLoc.data(), ndglo, recDGlo, recLabGlo, rec->getResid(ir), rec->getResErr(ir));
       //
       recLabGlo += ndglo; // next record
       recDGlo += ndglo;
       recLabLoc += ndloc;
       recDLoc += ndloc;
     }
-    mille->end();
+    mille.finalise();
   }
-  delete mille;
   br->SetAddress(nullptr);
-  delete rec;
-}
-
-//____________________________________________________________
-void Controller::fillStatHisto(int type, float w)
-{
-  if (!mHistoStat) {
-    createStatHisto();
-  }
-  mHistoStat->Fill((isCosmic() ? kNHVars : 0) + type, w);
-}
-
-//____________________________________________________________
-void Controller::createStatHisto()
-{
-  mHistoStat = new TH1F("stat", "stat", 2 * kNHVars, -0.5, 2 * kNHVars - 0.5);
-  mHistoStat->SetDirectory(nullptr);
-  TAxis* xax = mHistoStat->GetXaxis();
-  for (int j = 0; j < 2; j++) {
-    for (int i = 0; i < kNHVars; i++) {
-      xax->SetBinLabel(j * kNHVars + i + 1, Form("%s.%s", j ? "CSM" : "COL", sHStatName[i]));
-    }
-  }
 }
 
 //____________________________________________________________
@@ -1786,11 +1489,11 @@ void Controller::printLabels() const
 int Controller::label2ParID(int lab) const
 {
   // convert Mille label to ParID (slow)
-  int ind = 0; // FIXME RS TODO // findKeyIndex(lab, mOrderedLbl, mNDOFs);
-  if (ind < 0) {
-    return -1;
+  auto it = mLbl2ID.find(lab);
+  if (it == mLbl2ID.end()) {
+    LOGP(fatal, "Label {} is not mapped to any parameter", lab);
   }
-  return mLbl2ID[ind];
+  return it->second - 1;
 }
 
 //____________________________________________________________
@@ -1811,75 +1514,16 @@ void Controller::addAutoConstraints()
 void Controller::writePedeConstraints() const
 {
   // write constraints file
-  FILE* conFl = fopen(mMPConFileName.c_str(), "w+");
+  FILE* conFl = fopen(AlignConfig::Instance().mpConFileName.c_str(), "w+");
   //
   int nconstr = getNConstraints();
   for (int icon = 0; icon < nconstr; icon++) {
-    getConstraint(icon)->writeChildrenConstraints(conFl);
+    getConstraint(icon).writeChildrenConstraints(conFl);
   }
   //
   fclose(conFl);
 }
 
-//____________________________________________________________
-void Controller::fixLowStatFromDOFStat(int thresh)
-{
-  // fix DOFs having stat below threshold
-  //
-  if (mNDOFs != mDOFStat.getNDOFs()) {
-    LOG(error) << "Discrepancy between NDOFs=" << mNDOFs << " of and statistics object: " << mDOFStat.getNDOFs();
-    return;
-  }
-  for (int parID = 0; parID < mNDOFs; parID++) {
-    if (mDOFStat.getStat(parID) >= thresh) {
-      continue;
-    }
-    mGloParErr[parID] = -999.;
-  }
-  //
-}
-
-//____________________________________________________________
-void Controller::loadStat(const char* flname)
-{
-  // load statistics histos from external file produced by alignment task
-  TFile* fl = TFile::Open(flname);
-  //
-  TObject *hdfO = nullptr, *hstO = nullptr;
-  TList* lst = (TList*)fl->Get("clist");
-  if (lst) {
-    hdfO = lst->FindObject("DOFstat");
-    if (hdfO) {
-      lst->Remove(hdfO);
-    }
-    hstO = lst->FindObject("stat");
-    if (hstO) {
-      lst->Remove(hstO);
-    }
-    delete lst;
-  } else {
-    hdfO = fl->Get("DOFstat");
-    hstO = fl->Get("stat");
-  }
-  TH1F* hst = nullptr;
-  if (hstO && (hst = dynamic_cast(hstO))) {
-    hst->SetDirectory(nullptr);
-  } else {
-    LOG(warning) << "did not find stat histo";
-  }
-  //
-  DOFStatistics* dofSt = nullptr;
-  if (!hdfO || !(dofSt = dynamic_cast(hdfO))) {
-    LOG(warning) << "did not find DOFstat object";
-  }
-  //
-  setHistoStat(hst);
-  setDOFStat(*dofSt); // FIXME RS TODO
-  //
-  fl->Close();
-  delete fl;
-}
-
 //______________________________________________
 void Controller::checkSol(TTree* mpRecTree, bool store,
                           bool verbose, bool loc, const char* outName)
@@ -1934,6 +1578,8 @@ void Controller::checkSol(TTree* mpRecTree, bool store,
     outFile->Close();
     delete outFile;
   }
+  delete rLG;
+  delete rL;
   //
 }
 
@@ -2226,10 +1872,20 @@ void Controller::expandGlobalsBy(int n)
   mGloParVal.resize(snew);
   mGloParErr.resize(snew);
   mGloParLab.resize(snew);
-  mOrderedLbl.resize(snew);
-  mLbl2ID.resize(snew);
   mNDOFs += n;
 }
 
+//______________________________________________
+void Controller::setTPCVDrift(const o2::tpc::VDriftCorrFact& v)
+{
+  mTPCDrift = v;
+}
+
+//______________________________________________
+void Controller::setTPCCorrMaps(o2::gpu::CorrectionMapsHelper* maph)
+{
+  mTPCCorrMapsHelper = maph;
+}
+
 } // namespace align
 } // namespace o2
diff --git a/Detectors/Align/src/DOFStatistics.cxx b/Detectors/Align/src/DOFStatistics.cxx
deleted file mode 100644
index 8df1c3f6b2a69..0000000000000
--- a/Detectors/Align/src/DOFStatistics.cxx
+++ /dev/null
@@ -1,65 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-/// @file   DOFStatistics.h
-/// @author ruben.shahoyan@cern.ch, michael.lettrich@cern.ch
-/// @since  2021-02-01
-/// @brief  Mergable bbject for statistics of points used by each DOF
-
-#include "Align/DOFStatistics.h"
-#include "Align/Controller.h"
-#include "Framework/Logger.h"
-#include 
-
-ClassImp(o2::align::DOFStatistics);
-
-namespace o2
-{
-namespace align
-{
-
-//____________________________________________
-std::unique_ptr DOFStatistics::buildHistogram(Controller* controller) const
-{
-  // create histo with stat. If steer object is supplied, build labels
-  auto histogram = std::make_unique("DOFstat", "statistics per DOF", getNDOFs(), 0, getNDOFs());
-  for (size_t i = 0; i < getNDOFs(); ++i) {
-    // Bin 0 is underflow bin
-    histogram->SetBinContent(i + 1, mStat[i]);
-    if (controller != nullptr) {
-      histogram->GetXaxis()->SetBinLabel(i + 1, controller->getDOFLabelTxt(i));
-    }
-  }
-  return histogram;
-}
-
-//______________________________________________________________________________
-int64_t DOFStatistics::merge(TCollection* list)
-{
-  // merge statistics
-  int nMerged = 0;
-  TIter next{list};
-  TObject* obj = nullptr;
-  while ((obj = next()) != nullptr) {
-    DOFStatistics* otherStats = dynamic_cast(obj);
-    if (!otherStats) {
-      continue;
-    }
-    assert(getNDOFs() == otherStats->getNDOFs());
-    std::transform(std::begin(otherStats->mStat), std::end(otherStats->mStat), std::begin(mStat), std::begin(mStat), std::plus<>{});
-    mNMerges += otherStats->mNMerges;
-    nMerged++;
-  }
-  return nMerged;
-}
-
-} // namespace align
-} // namespace o2
diff --git a/Detectors/Align/src/GeometricalConstraint.cxx b/Detectors/Align/src/GeometricalConstraint.cxx
index 2709af8afd09e..2bbf8bafc8d32 100644
--- a/Detectors/Align/src/GeometricalConstraint.cxx
+++ b/Detectors/Align/src/GeometricalConstraint.cxx
@@ -15,6 +15,7 @@
 /// @brief  Descriptor of geometrical constraint
 
 #include "Align/GeometricalConstraint.h"
+#include "Align/AlignConfig.h"
 #include "DetectorsCommonDataFormats/AlignParam.h"
 #include "Align/utils.h"
 #include "Framework/Logger.h"
@@ -32,37 +33,6 @@ namespace o2
 namespace align
 {
 
-//___________________________________________________________________
-GeometricalConstraint::GeometricalConstraint(const char* name, const char* title)
-  : TNamed(name, title), mConstraint(0), mParent(nullptr), mChildren(2)
-{
-  // def. c-tor
-  for (int i = kNDOFGeom; i--;) {
-    mSigma[i] = 0;
-  }
-}
-
-//___________________________________________________________________
-GeometricalConstraint::~GeometricalConstraint()
-{
-  // d-tor
-  delete mParent;
-}
-
-//___________________________________________________________________
-void GeometricalConstraint::setParent(const AlignableVolume* par)
-{
-  mParent = par;
-  TString nm = GetName();
-  if (nm.IsNull()) {
-    if (par) {
-      SetNameTitle(par->getSymName(), "Automatic");
-    } else {
-      SetNameTitle("GLOBAL", "Automatic");
-    }
-  }
-}
-
 //______________________________________________________
 void GeometricalConstraint::writeChildrenConstraints(FILE* conOut) const
 {
@@ -78,13 +48,14 @@ void GeometricalConstraint::writeChildrenConstraints(FILE* conOut) const
   //
   bool doJac = !getNoJacobian(); // do we need jacobian evaluation?
   int nch = getNChildren();
-  float* cstrArr = new float[nch * kNDOFGeom * kNDOFGeom];
-  memset(cstrArr, 0, nch * kNDOFGeom * kNDOFGeom * sizeof(float));
+  std::unique_ptr cstrArr(new double[nch * kNDOFGeom * kNDOFGeom]);
+  memset(cstrArr.get(), 0, nch * kNDOFGeom * kNDOFGeom * sizeof(double));
   // we need for each children the matrix for vector transformation from children frame
   // (in which its DOFs are defined, LOC or TRA) to this parent variation frame
   // matRel = mPar^-1*mChild
   TGeoHMatrix mPar;
   //
+  const auto& algConf = AlignConfig::Instance();
   // in case of parent assigned use its matrix,
   // otherwise Alice global frame is assumed to be the parent -> Unit matrix
   if (mParent && doJac) {
@@ -97,11 +68,10 @@ void GeometricalConstraint::writeChildrenConstraints(FILE* conOut) const
     mPar = mPar.Inverse();
   }
   //
-  float* jac = cstrArr;
-  int nContCh[kNDOFGeom] = {0}; // we need at least on contributing children DOF to constrain the parent DOF
+  auto jac = cstrArr.get();
+  int nContCh[kNDOFGeom] = {0}; // we need at least one contributing children DOF to constrain the parent DOF
   for (int ich = 0; ich < nch; ich++) {
-    AlignableVolume* child = getChild(ich);
-    //
+    auto child = getChild(ich);
     if (doJac) { // calculate jacobian
       TGeoHMatrix matRel;
       if (child->isFrameTRA()) {
@@ -115,14 +85,13 @@ void GeometricalConstraint::writeChildrenConstraints(FILE* conOut) const
       //
       for (int ics = 0; ics < kNDOFGeom; ics++) { // DOF of parent to be constrained
         for (int ip = 0; ip < kNDOFGeom; ip++) {  // count contributing DOFs
-          float jv = jac[ics * kNDOFGeom + ip];
+          double jv = jac[ics * kNDOFGeom + ip];
           if (!isZeroAbs(jv) && child->isFreeDOF(ip) && child->getParErr(ip) >= 0) {
             nContCh[ip]++;
           }
         }
       }
     } else { // simple constraint on the sum of requested DOF
-      //
       for (int ip = 0; ip < kNDOFGeom; ip++) {
         if (child->isFreeDOF(ip) && child->getParErr(ip) >= 0) {
           nContCh[ip]++;
@@ -132,32 +101,32 @@ void GeometricalConstraint::writeChildrenConstraints(FILE* conOut) const
     }
     jac += kNDOFGeom * kNDOFGeom; // matrix for next slot
   }
-  //
   for (int ics = 0; ics < kNDOFGeom; ics++) {
     if (!isDOFConstrained(ics)) {
       continue;
     }
     int cmtStatus = nContCh[ics] > 0 ? kOff : kOn; // do we comment this constraint?
-    //
     if (cmtStatus) {
-      LOG(info) << "No contributors to constraint of " << getDOFName(ics) << " of " << GetName();
+      if (algConf.verbose > 0) {
+        LOG(info) << "No contributors to constraint of " << getDOFName(ics) << " of " << getName();
+      }
     }
     if (mSigma[ics] > 0) {
-      fprintf(conOut, "\n%s%s\t%e\t%e\t%s %s of %s %s\n", comment[cmtStatus], kKeyConstr[kMeas], 0.0, mSigma[ics],
-              comment[kOnOn], getDOFName(ics), GetName(), GetTitle());
+      fprintf(conOut, "\n%s%s\t%e\t%e\t%s %s of %s Auto\n", comment[cmtStatus], kKeyConstr[kMeas], 0.0, mSigma[ics],
+              comment[kOnOn], getDOFName(ics), getName().c_str());
     } else {
-      fprintf(conOut, "\n%s%s\t%e\t%s %s of %s %s\n", comment[cmtStatus], kKeyConstr[kConstr], 0.0,
-              comment[kOnOn], getDOFName(ics), GetName(), GetTitle());
+      fprintf(conOut, "\n%s%s\t%e\t%s %s of %s Auto\n", comment[cmtStatus], kKeyConstr[kConstr], 0.0,
+              comment[kOnOn], getDOFName(ics), getName().c_str());
     }
     for (int ich = 0; ich < nch; ich++) { // contribution from this children DOFs to constraint
-      AlignableVolume* child = getChild(ich);
-      jac = cstrArr + kNDOFGeom * kNDOFGeom * ich;
+      auto child = getChild(ich);
+      jac = cstrArr.get() + kNDOFGeom * kNDOFGeom * ich;
       if (cmtStatus) {
         fprintf(conOut, "%s", comment[cmtStatus]);
       } // comment out contribution
       // first write real constraints
       for (int ip = 0; ip < kNDOFGeom; ip++) {
-        float jv = jac[ics * kNDOFGeom + ip];
+        double jv = jac[ics * kNDOFGeom + ip];
         if (child->isFreeDOF(ip) && !isZeroAbs(jv) && child->getParErr(ip) >= 0) {
           fprintf(conOut, "%9d %+.3e\t", child->getParLab(ip), jv);
         }
@@ -166,18 +135,16 @@ void GeometricalConstraint::writeChildrenConstraints(FILE* conOut) const
       fprintf(conOut, "%s ", comment[kOn]);
       if (doJac) {
         for (int ip = 0; ip < kNDOFGeom; ip++) {
-          float jv = jac[ics * kNDOFGeom + ip];
+          double jv = jac[ics * kNDOFGeom + ip];
           if (child->isFreeDOF(ip) && !isZeroAbs(jv) && child->getParErr(ip) >= 0) {
             continue;
           }
           fprintf(conOut, "%9d %+.3e\t", child->getParLab(ip), jv);
         } // loop over DOF's of children contributing to this constraint
       }
-      fprintf(conOut, "%s from %s\n", comment[kOnOn], child->GetName());
+      fprintf(conOut, "%s from %s\n", comment[kOnOn], child->getSymName());
     } // loop over children
   }   // loop over constraints in parent volume
-  //
-  delete[] cstrArr;
 }
 
 //______________________________________________________
@@ -190,8 +157,8 @@ void GeometricalConstraint::checkConstraint() const
   }
   //
   bool doJac = !getNoJacobian(); // do we need jacobian evaluation?
-  float* cstrArr = new float[nch * kNDOFGeom * kNDOFGeom];
-  memset(cstrArr, 0, nch * kNDOFGeom * kNDOFGeom * sizeof(float));
+  std::unique_ptr cstrArr(new double[nch * kNDOFGeom * kNDOFGeom]);
+  memset(cstrArr.get(), 0, nch * kNDOFGeom * kNDOFGeom * sizeof(double));
   // we need for each children the matrix for vector transformation from children frame
   // (in which its DOFs are defined, LOC or TRA) to this parent variation frame
   // matRel = mPar^-1*mChild
@@ -208,11 +175,11 @@ void GeometricalConstraint::checkConstraint() const
     mPar = mPar.Inverse();
   }
   //
-  float* jac = cstrArr;
+  auto jac = cstrArr.get();
   double parsTotEx[kNDOFGeom] = {0}; // explicitly calculated total modification
   double parsTotAn[kNDOFGeom] = {0}; // analyticaly calculated total modification
   //
-  printf("\n\n ----- Constraints Validation for %s %s ------\n", GetName(), GetTitle());
+  printf("\n\n ----- Constraints Validation for %s Auto ------\n", getName().c_str());
   printf(" chld| ");
   for (int jp = 0; jp < kNDOFGeom; jp++) {
     printf("  %3s:%3s An/Ex  |", getDOFName(jp), isDOFConstrained(jp) ? "ON " : "OFF");
@@ -221,9 +188,9 @@ void GeometricalConstraint::checkConstraint() const
   for (int jp = 0; jp < kNDOFGeom; jp++) {
     printf("  D%3s   ", getDOFName(jp));
   }
-  printf(" ! %s\n", GetName());
+  printf(" ! %s\n", getName().c_str());
   for (int ich = 0; ich < nch; ich++) {
-    AlignableVolume* child = getChild(ich);
+    auto child = getChild(ich);
     double parsC[kNDOFGeom] = {0}, parsPAn[kNDOFGeom] = {0}, parsPEx[kNDOFGeom] = {0};
     for (int jc = kNDOFGeom; jc--;) {
       parsC[jc] = child->getParVal(jc);
@@ -240,7 +207,7 @@ void GeometricalConstraint::checkConstraint() const
       } // local to global
       //
       matRel.MultiplyLeft(&mPar);
-      constrCoefGeom(matRel, jac); // Jacobian for analytical constraint used by MillePeded
+      constrCoefGeom(matRel, jac); // Jacobian for analytical constraint used by MillePede
                                    //
       TGeoHMatrix tau;
       child->delta2Matrix(tau, parsC); // child correction matrix in the child frame
@@ -315,7 +282,7 @@ void GeometricalConstraint::checkConstraint() const
   else {
     printf(" no parent -> %s ", doJac ? "Global" : "Simple");
   }
-  printf(" ! <----- %s\n", GetName());
+  printf(" ! <----- %s\n", getName().c_str());
   //
   printf(" Sig | ");
   for (int jp = 0; jp < kNDOFGeom; jp++) {
@@ -326,14 +293,10 @@ void GeometricalConstraint::checkConstraint() const
     }
   }
   printf(" ! <----- \n");
-
-  //
-  delete[] cstrArr;
-  //
 }
 
 //_________________________________________________________________
-void GeometricalConstraint::constrCoefGeom(const TGeoHMatrix& matRD, float* jac /*[kNDOFGeom][kNDOFGeom]*/) const
+void GeometricalConstraint::constrCoefGeom(const TGeoHMatrix& matRD, double* jac /*[kNDOFGeom][kNDOFGeom]*/) const
 {
   // If the transformation R brings the vector from "local" frame to "master" frame as V=R*v
   // then application of the small LOCAL correction tau to vector v is equivalent to
@@ -355,8 +318,14 @@ void GeometricalConstraint::constrCoefGeom(const TGeoHMatrix& matRD, float* jac
   const double *rd = matRD.GetRotationMatrix(), *ri = matRI.GetRotationMatrix();
   const double /**td=matRD.GetTranslation(),*/* ti = matRI.GetTranslation();
   //
-  // the angles are in degrees, while we use sinX->X approximation...
-  const double cf[kNDOFGeom] = {1, 1, 1, DegToRad(), DegToRad(), DegToRad()};
+  //// the angles are in radians, while we use sinX->X approximation...
+  // const double cf[kNDOFGeom] = {1., 1., 1., DegToRad(), DegToRad(), DegToRad()};
+  // const double sgc[kNDOFGeom] = {1., 1., 1., -RadToDeg(), RadToDeg(), -RadToDeg()};
+
+  // the angles are in radians
+  const double cf[kNDOFGeom] = {1., 1., 1., 1., 1., 1.};
+  const double sgc[kNDOFGeom] = {1., 1., 1., -1., 1., -1.};
+
   //
   // since the TAU is supposed to convert local corrections in the child frame to corrections
   // in the parent frame, we scale angular degrees of freedom back to degrees and assign the
@@ -367,7 +336,6 @@ void GeometricalConstraint::constrCoefGeom(const TGeoHMatrix& matRD, float* jac
   //  -cospsi*sinthe*cosphi + sinpsi*sinphi;  cospsi*sinthe*sinphi + sinpsi*cosphi;  costhe*cospsi;
   //
   const double kJTol = 1e-4; // treat derivatives below this threshold as 0
-  const double sgc[kNDOFGeom] = {1., 1., 1., -RadToDeg(), RadToDeg(), -RadToDeg()};
   //
   double dDPar[kNDOFGeom][4][4] = {
     // dDX[4][4]
@@ -403,7 +371,7 @@ void GeometricalConstraint::constrCoefGeom(const TGeoHMatrix& matRD, float* jac
 }
 
 //______________________________________________________
-void GeometricalConstraint::Print(const Option_t*) const
+void GeometricalConstraint::print() const
 {
   // print info
   printf("Constraint on ");
@@ -412,13 +380,13 @@ void GeometricalConstraint::Print(const Option_t*) const
       printf("%3s (Sig:%+e) ", getDOFName(i), getSigma(i));
     }
   }
-  printf(" | %s %s\n", GetName(), GetTitle());
+  printf(" | %s Auto\n", getName().c_str());
   if (getNoJacobian()) {
     printf("!!! This is explicit constraint on sum of DOFs (no Jacobian)!!!\n");
   }
   for (int i = 0; i < getNChildren(); i++) {
-    const AlignableVolume* child = getChild(i);
-    printf("%3d %s\n", i, child->GetName());
+    auto child = getChild(i);
+    printf("%3d %s\n", i, child->getSymName());
   }
 }
 
diff --git a/Detectors/Align/src/Mille.cxx b/Detectors/Align/src/Mille.cxx
index 03fb6bed9858c..90353a73e23f9 100644
--- a/Detectors/Align/src/Mille.cxx
+++ b/Detectors/Align/src/Mille.cxx
@@ -16,12 +16,10 @@
 
 /*
   RS: original Mille.cc from http://svnsrv.desy.de/public/MillepedeII/tags/V04-02-03
-  jeudi 30 avril 2015: renamed to cxx
-  jeudi 30 avril 2015: added automatic buffer expansion
 */
 
 #include "Align/Mille.h"
-
+#include "Framework/Logger.h"
 #include 
 #include 
 
@@ -30,37 +28,17 @@ namespace o2
 namespace align
 {
 //___________________________________________________________________________
-
-/// Opens outFileName (by default as binary file).
-/**
- * \param[in] outFileName  file name
- * \param[in] asBinary     flag for binary
- * \param[in] writeZero    flag for keeping of zeros
- */
-Mille::Mille(const char* outFileName, bool asBinary, bool writeZero) : myOutFile(outFileName,
-                                                                                 (asBinary ? (std::ios::binary | std::ios::out | std::ios::trunc) : (std::ios::out | std::ios::trunc))),
-                                                                       myAsBinary(asBinary),
-                                                                       myWriteZero(writeZero),
-                                                                       myBufferSize(0),
-                                                                       myBufferInt(5000),
-                                                                       myBufferFloat(5000),
-                                                                       myBufferPos(-1),
-                                                                       myHasSpecial(false)
+Mille::Mille(const std::string& outFileName, bool asBinary, bool writeZero)
+  : mOutFile(outFileName, (asBinary ? (std::ios::binary | std::ios::out | std::ios::trunc) : (std::ios::out | std::ios::trunc))),
+    mAsBinary(asBinary),
+    mWriteZero(writeZero)
 {
-  // Instead myBufferPos(-1), myHasSpecial(false) and the following two lines
-  // we could call newSet() and kill()...
-
-  if (!myOutFile.is_open()) {
-    std::cerr << "Mille::Mille: Could not open " << outFileName
-              << " as output file." << std::endl;
+  if (!mOutFile.is_open()) {
+    LOG(fatal) << "Failed to open Mille file " << outFileName;
   }
-}
-
-//___________________________________________________________________________
-/// Closes file.
-Mille::~Mille()
-{
-  myOutFile.close();
+  mBufferInt.reserve(1024);
+  mBufferFloat.reserve(1024);
+  clear();
 }
 
 //___________________________________________________________________________
@@ -81,44 +59,30 @@ void Mille::mille(int NLC, const float* derLc,
   if (sigma <= 0.) {
     return;
   }
-  if (myBufferPos == -1) {
-    this->newSet();
-  } // start, e.g. new track
-  if (!this->checkBufferSize(NLC, NGL)) {
-    return;
-  }
-
   // first store measurement
-  ++myBufferPos;
-  float* bufferFloat = myBufferFloat.GetArray();
-  int* bufferInt = myBufferInt.GetArray();
-  bufferFloat[myBufferPos] = rMeas;
-  bufferInt[myBufferPos] = 0;
+  mBufferFloat.push_back(rMeas);
+  mBufferInt.push_back(0);
 
   // store local derivatives and local 'lables' 1,...,NLC
   for (int i = 0; i < NLC; ++i) {
-    if (derLc[i] || myWriteZero) { // by default store only non-zero derivatives
-      ++myBufferPos;
-      bufferFloat[myBufferPos] = derLc[i]; // local derivatives
-      bufferInt[myBufferPos] = i + 1;      // index of local parameter
+    if (derLc[i] || mWriteZero) {       // by default store only non-zero derivatives
+      mBufferFloat.push_back(derLc[i]); // local derivatives
+      mBufferInt.push_back(i + 1);      // index of local parameter
     }
   }
 
   // store uncertainty of measurement in between locals and globals
-  ++myBufferPos;
-  bufferFloat[myBufferPos] = sigma;
-  bufferInt[myBufferPos] = 0;
+  mBufferFloat.push_back(sigma);
+  mBufferInt.push_back(0);
 
   // store global derivatives and their labels
   for (int i = 0; i < NGL; ++i) {
-    if (derGl[i] || myWriteZero) {                                   // by default store only non-zero derivatives
-      if ((label[i] > 0 || myWriteZero) && label[i] <= myMaxLabel) { // and for valid labels
-        ++myBufferPos;
-        bufferFloat[myBufferPos] = derGl[i]; // global derivatives
-        bufferInt[myBufferPos] = label[i];   // index of global parameter
+    if (derGl[i] || mWriteZero) {                                 // by default store only non-zero derivatives
+      if ((label[i] > 0 || mWriteZero) && label[i] <= MaxLabel) { // and for valid labels
+        mBufferFloat.push_back(derGl[i]);                         // global derivatives
+        mBufferInt.push_back(label[i]);                           // index of global parameter
       } else {
-        std::cerr << "Mille::mille: Invalid label " << label[i]
-                  << " <= 0 or > " << myMaxLabel << std::endl;
+        LOGP(error, "Mille: invalid label {} <=0 or > {}", label[i], MaxLabel);
       }
     }
   }
@@ -128,125 +92,77 @@ void Mille::mille(int NLC, const float* derLc,
 /// Add special data to buffer.
 /**
  * \param[in]    nSpecial   number of floats/ints
- * \param[in]    floatings  floats
- * \param[in]    integers   ints
+ * \param[in]    floats  floats
+ * \param[in]    ints   ints
  */
-void Mille::special(int nSpecial, const float* floatings, const int* integers)
+void Mille::special(int nSpecial, const float* floats, const int* ints)
 {
   if (nSpecial == 0) {
     return;
   }
-  if (myBufferPos == -1) {
-    this->newSet();
-  } // start, e.g. new track
-  if (myHasSpecial) {
-    std::cerr << "Mille::special: Special values already stored for this record."
-              << std::endl;
+  if (mHasSpecial) {
+    LOG(error) << "Mille: special values already stored for this record.";
     return;
   }
-  if (!this->checkBufferSize(nSpecial, 0)) {
-    return;
-  }
-  myHasSpecial = true; // after newSet() (Note: MILLSP sets to buffer position...)
-
-  //  myBufferFloat[.]  | myBufferInt[.]
+  mHasSpecial = true;
+  //  mBufferFloat[.]  | mBufferInt[.]
   // ------------------------------------
   //      0.0           |      0
   //  -float(nSpecial)  |      0
   //  The above indicates special data, following are nSpecial floating and nSpecial integer data.
   //
-  float* bufferFloat = myBufferFloat.GetArray();
-  int* bufferInt = myBufferInt.GetArray();
-  //
-  ++myBufferPos; // zero pair
-  bufferFloat[myBufferPos] = 0.;
-  bufferInt[myBufferPos] = 0;
-
-  ++myBufferPos;                        // nSpecial and zero
-  bufferFloat[myBufferPos] = -nSpecial; // automatic conversion to float
-  bufferInt[myBufferPos] = 0;
+  // zero pair
+  mBufferFloat.push_back(0.);
+  mBufferInt.push_back(0);
+  // nSpecial and zero
+  mBufferFloat.push_back(-nSpecial); // automatic conversion to float
+  mBufferInt.push_back(0);
 
   for (int i = 0; i < nSpecial; ++i) {
-    ++myBufferPos;
-    bufferFloat[myBufferPos] = floatings[i];
-    bufferInt[myBufferPos] = integers[i];
+    mBufferFloat.push_back(floats[i]);
+    mBufferInt.push_back(ints[i]);
   }
 }
 
-//___________________________________________________________________________
-/// Reset buffers, i.e. kill derivatives accumulated for current set.
-void Mille::kill()
-{
-  myBufferPos = -1;
-}
-
 //___________________________________________________________________________
 /// Write buffer (set of derivatives with same local parameters) to file.
-int Mille::end()
+int Mille::finalise()
 {
   int wrote = 0;
-  if (myBufferPos > 0) { // only if anything stored...
-    const int numWordsToWrite = (myBufferPos + 1) * 2;
-    float* bufferFloat = myBufferFloat.GetArray();
-    int* bufferInt = myBufferInt.GetArray();
-
-    if (myAsBinary) {
-      myOutFile.write(reinterpret_cast(&numWordsToWrite),
-                      sizeof(numWordsToWrite));
-      myOutFile.write(reinterpret_cast(bufferFloat),
-                      (myBufferPos + 1) * sizeof(bufferFloat[0]));
-      myOutFile.write(reinterpret_cast(bufferInt),
-                      (myBufferPos + 1) * sizeof(bufferInt[0]));
+  int nw = mBufferInt.size();
+  if (nw) { // only if anything stored...
+    const int numWordsToWrite = nw * 2;
+
+    if (mAsBinary) {
+      mOutFile.write(reinterpret_cast(&numWordsToWrite), sizeof(int));
+      mOutFile.write(reinterpret_cast(mBufferFloat.data()), nw * sizeof(mBufferFloat[0]));
+      mOutFile.write(reinterpret_cast(mBufferInt.data()), nw * sizeof(mBufferInt[0]));
     } else {
-      myOutFile << numWordsToWrite << "\n";
-      for (int i = 0; i < myBufferPos + 1; ++i) {
-        myOutFile << bufferFloat[i] << " ";
+      mOutFile << numWordsToWrite << "\n";
+      for (int i = 0; i < nw; i++) {
+        mOutFile << mBufferFloat[i] << " ";
       }
-      myOutFile << "\n";
-
-      for (int i = 0; i < myBufferPos + 1; ++i) {
-        myOutFile << bufferInt[i] << " ";
+      mOutFile << "\n";
+      for (int i = 0; i < nw; i++) {
+        mOutFile << mBufferInt[i] << " ";
       }
-      myOutFile << "\n";
+      mOutFile << "\n";
     }
-    wrote = (myBufferPos + 1) * (sizeof(bufferFloat[0]) + sizeof(bufferInt[0])) + sizeof(int);
+    wrote = nw * (sizeof(mBufferFloat[0]) + sizeof(mBufferInt[0])) + sizeof(int);
   }
-  myBufferPos = -1; // reset buffer for next set of derivatives
+  clear();
   return wrote;
 }
 
 //___________________________________________________________________________
 /// Initialize for new set of locals, e.g. new track.
-void Mille::newSet()
+void Mille::clear()
 {
-  myBufferPos = 0;
-  myHasSpecial = false;
-  myBufferFloat[0] = 0.0;
-  myBufferInt[0] = 0; // position 0 used as error counter
-}
-
-//___________________________________________________________________________
-/// Enough space for next nLocal + nGlobal derivatives incl. measurement?
-/**
- * \param[in]   nLocal  number of local derivatives
- * \param[in]   nGlobal number of global derivatives
- * \return      true if sufficient space available (else false)
- */
-bool Mille::checkBufferSize(int nLocal, int nGlobal)
-{
-  if (myBufferPos + nLocal + nGlobal + 2 >= myBufferInt.GetSize()) {
-    ++(myBufferInt[0]); // increase error count
-    std::cerr << "Mille::checkBufferSize: Buffer too short ("
-              << myBufferInt.GetSize() << "),"
-              << "\n need space for nLocal (" << nLocal << ")"
-              << "/nGlobal (" << nGlobal << ") local/global derivatives, "
-              << myBufferPos + 1 << " already stored!"
-              << std::endl;
-    //    return false;
-    myBufferInt.Set(myBufferPos + nLocal + nGlobal + 1000);
-    myBufferFloat.Set(myBufferPos + nLocal + nGlobal + 1000);
-  }
-  return true;
+  mHasSpecial = false;
+  mBufferInt.clear();
+  mBufferFloat.clear();
+  mBufferFloat.push_back(0.f);
+  mBufferInt.push_back(0); // position 0 used as error counter
 }
 
 } // namespace align
diff --git a/Detectors/Align/src/Millepede2Record.cxx b/Detectors/Align/src/Millepede2Record.cxx
index 6447970d2c3a8..a9668105f5dfe 100644
--- a/Detectors/Align/src/Millepede2Record.cxx
+++ b/Detectors/Align/src/Millepede2Record.cxx
@@ -29,18 +29,6 @@ namespace o2
 namespace align
 {
 
-//_________________________________________________________
-Millepede2Record::Millepede2Record()
-  : mTrackID(0), mTimeStamp(0), mNResid(0), mNVarLoc(0), mNVarGlo(0), mNDLocTot(0), mNDGloTot(0), mNMeas(0), mChi2Ini(0), mQ2Pt(0), mTgl(0), mNDLoc(nullptr), mNDGlo(nullptr), mVolID(nullptr), mResid(nullptr), mResErr(nullptr), mIDLoc(nullptr), mIDGlo(nullptr), mDLoc(nullptr), mDGlo(nullptr)
-    //
-    ,
-    mNResidBook(0),
-    mNDLocTotBook(0),
-    mNDGloTotBook(0)
-{
-  // def c-tor
-}
-
 //_________________________________________________________
 Millepede2Record::~Millepede2Record()
 {
@@ -49,6 +37,7 @@ Millepede2Record::~Millepede2Record()
   delete[] mNDGlo;
   delete[] mVolID;
   delete[] mResid;
+  delete[] mMeasType;
   delete[] mResErr;
   delete[] mIDLoc;
   delete[] mIDGlo;
@@ -73,6 +62,7 @@ void Millepede2Record::dummyRecord(float res, float err, float dGlo, int labGlo)
   mNDGlo[0] = 1;
   mVolID[0] = -1;
   mResid[0] = res;
+  mMeasType[0] = -1;
   mResErr[0] = err;
   //
   mIDLoc[0] = 0;
@@ -84,30 +74,30 @@ void Millepede2Record::dummyRecord(float res, float err, float dGlo, int labGlo)
 }
 
 //_________________________________________________________
-bool Millepede2Record::fillTrack(AlignmentTrack* trc, const int* id2Lab)
+bool Millepede2Record::fillTrack(AlignmentTrack& trc, const std::vector& id2Lab)
 {
   // fill track info, optionally substitutind glopar par ID by label
   //
-  if (!trc->getDerivDone()) {
+  if (!trc.getDerivDone()) {
     LOG(error) << "Track derivatives are not yet evaluated";
     return false;
   }
-  mNVarLoc = trc->getNLocPar(); // number of local degrees of freedom in the track
+  mNVarLoc = trc.getNLocPar(); // number of local degrees of freedom in the track
   mNResid = 0;
   mNDLocTot = 0;
   mNDGloTot = 0;
-  mChi2Ini = trc->getChi2Ini();
-  mQ2Pt = trc->getQ2Pt();
-  mTgl = trc->getTgl();
+  mChi2Ini = trc.getChi2Ini();
+  mQ2Pt = trc.getQ2Pt();
+  mTgl = trc.getTgl();
   mNMeas = 0;
-  setCosmic(trc->isCosmic());
+  setCosmic(trc.isCosmic());
   // 1) check sizes for buffers, expand if needed
-  int np = trc->getNPoints();
+  int np = trc.getNPoints();
   int nres = 0;
   int nlocd = 0;
   int nglod = 0;
-  for (int ip = np; ip--;) {
-    auto pnt = trc->getPoint(ip);
+  for (int ip = 0; ip < np; ip++) {
+    auto pnt = trc.getPoint(ip);
     int ngl = pnt->getNGloDOFs(); // number of DOF's this point depends on
     if (pnt->containsMeasurement()) {
       nres += 2;                    // every point has 2 residuals
@@ -123,11 +113,11 @@ bool Millepede2Record::fillTrack(AlignmentTrack* trc, const int* id2Lab)
   }
   //
   resize(nres, nlocd, nglod);
-  int nParETP = trc->getNLocExtPar(); // numnber of local parameters for reference track param
+  int nParETP = trc.getNLocExtPar(); // numnber of local parameters for reference track param
   //
-  const int* gloParID = trc->getGloParID(); // IDs of global DOFs this track depends on
+  const int* gloParID = trc.getGloParID(); // IDs of global DOFs this track depends on
   for (int ip = 0; ip < np; ip++) {
-    auto* pnt = trc->getPoint(ip);
+    auto* pnt = trc.getPoint(ip);
     if (pnt->containsMeasurement()) {
       int gloOffs = pnt->getDGloOffs(); // 1st entry of global derivatives for this point
       int nDGlo = pnt->getNGloDOFs();   // number of global derivatives (number of DOFs it depends on)
@@ -140,11 +130,12 @@ bool Millepede2Record::fillTrack(AlignmentTrack* trc, const int* id2Lab)
         mVolID[mNResid] = pnt->getSensor()->getVolID() + 1;
         //
         // measured residual/error
-        mResid[mNResid] = trc->getResidual(idim, ip);
+        mMeasType[mNResid] = idim;
+        mResid[mNResid] = trc.getResidual(idim, ip);
         mResErr[mNResid] = Sqrt(pnt->getErrDiag(idim));
         //
         // derivatives over local params
-        const double* deriv = trc->getDResDLoc(idim, ip); // array of Dresidual/Dparams_loc
+        const double* deriv = trc.getDResDLoc(idim, ip); // array of Dresidual/Dparams_loc
         int nnon0 = 0;
         for (int j = 0; j < nParETP; j++) { // derivatives over reference track parameters
           if (isZeroAbs(deriv[j])) {
@@ -158,6 +149,9 @@ bool Millepede2Record::fillTrack(AlignmentTrack* trc, const int* id2Lab)
         int lp0 = pnt->getMinLocVarID();  // point may depend on material variables starting from this one
         int lp1 = pnt->getMaxLocVarID();  // and up to this one (exclusive)
         for (int j = lp0; j < lp1; j++) { // derivatives over material variables
+          if (TMath::IsNaN(deriv[j])) {
+            LOGP(error, "Deriv {} of {}:{} is NAN", j, lp0, lp1);
+          }
           if (isZeroAbs(deriv[j])) {
             continue;
           }
@@ -171,7 +165,7 @@ bool Millepede2Record::fillTrack(AlignmentTrack* trc, const int* id2Lab)
         //
         // derivatives over global params
         nnon0 = 0;
-        deriv = trc->getDResDGlo(idim, gloOffs);
+        deriv = trc.getDResDGlo(idim, gloOffs);
         const int* gloIDP = gloParID + gloOffs;
         for (int j = 0; j < nDGlo; j++) {
           if (isZeroAbs(deriv[j])) {
@@ -179,7 +173,7 @@ bool Millepede2Record::fillTrack(AlignmentTrack* trc, const int* id2Lab)
           }
           nnon0++;
           mDGlo[mNDGloTot] = deriv[j];                                    // value of derivative
-          mIDGlo[mNDGloTot] = id2Lab ? id2Lab[gloIDP[j]] : gloIDP[j] + 1; // global DOF ID
+          mIDGlo[mNDGloTot] = id2Lab.empty() ? gloIDP[j] + 1 : id2Lab[gloIDP[j]]; // global DOF ID
           mNDGloTot++;
         }
         mNDGlo[mNResid] = nnon0;
@@ -194,6 +188,7 @@ bool Millepede2Record::fillTrack(AlignmentTrack* trc, const int* id2Lab)
       int offs = pnt->getMaxLocVarID() - nmatpar;    // start of material variables
       // here all derivatives are 1 = dx/dx
       for (int j = 0; j < nmatpar; j++) {
+        mMeasType[mNResid] = 10 + j;
         mNDGlo[mNResid] = 0; // mat corrections don't depend on global params
         mVolID[mNResid] = 0; // not associated to global parameter
         mResid[mNResid] = 0; // expectation for MS effects is 0
@@ -219,17 +214,20 @@ void Millepede2Record::resize(int nresid, int nloc, int nglo)
 {
   // resize container
   if (nresid > mNResidBook) {
+    delete[] mMeasType;
     delete[] mNDLoc;
     delete[] mNDGlo;
     delete[] mVolID;
     delete[] mResid;
     delete[] mResErr;
+    mMeasType = new int16_t[nresid];
     mNDLoc = new int16_t[nresid];
     mNDGlo = new int[nresid];
     mVolID = new int[nresid];
     mResid = new float[nresid];
     mResErr = new float[nresid];
     mNResidBook = nresid;
+    memset(mMeasType, 0, nresid * sizeof(int16_t));
     memset(mNDLoc, 0, nresid * sizeof(int16_t));
     memset(mNDGlo, 0, nresid * sizeof(int));
     memset(mVolID, 0, nresid * sizeof(int));
@@ -258,11 +256,10 @@ void Millepede2Record::resize(int nresid, int nloc, int nglo)
 }
 
 //____________________________________________
-void Millepede2Record::Clear(const Option_t*)
+void Millepede2Record::clear()
 {
   // reset record
-  TObject::Clear();
-  ResetBit(0xffffffff);
+  mBits = 0;
   mNResid = 0;
   mNVarLoc = 0;
   mNVarGlo = 0;
@@ -271,12 +268,11 @@ void Millepede2Record::Clear(const Option_t*)
 }
 
 //____________________________________________
-void Millepede2Record::Print(const Option_t*) const
+void Millepede2Record::print() const
 {
   // print info
   //
-  printf("Track %d Event TimeStamp:%d Run:%d\n",
-         mTrackID, mTimeStamp, getRun());
+  printf("Track %s TForbit:%d Run:%d\n", mTrackID.asString().c_str(), mFirstTFOrbit, mRunNumber);
   printf("Nmeas:%3d Q/pt:%+.2e Tgl:%+.2e Chi2Ini:%.1f\n", mNMeas, mQ2Pt, mTgl, mChi2Ini);
   printf("NRes: %3d NLoc: %3d NGlo:%3d | Stored: Loc:%3d Glo:%5d\n",
          mNResid, mNVarLoc, mNVarGlo, mNDLocTot, mNDGloTot);
@@ -285,8 +281,8 @@ void Millepede2Record::Print(const Option_t*) const
   const int kNColLoc = 5;
   for (int ir = 0; ir < mNResid; ir++) {
     int ndloc = mNDLoc[ir], ndglo = mNDGlo[ir];
-    printf("Res:%3d %+e (%+e) | NDLoc:%3d NDGlo:%4d [VID:%5d]\n",
-           ir, mResid[ir], mResErr[ir], ndloc, ndglo, getVolID(ir));
+    printf("Res:%3d Type:%d %+e (%+e) | NDLoc:%3d NDGlo:%4d [VID:%5d]\n",
+           ir, mMeasType[ir], mResid[ir], mResErr[ir], ndloc, ndglo, getVolID(ir));
     //
     printf("Local Derivatives:\n");
     bool eolOK = true;
diff --git a/Detectors/Align/src/ResidualsController.cxx b/Detectors/Align/src/ResidualsController.cxx
index 2b4631ea4de4a..b8d64ecea202f 100644
--- a/Detectors/Align/src/ResidualsController.cxx
+++ b/Detectors/Align/src/ResidualsController.cxx
@@ -25,20 +25,11 @@
 
 using namespace TMath;
 
-ClassImp(o2::align::ResidualsController);
-
 namespace o2
 {
 namespace align
 {
 
-//____________________________________
-ResidualsController::ResidualsController()
-  : mRun(0), mBz(0), mTimeStamp(0), mTrackID(0), mNPoints(0), mNBook(0), mChi2(0), mChi2Ini(0), mChi2K(0), mQ2Pt(0), mX(nullptr), mY(nullptr), mZ(nullptr), mSnp(nullptr), mTgl(nullptr), mAlpha(nullptr), mDY(nullptr), mDZ(nullptr), mDYK(nullptr), mDZK(nullptr), mSigY2(nullptr), mSigYZ(nullptr), mSigZ2(nullptr), mSigY2K(nullptr), mSigYZK(nullptr), mSigZ2K(nullptr), mVolID(nullptr), mLabel(nullptr)
-{
-  // def c-tor
-}
-
 //________________________________________________
 ResidualsController::~ResidualsController()
 {
@@ -130,15 +121,13 @@ void ResidualsController::resize(int np)
 }
 
 //____________________________________________
-void ResidualsController::Clear(const Option_t*)
+void ResidualsController::clear()
 {
   // reset record
-  TObject::Clear();
-  ResetBit(0xffffffff);
+  mBits = 0;
   mNPoints = 0;
-  mRun = 0;
-  mTimeStamp = 0;
-  mTrackID = 0;
+  mRunNumber = 0;
+  mFirstTFOrbit = 0;
   mChi2 = 0;
   mChi2K = 0;
   mQ2Pt = 0;
@@ -146,20 +135,17 @@ void ResidualsController::Clear(const Option_t*)
 }
 
 //____________________________________________
-void ResidualsController::Print(const Option_t* opt) const
+void ResidualsController::print(const Option_t* opt) const
 {
   // print info
   TString opts = opt;
   opts.ToLower();
   bool lab = opts.Contains("l");
+  printf("Track %s TForbit:%d Run:%d\n", mTrackID.asString().c_str(), mFirstTFOrbit, mRunNumber);
   printf("%5sTr.", isCosmic() ? "Cosm." : "Coll.");
-  if (isCosmic()) {
-    printf("%2d/%2d ", mTrackID >> 16, mTrackID & 0xffff);
-  } else {
-    printf("%5d ", mTrackID);
-  }
-  printf("Run:%6d Bz:%+4.1f Np: %3d q/Pt:%+.4f | Chi2: Ini: %6.1f LinSol:%6.1f Kalm:%6.1f |Vtx:%3s| TStamp:%d\n",
-         mRun, mBz, mNPoints, mQ2Pt, mChi2Ini, mChi2, mChi2K, hasVertex() ? "ON" : "OFF", mTimeStamp);
+  // in case of cosmic, other leg should be shown
+  printf("Run:%6d Bz:%+4.1f Np: %3d q/Pt:%+.4f | Chi2: Ini: %6.1f LinSol:%6.1f Kalm:%6.1f |Vtx:%3s\n",
+         mRunNumber, mBz, mNPoints, mQ2Pt, mChi2Ini, mChi2, mChi2K, hasVertex() ? "ON" : "OFF");
   if (opts.Contains("r")) {
     bool ers = opts.Contains("e");
     printf("%5s %7s %s %7s %7s %7s %5s %5s %9s %9s",
@@ -198,11 +184,11 @@ void ResidualsController::Print(const Option_t* opt) const
 }
 
 //____________________________________________________________
-bool ResidualsController::fillTrack(AlignmentTrack* trc, bool doKalman)
+bool ResidualsController::fillTrack(AlignmentTrack& trc, bool doKalman)
 {
   // fill tracks residuals info
-  int nps, np = trc->getNPoints();
-  if (trc->getInnerPoint()->containsMeasurement()) {
+  int nps, np = trc.getNPoints();
+  if (trc.getInnerPoint()->containsMeasurement()) {
     setHasVertex();
     nps = np;
   } else {
@@ -211,16 +197,16 @@ bool ResidualsController::fillTrack(AlignmentTrack* trc, bool doKalman)
   if (nps < 0) {
     return true;
   }
-  setCosmic(trc->isCosmic());
+  setCosmic(trc.isCosmic());
   //
   setNPoints(nps);
-  mQ2Pt = trc->getQ2Pt();
-  mChi2 = trc->getChi2();
-  mChi2Ini = trc->getChi2Ini();
+  mQ2Pt = trc.getQ2Pt();
+  mChi2 = trc.getChi2();
+  mChi2Ini = trc.getChi2Ini();
   int nfill = 0;
   for (int i = 0; i < np; i++) {
-    AlignmentPoint* pnt = trc->getPoint(i);
-    int inv = pnt->isInvDir() ? -1 : 1; // Flag invertion for cosmic upper leg
+    auto pnt = trc.getPoint(i);
+    int inv = pnt->isInvDir() ? -1 : 1; // Flag inversion for cosmic upper leg
     if (!pnt->containsMeasurement()) {
       continue;
     }
@@ -230,7 +216,7 @@ bool ResidualsController::fillTrack(AlignmentTrack* trc, bool doKalman)
     mVolID[nfill] = pnt->getVolID();
     mLabel[nfill] = pnt->getSensor()->getInternalID();
     mAlpha[nfill] = pnt->getAlphaSens();
-    mX[nfill] = pnt->getXPoint() * inv;
+    mX[nfill] = pnt->getXTracking();
     mY[nfill] = pnt->getYTracking();
     mZ[nfill] = pnt->getZTracking();
     mDY[nfill] = pnt->getResidY();
@@ -245,15 +231,15 @@ bool ResidualsController::fillTrack(AlignmentTrack* trc, bool doKalman)
     nfill++;
   }
   if (nfill != nps) {
-    trc->Print("p");
+    trc.Print("p");
     LOG(fatal) << nfill << " residuals were stored instead of " << nps;
   }
   //
   setKalmanDone(false);
   int nfilk = 0;
-  if (doKalman && trc->residKalman()) {
+  if (doKalman && trc.residKalman()) {
     for (int i = 0; i < np; i++) {
-      AlignmentPoint* pnt = trc->getPoint(i);
+      AlignmentPoint* pnt = trc.getPoint(i);
       if (!pnt->containsMeasurement()) {
         continue;
       }
@@ -270,7 +256,7 @@ bool ResidualsController::fillTrack(AlignmentTrack* trc, bool doKalman)
       nfilk++;
     }
     //
-    mChi2K = trc->getChi2();
+    mChi2K = trc.getChi2();
     setKalmanDone(true);
   }
 
diff --git a/Detectors/Base/CMakeLists.txt b/Detectors/Base/CMakeLists.txt
index 93de0acfc561c..83a9193274e4f 100644
--- a/Detectors/Base/CMakeLists.txt
+++ b/Detectors/Base/CMakeLists.txt
@@ -8,12 +8,14 @@
 # In applying this license CERN does not waive the privileges and immunities
 # granted to it by virtue of its status as an Intergovernmental Organization
 # or submit itself to any jurisdiction.
+#add_compile_options(-O0 -g -fPIC)
 
 o2_add_library(DetectorsBase
                SOURCES src/Detector.cxx
                        src/GeometryManager.cxx
                        src/MaterialManager.cxx
                        src/MaterialManagerParam.cxx
+                       src/GeometryManagerParam.cxx
                        src/Propagator.cxx
                        src/MatLayerCyl.cxx
                        src/MatLayerCylSet.cxx
@@ -24,9 +26,15 @@ o2_add_library(DetectorsBase
                        src/SimFieldUtils.cxx
                        src/TFIDInfoHelper.cxx
                        src/GRPGeomHelper.cxx
+                       src/Stack.cxx
+                       src/VMCSeederService.cxx
+                       src/GlobalParams.cxx
+                       src/O2Tessellated.cxx
+                       src/TGeoGeometryUtils.cxx
                PUBLIC_LINK_LIBRARIES FairRoot::Base
                                      O2::CommonUtils
                                      O2::DetectorsCommonDataFormats
+                                     O2::DataFormatsCTP
                                      O2::GPUCommon
                                      O2::GPUUtils
                                      O2::ReconstructionDataFormats
@@ -34,11 +42,13 @@ o2_add_library(DetectorsBase
                                      O2::Framework
                                      FairMQ::FairMQ
                                      O2::DataFormatsParameters
+                                     O2::SimulationDataFormat
                                      O2::SimConfig
                                      O2::CCDB
+                                     O2::GPUDataTypes
                                      MC::VMC
                                      TBB::tbb
-               PRIVATE_INCLUDE_DIRECTORIES ${CMAKE_SOURCE_DIR}/GPU/GPUTracking/Merger # Must not link to avoid cyclic dependency
+                                     ROOT::Gdml
                              )
 
 o2_target_root_dictionary(DetectorsBase
@@ -46,25 +56,54 @@ o2_target_root_dictionary(DetectorsBase
                                   include/DetectorsBase/GeometryManager.h
                                   include/DetectorsBase/MaterialManager.h
                                   include/DetectorsBase/MaterialManagerParam.h
+                                  include/DetectorsBase/GeometryManagerParam.h
                                   include/DetectorsBase/Propagator.h
                                   include/DetectorsBase/Ray.h
                                   include/DetectorsBase/MatCell.h
                                   include/DetectorsBase/MatLayerCyl.h
                                   include/DetectorsBase/MatLayerCylSet.h
                                   include/DetectorsBase/Aligner.h
-                                  include/DetectorsBase/SimFieldUtils.h)
+                                  include/DetectorsBase/Stack.h
+                                  include/DetectorsBase/SimFieldUtils.h
+                                  include/DetectorsBase/GlobalParams.h
+                                  include/DetectorsBase/O2Tessellated.h
+                                  )
 
 if(BUILD_SIMULATION)
+  if (NOT APPLE)
+    o2_add_test(
+      MatBudLUT
+      SOURCES test/testMatBudLUT.cxx
+      COMPONENT_NAME DetectorsBase
+      PUBLIC_LINK_LIBRARIES O2::DetectorsBase O2::ITSMFTReconstruction
+      LABELS detectorsbase
+      ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage
+                VMCWORKDIR=${CMAKE_BINARY_DIR}/stage/${CMAKE_INSTALL_DATADIR})
+  endif()
+
   o2_add_test(
-    MatBudLUT
-    SOURCES test/testMatBudLUT.cxx
+    MCStack
+    SOURCES test/testStack.cxx
     COMPONENT_NAME DetectorsBase
-    PUBLIC_LINK_LIBRARIES O2::DetectorsBase O2::ITSMFTReconstruction
+    PUBLIC_LINK_LIBRARIES O2::DetectorsBase
     LABELS detectorsbase
     ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage
                 VMCWORKDIR=${CMAKE_BINARY_DIR}/stage/${CMAKE_INSTALL_DATADIR})
 endif()
 
+install(FILES test/buildMatBudLUT.C
+              test/extractLUTLayers.C
+              test/rescaleLUT.C
+              DESTINATION share/macro/)
+
 o2_add_test_root_macro(test/buildMatBudLUT.C
                        PUBLIC_LINK_LIBRARIES O2::DetectorsBase
                        LABELS detectorsbase)
+
+o2_add_test_root_macro(test/extractLUTLayers.C
+                       PUBLIC_LINK_LIBRARIES O2::DetectorsBase
+                       LABELS detectorsbase)
+
+o2_add_test_root_macro(test/rescaleLUT.C
+                       PUBLIC_LINK_LIBRARIES O2::DetectorsBase
+                       LABELS detectorsbase)
diff --git a/Detectors/Base/include/DetectorsBase/CTFCoderBase.h b/Detectors/Base/include/DetectorsBase/CTFCoderBase.h
index efcb4792c0ae2..e94123bb2b7ff 100644
--- a/Detectors/Base/include/DetectorsBase/CTFCoderBase.h
+++ b/Detectors/Base/include/DetectorsBase/CTFCoderBase.h
@@ -21,14 +21,20 @@
 #include 
 #include "DetectorsCommonDataFormats/DetID.h"
 #include "CommonUtils/NameConf.h"
+#include "CommonUtils/IRFrameSelector.h"
 #include "DetectorsCommonDataFormats/CTFDictHeader.h"
 #include "DetectorsCommonDataFormats/CTFHeader.h"
 #include "DetectorsCommonDataFormats/CTFIOSize.h"
-#include "rANS/rans.h"
+#include "DataFormatsCTP/TriggerOffsetsParam.h"
+#include "DetectorsCommonDataFormats/ANSHeader.h"
+#include "rANS/factory.h"
+#include "rANS/compat.h"
+#include "rANS/histogram.h"
 #include 
 #include "Framework/InitContext.h"
 #include "Framework/ConcreteDataMatcher.h"
 #include "Framework/ConfigParamRegistry.h"
+#include 
 
 namespace o2
 {
@@ -52,8 +58,8 @@ class CTFCoderBase
                             Decoder };
 
   CTFCoderBase() = delete;
-  CTFCoderBase(int n, DetID det, float memFactor = 1.f) : mCoders(n), mDet(det), mMemMarginFactor(memFactor > 1.f ? memFactor : 1.f) {}
-  CTFCoderBase(OpType op, int n, DetID det, float memFactor = 1.f) : mOpType(op), mCoders(n), mDet(det), mMemMarginFactor(memFactor > 1.f ? memFactor : 1.f) {}
+  CTFCoderBase(int n, DetID det, float memFactor = 1.f, const std::string& ctfdictOpt = "none") : mCoders(n), mDet(det), mMemMarginFactor(memFactor > 1.f ? memFactor : 1.f), mDictOpt{ctfdictOpt} {}
+  CTFCoderBase(OpType op, int n, DetID det, float memFactor = 1.f, const std::string& ctfdictOpt = "none") : mOpType(op), mCoders(n), mDet(det), mMemMarginFactor(memFactor > 1.f ? memFactor : 1.f), mDictOpt{ctfdictOpt} {}
   virtual ~CTFCoderBase() = default;
 
   virtual void createCoders(const std::vector& bufVec, o2::ctf::CTFCoderBase::OpType op) = 0;
@@ -61,6 +67,13 @@ class CTFCoderBase
   // detector coder need to redefine this method if uses no default version, see comment in the cxx file
   virtual void assignDictVersion(CTFDictHeader& h) const;
 
+  template 
+  void setSelectedIRFrames(const SPAN& sp)
+  {
+    reportIRFrames();
+    mIRFrameSelector.setSelectedIRFrames(sp, mIRFrameSelMarginBwd, mIRFrameSelMarginFwd, mIRFrameSelShift, true);
+  }
+
   template 
   std::vector readDictionaryFromFile(const std::string& dictPath, bool mayFail = false);
 
@@ -68,19 +81,30 @@ class CTFCoderBase
   void createCodersFromFile(const std::string& dictPath, o2::ctf::CTFCoderBase::OpType op, bool mayFail = false);
 
   template 
-  void createCoder(OpType op, const o2::rans::RenormedFrequencyTable& renormedFrequencyTable, int slot)
+  void createCoder(OpType op, const o2::rans::RenormedDenseHistogram& renormedHistogram, int slot)
   {
-    if (renormedFrequencyTable.empty()) {
-      LOG(warning) << "Empty dictionary provided for slot " << slot << ", " << (op == OpType::Encoder ? "encoding" : "decoding") << " will assume literal symbols only";
-    }
-
-    switch (op) {
-      case OpType::Encoder:
-        mCoders[slot].reset(new o2::rans::LiteralEncoder64(renormedFrequencyTable));
-        break;
-      case OpType::Decoder:
-        mCoders[slot].reset(new o2::rans::LiteralDecoder64(renormedFrequencyTable));
-        break;
+    LOG_IF(warning, renormedHistogram.empty()) << fmt::format("Empty dictionary provided for slot {}, {} will assume literal symbols only", slot, (op == OpType::Encoder ? "encoding" : "decoding"));
+
+    if (mANSVersion == ANSVersionCompat) {
+      switch (op) {
+        case OpType::Encoder:
+          mCoders[slot] = std::make_any>(rans::compat::makeEncoder::fromRenormed(renormedHistogram));
+          break;
+        case OpType::Decoder:
+          mCoders[slot] = std::make_any>(rans::compat::makeDecoder::fromRenormed(renormedHistogram));
+          break;
+      }
+    } else if (mANSVersion == ANSVersion1) {
+      switch (op) {
+        case OpType::Encoder:
+          mCoders[slot] = std::make_any>(rans::makeDenseEncoder<>::fromRenormed(renormedHistogram));
+          break;
+        case OpType::Decoder:
+          mCoders[slot] = std::make_any>(rans::makeDecoder<>::fromRenormed(renormedHistogram));
+          break;
+      }
+    } else {
+      throw std::runtime_error("unsupported ANS version");
     }
   }
 
@@ -112,19 +136,74 @@ class CTFCoderBase
   template 
   bool finaliseCCDB(o2::framework::ConcreteDataMatcher& matcher, void* obj);
 
-  void updateTimeDependentParams(o2::framework::ProcessingContext& pc);
+  void updateTimeDependentParams(o2::framework::ProcessingContext& pc, bool askTree = false);
+
+  o2::utils::IRFrameSelector& getIRFramesSelector() { return mIRFrameSelector; }
+  size_t getIRFrameSelMarginBwd() const { return mIRFrameSelMarginBwd; }
+  size_t getIRFrameSelMarginFwd() const { return mIRFrameSelMarginFwd; }
+  long getIRFrameSelShift() const { return mIRFrameSelShift; }
+
+  inline const ctf::ANSHeader& getANSVersion() const noexcept { return mANSVersion; };
+  inline ctf::ANSHeader& getANSVersion() { return const_cast(const_cast(*this).getANSVersion()); };
+  inline void setANSVersion(const ctf::ANSHeader& ansVersion) noexcept { mANSVersion = ansVersion; };
+  void setBCShift(int64_t n) { mBCShift = n; }
+  void setFirstTFOrbit(uint32_t n) { mFirstTFOrbit = n; }
+  auto getBCShift() const { return mBCShift; }
+  auto getFirstTFOrbit() const { return mFirstTFOrbit; }
+  void setSupportBCShifts(bool v = true) { mSupportBCShifts = v; }
+  bool getSupportBCShifts() const { return mSupportBCShifts; }
+
+  void setDictBinding(const std::string& s) { mDictBinding = s; }
+  const std::string& getDictBinding() const { return mDictBinding; }
+
+  void setTrigOffsBinding(const std::string& s) { mTrigOffsBinding = s; }
+  const std::string& getTrigOffsBinding() const { return mTrigOffsBinding; }
+
+  const DetID getDet() const { return mDet; }
 
  protected:
+  void reportIRFrames();
   std::string getPrefix() const { return o2::utils::Str::concat_string(mDet.getName(), "_CTF: "); }
-
   void checkDictVersion(const CTFDictHeader& h) const;
+  bool isTreeDictionary(const void* buff) const;
+  bool canApplyBCShift(const o2::InteractionRecord& ir, long shift) const
+  {
+    auto diff = ir.differenceInBC({0, mFirstTFOrbit});
+    return diff < 0 ? true : diff >= shift;
+  }
+  bool canApplyBCShift(const o2::InteractionRecord& ir) const { return canApplyBCShift(ir, mBCShift); }
+
+  template 
+  [[nodiscard]] size_t estimateBufferSize(size_t slot, source_IT samplesBegin, source_IT samplesEnd);
 
-  std::vector> mCoders; // encoders/decoders
+  template 
+  size_t estimateBufferSize(size_t slot, size_t nSamples);
+
+  template 
+  [[nodiscard]] size_t estimateBufferSize(size_t slot, const std::vector& samples)
+  {
+    return estimateBufferSize(slot, samples.begin(), samples.end());
+  }
+
+  template 
+  std::vector loadDictionaryFromTree(TTree* tree);
+  std::vector mCoders; // encoders/decoders
   DetID mDet;
-  CTFDictHeader mExtHeader;      // external dictionary header
-  float mMemMarginFactor = 1.0f; // factor for memory allocation in EncodedBlocks
+  std::string mDictOpt{};
+  std::string mDictBinding{"ctfdict"};
+  std::string mTrigOffsBinding{"trigoffset"};
+  CTFDictHeader mExtHeader;                    // external dictionary header
+  o2::utils::IRFrameSelector mIRFrameSelector; // optional IR frames selector
+  float mMemMarginFactor = 1.0f;               // factor for memory allocation in EncodedBlocks
   bool mLoadDictFromCCDB{true};
-  OpType mOpType; // Encoder or Decoder
+  bool mSupportBCShifts{false};
+  OpType mOpType;                                    // Encoder or Decoder
+  ctf::ANSHeader mANSVersion{ctf::ANSVersionCompat}; // Version of the ANSEncoder/Decoder
+  int64_t mBCShift = 0;                              // shift to apply to decoded IR (i.e. CTP offset if was not corrected on raw data decoding level)
+  uint32_t mFirstTFOrbit = 0;
+  size_t mIRFrameSelMarginBwd = 0; // margin in BC to add to the IRFrame lower boundary when selection is requested
+  size_t mIRFrameSelMarginFwd = 0; // margin in BC to add to the IRFrame upper boundary when selection is requested
+  long mIRFrameSelShift = 0;       // Global shift of the IRFrames, to account for e.g. detector latency
   int mVerbosity = 0;
 };
 
@@ -157,6 +236,18 @@ void CTFCoderBase::createCodersFromFile(const std::string& dictPath, o2::ctf::CT
   createCoders(buff, op);
 }
 
+///________________________________
+template 
+std::vector CTFCoderBase::loadDictionaryFromTree(TTree* tree)
+{
+  std::vector bufVec;
+  CTFHeader ctfHeader;
+  if (readFromTree(*tree, "CTFHeader", ctfHeader) && ctfHeader.detectors[mDet]) {
+    CTF::readFromTree(bufVec, *tree, mDet.getName());
+  }
+  return bufVec;
+}
+
 ///________________________________
 template 
 std::vector CTFCoderBase::readDictionaryFromFile(const std::string& dictPath, bool mayFail)
@@ -175,26 +266,22 @@ std::vector CTFCoderBase::readDictionaryFromFile(const std::string& dictPa
     }
     return bufVec;
   }
-  std::unique_ptr tree((TTree*)fileDict->Get(std::string(o2::base::NameConf::CTFDICT).c_str()));
+  std::unique_ptr tree;
   std::unique_ptr> bv((std::vector*)fileDict->GetObjectChecked(o2::base::NameConf::CCDBOBJECT.data(), "std::vector"));
+  tree.reset((TTree*)fileDict->GetObjectChecked(o2::base::NameConf::CTFDICT.data(), "TTree"));
+  if (!tree) {
+    tree.reset((TTree*)fileDict->GetObjectChecked(o2::base::NameConf::CCDBOBJECT.data(), "TTree"));
+  }
   if (tree) {
-    CTFHeader ctfHeader;
-    if (!readFromTree(*tree.get(), "CTFHeader", ctfHeader) || !ctfHeader.detectors[mDet]) {
-      std::string errstr = fmt::format("CTF dictionary file for detector {} is absent in the tree from file {}", mDet.getName(), dictPath);
-      if (mayFail) {
-        LOGP(info, "{}, will assume dictionary stored in CTF", errstr);
-      } else {
-        throw std::runtime_error(errstr);
-      }
-      return bufVec;
-    }
-    CTF::readFromTree(bufVec, *tree.get(), mDet.getName());
+    auto v = loadDictionaryFromTree(tree.get());
+    bufVec.swap(v);
   } else if (bv) {
     bufVec.swap(*bv);
     if (bufVec.size()) {
       auto dictHeader = static_cast(CTF::get(bufVec.data())->getHeader());
       if (dictHeader.det != mDet) {
-        throw std::runtime_error(fmt::format("{} contains dictionary vector for {}, expected {}", dictPath, dictHeader.det.getName(), mDet.getName()));
+        bufVec.clear();
+        LOGP(error, "{} contains dictionary vector for {}, expected {}", dictPath, dictHeader.det.getName(), mDet.getName());
       }
     }
   }
@@ -202,11 +289,10 @@ std::vector CTFCoderBase::readDictionaryFromFile(const std::string& dictPa
     mExtHeader = static_cast(CTF::get(bufVec.data())->getHeader());
     LOGP(debug, "Found {} in {}", mExtHeader.asString(), dictPath);
   } else {
-    std::string errstr = fmt::format("CTF dictionary file for detector {} is empty", mDet.getName());
     if (mayFail) {
-      LOGP(info, "{}, will assume dictionary stored in CTF", errstr);
+      LOGP(info, "{}, will assume dictionary stored in CTF", mDet.getName());
     } else {
-      throw std::runtime_error(errstr);
+      LOGP(fatal, "CTF dictionary file for detector {} is empty", mDet.getName());
     }
   }
   return bufVec;
@@ -219,15 +305,40 @@ void CTFCoderBase::init(o2::framework::InitContext& ic)
   if (ic.options().hasOption("mem-factor")) {
     setMemMarginFactor(ic.options().get("mem-factor"));
   }
-  auto dict = ic.options().get("ctf-dict");
-  if (dict.empty() || dict == "ccdb") { // load from CCDB
+  if (ic.options().hasOption("irframe-margin-bwd")) {
+    mIRFrameSelMarginBwd = ic.options().get("irframe-margin-bwd");
+  }
+  if (ic.options().hasOption("irframe-margin-fwd")) {
+    mIRFrameSelMarginFwd = ic.options().get("irframe-margin-fwd");
+  }
+  if (ic.options().hasOption("irframe-shift")) {
+    mIRFrameSelShift = (long)ic.options().get("irframe-shift");
+  }
+  bool ansVersionProvided = false;
+  if (ic.options().hasOption("ans-version")) {
+    if (ic.options().isSet("ans-version")) {
+      const std::string ansVersionString = ic.options().get("ans-version");
+      if (!ansVersionString.empty()) {
+        mANSVersion = ansVersionFromString(ansVersionString);
+        LOGP(info, "parsing ansVersionString {} into {}", ansVersionString, static_cast(mANSVersion));
+        if (mANSVersion == ANSVersionUnspecified) {
+          throw std::invalid_argument(fmt::format("Invalid ANS Version {}", ansVersionString));
+        }
+        ansVersionProvided = true;
+      }
+    }
+  }
+  if (mDictOpt.empty() || mDictOpt == "ccdb") { // load from CCDB
     mLoadDictFromCCDB = true;
   } else {
-    if (dict != "none") { // none means per-CTF dictionary will created on the fly
-      createCodersFromFile(dict, mOpType);
-      LOGP(info, "Loaded {} from {}", mExtHeader.asString(), dict);
+    if (mDictOpt != "none") { // none means per-CTF dictionary will created on the fly
+      createCodersFromFile(mDictOpt, mOpType);
+      LOGP(info, "Loaded {} from {}, ANS Version {}", mExtHeader.asString(), mDictOpt, std::string(mANSVersion));
     } else {
-      LOGP(info, "Internal per-TF CTF Dict will be created");
+      if (!ansVersionProvided) {
+        mANSVersion = ANSVersion1;
+      }
+      LOGP(info, "Internal per-TF CTF Dict will be created, ANS Version {}", std::string(mANSVersion));
     }
     mLoadDictFromCCDB = false; // don't try to load from CCDB
   }
@@ -254,15 +365,65 @@ bool CTFCoderBase::finaliseCCDB(o2::framework::ConcreteDataMatcher& matcher, voi
     if (dict->empty()) {
       LOGP(info, "Empty dictionary object fetched from CCDB, internal per-TF CTF Dict will be created");
     } else {
+      std::vector bufVec;
+      if (isTreeDictionary(obj)) {
+        auto v = loadDictionaryFromTree(reinterpret_cast(obj));
+        bufVec.swap(v);
+        dict = &bufVec;
+      }
       createCoders(*dict, mOpType);
       mExtHeader = static_cast(CTF::get(dict->data())->getHeader());
-      LOGP(info, "Loaded {} from CCDB", mExtHeader.asString());
+      LOGP(info, "Loaded {} from CCDB, ANS Version {}", mExtHeader.asString(), std::string(mANSVersion));
     }
     mLoadDictFromCCDB = false; // we read the dictionary at most once!
+  } else if ((match = (matcher == o2::framework::ConcreteDataMatcher("CTP", "Trig_Offset", 0)))) {
+    const auto& trigOffsParam = o2::ctp::TriggerOffsetsParam::Instance();
+    auto bcshift = trigOffsParam.customOffset[mDet.getID()];
+    if (bcshift) {
+      if (mSupportBCShifts) {
+        LOGP(info, "Decoded IRs will be augmented by {} BCs, discarded if become prior to 1st orbit", bcshift);
+        setBCShift(-bcshift); // offset is subtracted
+      } else {
+        LOGP(alarm, "Decoding with {} BCs shift is requested, but the {} does not support this operation, ignoring request", bcshift, mDet.getName());
+      }
+    }
   }
   return match;
 }
 
+template 
+[[nodiscard]] inline size_t CTFCoderBase::estimateBufferSize(size_t slot, IT samplesBegin, IT samplesEnd)
+{
+  using source_type = typename std::iterator_traits::value_type;
+  const size_t nSamples = std::distance(samplesBegin, samplesEnd);
+  return estimateBufferSize(slot, nSamples);
+};
+
+template 
+[[nodiscard]] inline size_t CTFCoderBase::estimateBufferSize(size_t slot, size_t nSamples)
+{
+
+  std::any& coder = mCoders[slot];
+  if (coder.has_value()) {
+    const size_t alphabetRangeBits = [this, &coder]() {
+      if (mANSVersion == ANSVersionCompat) {
+        const auto& encoder = std::any_cast&>(coder);
+        auto view = rans::trim(rans::makeHistogramView(encoder.getSymbolTable()));
+        return rans::utils::getRangeBits(view.getMin(), view.getMax());
+      } else if (mANSVersion == ANSVersion1) {
+        const auto& encoder = std::any_cast&>(coder);
+        auto view = rans::trim(rans::makeHistogramView(encoder.getSymbolTable()));
+        return rans::utils::getRangeBits(view.getMin(), view.getMax());
+      } else {
+        throw std::runtime_error("unsupported ANS version");
+      }
+    }();
+    return rans::compat::calculateMaxBufferSizeB(nSamples, alphabetRangeBits);
+  } else {
+    return nSamples * sizeof(source_T);
+  }
+};
+
 } // namespace ctf
 } // namespace o2
 
diff --git a/Detectors/Base/include/DetectorsBase/DPLWorkflowUtils.h b/Detectors/Base/include/DetectorsBase/DPLWorkflowUtils.h
index ba87b6ecd7ddb..ba3dd821fa118 100644
--- a/Detectors/Base/include/DetectorsBase/DPLWorkflowUtils.h
+++ b/Detectors/Base/include/DetectorsBase/DPLWorkflowUtils.h
@@ -25,9 +25,15 @@
 #include "Framework/CompletionPolicy.h"
 #include "Framework/CompletionPolicyHelpers.h"
 #include "Framework/DeviceSpec.h"
+#include "Framework/Task.h"
 #include "Framework/DataSpecUtils.h"
 #include 
 #include 
+#include 
+#ifdef WITH_OPENMP
+#include 
+#include "TROOT.h"
+#endif
 
 namespace o2
 {
@@ -160,7 +166,12 @@ o2::framework::DataProcessorSpec specCombiner(std::string const& name, std::vect
     }
     for (auto& is : spec.inputs) {
       if (inputBindings.find(is.binding) != inputBindings.end()) {
-        LOG(error) << "Found duplicate input binding " << is.binding;
+        // we can accept duplicate binding if it is bound to the same spec (e.g. ccdbParamSpec)
+        if (std::find(combinedInputSpec.begin(), combinedInputSpec.end(), is) == combinedInputSpec.end()) {
+          LOG(error) << "Found duplicate input binding with different spec.:" << is;
+        } else {
+          continue; // consider as already accounted
+        }
       }
       combinedInputSpec.push_back(is);
       inputBindings[is.binding] = 1;
@@ -212,6 +223,16 @@ o2::framework::DataProcessorSpec specCombiner(std::string const& name, std::vect
     void init(o2::framework::InitContext& ic)
     {
       std::cerr << "Init Combined\n";
+      mNThreads = std::max(1, ic.options().get("combine-nthreads"));
+      if (mNThreads > 1) {
+#ifdef WITH_OPENMP
+        LOGP(info, "Combined tasks will be run with {} threads", mNThreads);
+        ROOT::EnableThreadSafety();
+#else
+        LOGP(warn, "{} threads requested for combined tasks but OpenMP is not detected, link it from the workflow CMakeList", mNThreads);
+        mNThreads = 1;
+#endif
+      }
       for (auto& t : tasks) {
         // the init function actually creates the onProcess function
         // which we have to do here (maybe some more stuff needed)
@@ -236,18 +257,33 @@ o2::framework::DataProcessorSpec specCombiner(std::string const& name, std::vect
 
     void run(o2::framework::ProcessingContext& pc)
     {
-      std::cerr << "Processing Combined\n";
-      for (auto& t : tasks) {
-        std::cerr << " Executing sub-device " << t.name << "\n";
-        t.algorithm.onProcess(pc);
+      std::cerr << "Processing Combined with " << mNThreads << " threads\n";
+      if (mNThreads > 1) {
+        size_t nt = tasks.size();
+#ifdef WITH_OPENMP
+#pragma omp parallel for schedule(dynamic) num_threads(mNThreads)
+#endif
+        for (size_t i = 0; i < nt; i++) {
+          auto& t = tasks[i];
+          std::cerr << " Executing sub-device " << t.name << "\n";
+          t.algorithm.onProcess(pc);
+        }
+      } else {
+        for (auto& t : tasks) {
+          std::cerr << " Executing sub-device " << t.name << "\n";
+          t.algorithm.onProcess(pc);
+        }
       }
     }
 
    private:
     std::vector tasks;
     std::unordered_map> optionsRemap;
+    int mNThreads = 1;
   };
 
+  combinedOptions.emplace_back(ConfigParamSpec{"combine-nthreads", VariantType::Int, 1, {"Number of threads for combined tasks"}});
+
   return DataProcessorSpec{
     name,
     combinedInputSpec,
diff --git a/Detectors/Base/include/DetectorsBase/Detector.h b/Detectors/Base/include/DetectorsBase/Detector.h
index 73764e8b7021b..f1744086d6a05 100644
--- a/Detectors/Base/include/DetectorsBase/Detector.h
+++ b/Detectors/Base/include/DetectorsBase/Detector.h
@@ -29,7 +29,6 @@
 #include 
 #include 
 #include 
-#include 
 #include "CommonUtils/ShmManager.h"
 #include "CommonUtils/ShmAllocator.h"
 #include 
@@ -42,9 +41,7 @@
 
 #include 
 
-namespace o2
-{
-namespace base
+namespace o2::base
 {
 
 /// This is the basic class for any AliceO2 detector module, whether it is
@@ -109,6 +106,9 @@ class Detector : public FairDetector
   /// declare alignable volumes of detector
   virtual void addAlignableVolumes() const;
 
+  /// fill parallel geometry with sensitive volumes of detector
+  virtual void fillParallelWorld() const;
+
   /// Sets per wrapper volume parameters
   virtual void defineWrapperVolume(Int_t id, Double_t rmin, Double_t rmax, Double_t zspan);
 
@@ -216,6 +216,10 @@ class Detector : public FairDetector
   // which is required for media creation
   static void initFieldTrackingParams(int& mode, float& maxfield);
 
+  /// set the DetID to HitBitIndex mapping. Succeeds if not already set.
+  static void setDetId2HitBitIndex(std::vector const& v) { Detector::sDetId2HitBitIndex = v; }
+  static std::vector const& getDetId2HitBitIndex() { return Detector::sDetId2HitBitIndex; }
+
  protected:
   Detector(const Detector& origin);
 
@@ -233,6 +237,7 @@ class Detector : public FairDetector
 
   static Float_t mDensityFactor; //! factor that is multiplied to all material densities (ONLY for
   // systematic studies)
+  static std::vector sDetId2HitBitIndex; //! global lookup table keeping mapping of DetID to index in hit bit field (used in MCTrack)
 
   ClassDefOverride(Detector, 1); // Base class for ALICE Modules
 };
@@ -255,17 +260,12 @@ T decodeShmMessage(fair::mq::Parts& dataparts, int index, bool*& busy)
 }
 
 // this goes into the source
-void attachMessageBufferToParts(fair::mq::Parts& parts, fair::mq::Channel& channel,
-                                void* data, size_t size, void (*func_ptr)(void* data, void* hint), void* hint);
+void attachMessageBufferToParts(fair::mq::Parts& parts, fair::mq::Channel& channel, void* data, TClass* cl);
 
 template 
 void attachTMessage(Container const& hits, fair::mq::Channel& channel, fair::mq::Parts& parts)
 {
-  TMessage* tmsg = new TMessage();
-  tmsg->WriteObjectAny((void*)&hits, TClass::GetClass(typeid(hits)));
-  attachMessageBufferToParts(
-    parts, channel, tmsg->Buffer(), tmsg->BufferSize(),
-    [](void* data, void* hint) { delete static_cast(hint); }, tmsg);
+  attachMessageBufferToParts(parts, channel, (void*)&hits, TClass::GetClass(typeid(hits)));
 }
 
 void* decodeTMessageCore(fair::mq::Parts& dataparts, int index);
@@ -738,9 +738,9 @@ class DetImpl : public o2::base::Detector
   int mInitialized = false;
 
   char* mHitCollectorBufferPtr = nullptr; //! pointer to hit (collector) buffer location (strictly internal)
+
   ClassDefOverride(DetImpl, 0);
 };
-} // namespace base
-} // namespace o2
+} // namespace o2::base
 
 #endif
diff --git a/Detectors/Base/include/DetectorsBase/GRPGeomHelper.h b/Detectors/Base/include/DetectorsBase/GRPGeomHelper.h
index ef502a12695d6..ec1f9a91c3f20 100644
--- a/Detectors/Base/include/DetectorsBase/GRPGeomHelper.h
+++ b/Detectors/Base/include/DetectorsBase/GRPGeomHelper.h
@@ -22,6 +22,7 @@
 #include "DataFormatsParameters/GRPLHCIFData.h"
 #include "DataFormatsParameters/GRPECSObject.h"
 #include "DataFormatsParameters/GRPMagField.h"
+#include "DataFormatsParameters/AggregatedRunInfo.h"
 
 namespace o2::framework
 {
@@ -92,6 +93,7 @@ struct GRPGeomRequest {
                      Ideal,
                      Alignments };
 
+  bool askAggregateRunInfo = false;
   bool askGRPECS = false;
   bool askGRPLHCIF = false;
   bool askGRPMagField = false;
@@ -101,9 +103,11 @@ struct GRPGeomRequest {
   bool askGeomIdeal = false;  // load ideal geometry
   bool askAlignments = false; // load detector alignments but don't apply them
   bool askOnceAllButField = false; // for all entries but field query only once
+  bool needPropagatorD = false;    // init also PropagatorD
 
   GRPGeomRequest() = delete;
-  GRPGeomRequest(bool orbitResetTime, bool GRPECS, bool GRPLHCIF, bool GRPMagField, bool askMatLUT, GeomRequest geom, std::vector& inputs, bool askOnce = false);
+  GRPGeomRequest(bool orbitResetTime, bool GRPECS, bool GRPLHCIF, bool GRPMagField, bool askMatLUT, GeomRequest geom, std::vector& inputs, bool askOnce = false, bool needPropD = false, std::string detMaskString = "all");
+  void requireAggregateRunInfo(std::vector& inputs);
   void addInput(const o2::framework::InputSpec&& isp, std::vector& inputs);
 };
 
@@ -120,15 +124,16 @@ class GRPGeomHelper
   }
   void setRequest(std::shared_ptr req);
   bool finaliseCCDB(o2::framework::ConcreteDataMatcher& matcher, void* obj);
-  void checkUpdates(o2::framework::ProcessingContext& pc) const;
+  void checkUpdates(o2::framework::ProcessingContext& pc);
 
   auto getAlignment(o2::detectors::DetID det) const { return mAlignments[det]; }
   auto getMatLUT() const { return mMatLUT; }
   auto getGRPECS() const { return mGRPECS; }
   auto getGRPLHCIF() const { return mGRPLHCIF; }
   auto getGRPMagField() const { return mGRPMagField; }
-  auto getOrbitResetTimeMS() const { return mOrbitResetTimeMS; }
-
+  auto getOrbitResetTimeMS() const { return mOrbitResetTimeMUS / 1000; }
+  auto getOrbitResetTimeMUS() const { return mOrbitResetTimeMUS; }
+  const o2::parameters::AggregatedRunInfo& getAggregatedRunInfo() const { return mAggregatedRunInfo; }
   static int getNHBFPerTF();
 
  private:
@@ -141,7 +146,8 @@ class GRPGeomHelper
   const o2::parameters::GRPECSObject* mGRPECS = nullptr;
   const o2::parameters::GRPLHCIFData* mGRPLHCIF = nullptr;
   const o2::parameters::GRPMagField* mGRPMagField = nullptr;
-  long mOrbitResetTimeMS = 0; // orbit reset time in milliseconds
+  o2::parameters::AggregatedRunInfo mAggregatedRunInfo{};
+  long mOrbitResetTimeMUS = 0; // orbit reset time in microseconds
 };
 
 } // namespace base
diff --git a/Detectors/Base/include/DetectorsBase/GeometryManager.h b/Detectors/Base/include/DetectorsBase/GeometryManager.h
index 96998187cca63..ad1d77b10f49a 100644
--- a/Detectors/Base/include/DetectorsBase/GeometryManager.h
+++ b/Detectors/Base/include/DetectorsBase/GeometryManager.h
@@ -86,8 +86,7 @@ class GeometryManager : public TObject
     double meanZ = 0.;    // mean Z: sum(x_i*Z_i)/sum(x_i) [adimensional]
     double meanZ2A = 0.;  // Z/A mean: sum(x_i*Z_i/A_i)/sum(x_i) [adimensional]
     double length = -1.;  // length: sum(x_i) [cm]
-    int nCross = 0;
-    ; // number of boundary crosses
+    int nCross = 0;       // number of boundary crosses
 
     MatBudgetExt() = default;
     ~MatBudgetExt() = default;
@@ -132,6 +131,7 @@ class GeometryManager : public TObject
   /// detector geometry. The output global matrix is stored in 'm'.
   /// Returns kFALSE in case TGeo has not been initialized or the volume path is not valid.
   static Bool_t getOriginalMatrixFromPath(const char* path, TGeoHMatrix& m);
+
  private:
   /// sensitive volume identifier composed from (det_ID< {
+  bool useParallelWorld = false;
+  bool usePwGeoBVH = false;
+  bool usePwCaching = false;
+
+  int printLevel = -1; // geometry level to print out (determined by the volume path)
+
+  O2ParamDef(GeometryManagerParam, "GeometryManagerParam");
+};
+
+} // namespace o2
+
+#endif /* DETECTORS_BASE_INCLUDE_GEOMETRYMANAGERPARAM_H_ */
diff --git a/Detectors/Base/include/DetectorsBase/GlobalParams.h b/Detectors/Base/include/DetectorsBase/GlobalParams.h
new file mode 100644
index 0000000000000..25bee925196d3
--- /dev/null
+++ b/Detectors/Base/include/DetectorsBase/GlobalParams.h
@@ -0,0 +1,29 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef DETECTORS_BASE_GLOBALPARAMS_H_
+#define DETECTORS_BASE_GLOBALPARAMS_H_
+
+#include "CommonUtils/ConfigurableParam.h"
+#include "CommonUtils/ConfigurableParamHelper.h"
+
+namespace o2
+{
+
+struct GlobalParams : public o2::conf::ConfigurableParamHelper {
+  bool withITS3 = false; // detector geometry includes ITS3
+
+  O2ParamDef(GlobalParams, "GlobalParams");
+};
+
+} // namespace o2
+
+#endif // DETECTORS_BASE_GLOBALPARAMS_H_
diff --git a/Detectors/Base/include/DetectorsBase/MatCell.h b/Detectors/Base/include/DetectorsBase/MatCell.h
index b5af66439955d..40c5fd3db1f69 100644
--- a/Detectors/Base/include/DetectorsBase/MatCell.h
+++ b/Detectors/Base/include/DetectorsBase/MatCell.h
@@ -31,7 +31,13 @@ struct MatCell {
   float meanX2X0;                   ///< fraction of radiaton lenght
 
   GPUd() MatCell() : meanRho(0.f), meanX2X0(0.f) {}
-  GPUdDefault() MatCell(const MatCell& src) CON_DEFAULT;
+  GPUdDefault() MatCell(const MatCell& src) = default;
+
+  GPUd() void set(const MatCell& c)
+  {
+    meanRho = c.meanRho;
+    meanX2X0 = c.meanX2X0;
+  }
 
   GPUd() void scale(float scale)
   {
@@ -49,7 +55,7 @@ struct MatBudget : MatCell {
   float length;                     ///< length in material
 
   GPUd() MatBudget() : length(0.f) {}
-  GPUdDefault() MatBudget(const MatBudget& src) CON_DEFAULT;
+  GPUdDefault() MatBudget(const MatBudget& src) = default;
 
   GPUd() void scale(float scale)
   {
diff --git a/Detectors/Base/include/DetectorsBase/MatLayerCyl.h b/Detectors/Base/include/DetectorsBase/MatLayerCyl.h
index 4d509fdf9e3c1..e63de51e0a6ca 100644
--- a/Detectors/Base/include/DetectorsBase/MatLayerCyl.h
+++ b/Detectors/Base/include/DetectorsBase/MatLayerCyl.h
@@ -54,9 +54,11 @@ class MatLayerCyl : public o2::gpu::FlatObject
                              Within = 0,
                              Above = 1 };
 
+#ifndef GPUCA_GPUCODE
   MatLayerCyl();
-  MatLayerCyl(const MatLayerCyl& src) CON_DELETE;
-  ~MatLayerCyl() CON_DEFAULT;
+  MatLayerCyl(const MatLayerCyl& src) = delete;
+  ~MatLayerCyl() = default;
+#endif
 
 #ifndef GPUCA_ALIGPUCODE // this part is unvisible on GPU version
   MatLayerCyl(float rMin, float rMax, float zHalfSpan, float dzMin, float drphiMin);
@@ -90,6 +92,15 @@ class MatLayerCyl : public o2::gpu::FlatObject
   GPUd() const MatCell& getCellPhiBin(int iphi, int iz) const { return mCells[getCellIDPhiBin(iphi, iz)]; }
   GPUd() const MatCell& getCell(int iphiSlice, int iz) const { return mCells[getCellID(iphiSlice, iz)]; }
 
+#ifndef GPUCA_ALIGPUCODE // this part is unvisible on GPU version
+  MatCell& getCellPhiBin(int iphi, int iz)
+  {
+    return mCells[getCellIDPhiBin(iphi, iz)];
+  }
+
+  void scale(float factor, bool _x2x0 = true, bool _rho = true);
+#endif
+
   // ---------------------- Z slice manipulation
   // convert Z to Zslice
   GPUd() RangeStatus isZOutside(float z) const { return z < getZMin() ? Below : (z > getZMax() ? Above : Within); }
@@ -156,6 +167,8 @@ class MatLayerCyl : public o2::gpu::FlatObject
   /// Gives minimal alignment in bytes required for the flat buffer
   static constexpr size_t getBufferAlignmentBytes() { return 8; }
 #endif
+  // linearized cell ID from phi bin and z bin
+  GPUd() int getCellIDPhiBin(int iphi, int iz) const { return getCellID(phiBin2Slice(iphi), iz); }
 
  protected:
   GPUd() int getNCells() const { return getNZBins() * getNPhiSlices(); }
@@ -165,9 +178,6 @@ class MatLayerCyl : public o2::gpu::FlatObject
   // linearized cell ID from phi slice and z bin
   GPUd() int getCellID(int iphi, int iz) const { return iphi * getNZBins() + iz; }
 
-  // linearized cell ID from phi bin and z bin
-  GPUd() int getCellIDPhiBin(int iphi, int iz) const { return getCellID(phiBin2Slice(iphi), iz); }
-
   // convert Phi (in 0:2pi convention) to PhiBinID
   GPUd() int getPhiBinID(float phi) const
   {
diff --git a/Detectors/Base/include/DetectorsBase/MatLayerCylSet.h b/Detectors/Base/include/DetectorsBase/MatLayerCylSet.h
index 8c25c07c0ccc1..cba6e5cebcfc8 100644
--- a/Detectors/Base/include/DetectorsBase/MatLayerCylSet.h
+++ b/Detectors/Base/include/DetectorsBase/MatLayerCylSet.h
@@ -51,9 +51,11 @@ class MatLayerCylSet : public o2::gpu::FlatObject
 {
 
  public:
-  MatLayerCylSet() CON_DEFAULT;
-  ~MatLayerCylSet() CON_DEFAULT;
-  MatLayerCylSet(const MatLayerCylSet& src) CON_DELETE;
+#ifndef GPUCA_GPUCODE
+  MatLayerCylSet() = default;
+  ~MatLayerCylSet() = default;
+  MatLayerCylSet(const MatLayerCylSet& src) = delete;
+#endif
 
   GPUd() const MatLayerCylSetLayout* get() const { return reinterpret_cast(mFlatBufferPtr); }
   GPUd() MatLayerCylSetLayout* get() { return reinterpret_cast(mFlatBufferPtr); }
@@ -79,8 +81,15 @@ class MatLayerCylSet : public o2::gpu::FlatObject
   static MatLayerCylSet* loadFromFile(const std::string& inpFName = "matbud.root");
   static MatLayerCylSet* rectifyPtrFromFile(MatLayerCylSet* ptr);
 
+  // initializes internal voxel lookup
+  void initLayerVoxelLU();
+
   void flatten();
 
+  MatLayerCyl& getLayer(int i) { return get()->mLayers[i]; }
+  MatLayerCylSet* extractCopy(float rmin, float rmax, float tol = 1e-3, const MatLayerCylSet* toAdd = nullptr) const;
+  void finalizeStructures();
+
 #endif // !GPUCA_ALIGPUCODE
 
 #ifndef GPUCA_ALIGPUCODE // this part is unvisible on GPU version
@@ -89,11 +98,18 @@ class MatLayerCylSet : public o2::gpu::FlatObject
     // get material budget traversed on the line between point0 and point1
     return getMatBudget(point0.X(), point0.Y(), point0.Z(), point1.X(), point1.Y(), point1.Z());
   }
+
+  void scaleLayersByID(int lrFrom, int lrTo, float factor, bool _x2x0 = true, bool _rho = true);
+  void scaleLayersByR(float rFrom, float rTo, float factor, bool _x2x0 = true, bool _rho = true);
+
 #endif // !GPUCA_ALIGPUCODE
   GPUd() MatBudget getMatBudget(float x0, float y0, float z0, float x1, float y1, float z1) const;
 
   GPUd() int searchSegment(float val, int low = -1, int high = -1) const;
 
+  /// searches a layer based on r2 input, using a lookup table
+  GPUd() int searchLayerFast(float r2, int low = -1, int high = -1) const;
+
 #ifndef GPUCA_GPUCODE
   //-----------------------------------------------------------
   std::size_t estimateFlatBufferSize() const;
@@ -114,6 +130,14 @@ class MatLayerCylSet : public o2::gpu::FlatObject
   static constexpr size_t getBufferAlignmentBytes() { return 8; }
 #endif // !GPUCA_GPUCODE
 
+  static constexpr float LayerRMax = 500;    // maximum value of R lookup (corresponds to last layer of MatLUT)
+  static constexpr float VoxelRDelta = 0.05; // voxel spacing for layer lookup; seems a natural choice - corresponding ~ to smallest spacing
+  static constexpr float InvVoxelRDelta = 1.f / VoxelRDelta;
+  static constexpr int NumVoxels = int(LayerRMax / VoxelRDelta);
+
+  uint16_t mLayerVoxelLU[2 * NumVoxels]; //! helper structure to lookup a layer based on known radius (static dimension for easy copy to GPU)
+  bool mInitializedLayerVoxelLU = false; //! if the voxels have been initialized
+
   ClassDefNV(MatLayerCylSet, 1);
 };
 
diff --git a/Detectors/Base/include/DetectorsBase/MaterialManager.h b/Detectors/Base/include/DetectorsBase/MaterialManager.h
index 96f64274d7a4d..b0de75c2d6c84 100644
--- a/Detectors/Base/include/DetectorsBase/MaterialManager.h
+++ b/Detectors/Base/include/DetectorsBase/MaterialManager.h
@@ -131,6 +131,9 @@ class MaterialManager
   /// Custom setting of process or cut given parameter name and value
   void SpecialCut(const char* modname, int localindex, ECut parID, Float_t val);
 
+  /// Set flag fro low energy neutron transport
+  void SetLowEnergyNeutronTransport(bool flag) { mLowNeut = flag; }
+
   /// load cuts and process flags from a data file (like AliRoot did)
   void loadCutsAndProcessesFromFile(const char* modname, const char* filename);
   void loadCutsAndProcessesFromJSON(ESpecial special = ESpecial::kFALSE, std::string const& filename = "");
@@ -215,7 +218,7 @@ class MaterialManager
   std::unordered_map mDensityMap;
 
   void initDensityMap();
-  float getDensity(std::string const& modname);
+  float getDensity(std::string const& modname, std::string const& matname);
 
   // Hide details by providing these private methods so it cannot happen that special settings
   // are applied as default settings by accident using a boolean flag
@@ -264,6 +267,8 @@ class MaterialManager
   /// Decide whether special process and cut settings should be applied
   bool mApplySpecialProcesses = true;
   bool mApplySpecialCuts = true;
+  /// Flag for low energy neutron transport
+  bool mLowNeut = false;
 
  public:
   ClassDefNV(MaterialManager, 0);
diff --git a/Detectors/Base/include/DetectorsBase/O2Tessellated.h b/Detectors/Base/include/DetectorsBase/O2Tessellated.h
new file mode 100644
index 0000000000000..0a1cee8b3e01f
--- /dev/null
+++ b/Detectors/Base/include/DetectorsBase/O2Tessellated.h
@@ -0,0 +1,142 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef ALICEO2_BASE_O2TESSELLATED_
+#define ALICEO2_BASE_O2TESSELLATED_
+
+#include "TGeoShape.h"
+#include "TGeoBBox.h"
+#include "TGeoVector3.h"
+#include "TGeoTypedefs.h"
+#include "TGeoTessellated.h"
+
+namespace o2
+{
+namespace base
+{
+
+class O2Tessellated : public TGeoBBox
+{
+
+ public:
+  using Vertex_t = Tessellated::Vertex_t;
+
+ private:
+  int fNfacets = 0;         // Number of facets
+  int fNvert = 0;           // Number of vertices
+  int fNseg = 0;            // Number of segments
+  bool fDefined = false;    //! Shape fully defined
+  bool fClosedBody = false; // The faces are making a closed body
+
+  // for now separate vectors but might be better to group per face
+  std::vector fVertices;       // List of vertices
+  std::vector fFacets;        // List of facets
+  std::vector fOutwardNormals; // Vector of outward-facing normals (to be streamed !)
+
+  std::multimap fVerticesMap; //! Temporary map used to deduplicate vertices
+  bool fIsClosed = false;                //! to know if shape still needs closure/initialization
+  void* fBVH = nullptr;                  //! BVH acceleration structure for safety and navigation
+
+  O2Tessellated(const O2Tessellated&) = delete;
+  O2Tessellated& operator=(const O2Tessellated&) = delete;
+
+  // bvh helper functions
+  void BuildBVH();
+  void CalculateNormals();
+
+ public:
+  // constructors
+  O2Tessellated() {}
+  O2Tessellated(const char* name, int nfacets = 0);
+  O2Tessellated(const char* name, const std::vector& vertices);
+  // from a TGeoTessellated
+  O2Tessellated(TGeoTessellated const&, bool check = false);
+
+  // destructor
+  ~O2Tessellated() override {}
+
+  void ComputeBBox() override;
+  void CloseShape(bool check = true, bool fixFlipped = true, bool verbose = true);
+
+  bool AddFacet(const Vertex_t& pt0, const Vertex_t& pt1, const Vertex_t& pt2);
+  bool AddFacet(const Vertex_t& pt0, const Vertex_t& pt1, const Vertex_t& pt2, const Vertex_t& pt3);
+  bool AddFacet(int i1, int i2, int i3);
+  bool AddFacet(int i1, int i2, int i3, int i4);
+  int AddVertex(const Vertex_t& vert);
+
+  bool FacetCheck(int ifacet) const;
+  Vertex_t FacetComputeNormal(int ifacet, bool& degenerated) const;
+
+  int GetNfacets() const { return fFacets.size(); }
+  int GetNsegments() const { return fNseg; }
+  int GetNvertices() const { return fNvert; }
+  bool IsClosedBody() const { return fClosedBody; }
+  bool IsDefined() const { return fDefined; }
+
+  const TGeoFacet& GetFacet(int i) const { return fFacets[i]; }
+  const Vertex_t& GetVertex(int i) const { return fVertices[i]; }
+
+  int DistancetoPrimitive(int, int) override { return 99999; }
+  const TBuffer3D& GetBuffer3D(int reqSections, Bool_t localFrame) const override;
+  void GetMeshNumbers(int& nvert, int& nsegs, int& npols) const override;
+  int GetNmeshVertices() const override { return fNvert; }
+  void InspectShape() const override {}
+  TBuffer3D* MakeBuffer3D() const override;
+  void Print(Option_t* option = "") const override;
+  void SavePrimitive(std::ostream&, Option_t*) override {}
+  void SetPoints(double* points) const override;
+  void SetPoints(Float_t* points) const override;
+  void SetSegsAndPols(TBuffer3D& buff) const override;
+  void Sizeof3D() const override {}
+
+  /// Resize and center the shape in a box of size maxsize
+  void ResizeCenter(double maxsize);
+
+  /// Flip all facets
+  void FlipFacets()
+  {
+    for (auto facet : fFacets)
+      facet.Flip();
+  }
+
+  bool CheckClosure(bool fixFlipped = true, bool verbose = true);
+
+  /// Reader from .obj format
+  static O2Tessellated* ImportFromObjFormat(const char* objfile, bool check = false, bool verbose = false);
+
+  // navigation functions used by TGeoNavigator (attention: only the iact == 3 cases implemented for now)
+  Double_t DistFromOutside(const Double_t* point, const Double_t* dir, Int_t iact = 1,
+                           Double_t step = TGeoShape::Big(), Double_t* safe = nullptr) const override;
+  Double_t DistFromInside(const Double_t* point, const Double_t* dir, Int_t iact = 1, Double_t step = TGeoShape::Big(),
+                          Double_t* safe = nullptr) const override;
+  bool Contains(const Double_t* point) const override;
+  Double_t Safety(const Double_t* point, Bool_t in = kTRUE) const override;
+  void ComputeNormal(const Double_t* point, const Double_t* dir, Double_t* norm) const override;
+
+  // these are trivial implementations, just for debugging
+  Double_t DistFromInside_Loop(const Double_t* point, const Double_t* dir) const;
+  Double_t DistFromOutside_Loop(const Double_t* point, const Double_t* dir) const;
+  bool Contains_Loop(const Double_t* point) const;
+
+  Double_t Capacity() const override;
+
+ private:
+  // a safety kernel used in multiple implementations
+  template 
+  Double_t SafetyKernel(const Double_t* point, bool in, int* closest_facet_id = nullptr) const;
+
+  ClassDefOverride(O2Tessellated, 1) // tessellated shape class
+};
+
+} // namespace base
+} // namespace o2
+
+#endif
diff --git a/Detectors/Base/include/DetectorsBase/Propagator.h b/Detectors/Base/include/DetectorsBase/Propagator.h
index c1fa57e41f99e..75b9446aebade 100644
--- a/Detectors/Base/include/DetectorsBase/Propagator.h
+++ b/Detectors/Base/include/DetectorsBase/Propagator.h
@@ -17,7 +17,6 @@
 #define ALICEO2_BASE_PROPAGATOR_
 
 #include "GPUCommonRtypes.h"
-#include "GPUCommonArray.h"
 #include "CommonConstants/PhysicsConstants.h"
 #include "ReconstructionDataFormats/Track.h"
 #include "ReconstructionDataFormats/DCA.h"
@@ -25,6 +24,7 @@
 #include "DetectorsBase/MatLayerCylSet.h"
 
 #ifndef GPUCA_GPUCODE
+#include 
 #include 
 #endif
 
@@ -34,7 +34,7 @@ namespace parameters
 {
 class GRPObject;
 class GRPMagField;
-}
+} // namespace parameters
 
 namespace dataformats
 {
@@ -45,7 +45,7 @@ namespace field
 {
 class MagFieldFast;
 class MagneticField;
-}
+} // namespace field
 
 namespace gpu
 {
@@ -76,6 +76,10 @@ class PropagatorImpl
                                  value_type maxSnp = MAX_SIN_PHI, value_type maxStep = MAX_STEP, MatCorrType matCorr = MatCorrType::USEMatCorrLUT,
                                  track::TrackLTIntegral* tofInfo = nullptr, int signCorr = 0) const;
 
+  GPUd() bool PropagateToXBxByBz(TrackParCov_t& track, TrackPar_t& linRef, value_type x,
+                                 value_type maxSnp = MAX_SIN_PHI, value_type maxStep = MAX_STEP, MatCorrType matCorr = MatCorrType::USEMatCorrLUT,
+                                 track::TrackLTIntegral* tofInfo = nullptr, int signCorr = 0) const;
+
   GPUd() bool PropagateToXBxByBz(TrackPar_t& track, value_type x,
                                  value_type maxSnp = MAX_SIN_PHI, value_type maxStep = MAX_STEP, MatCorrType matCorr = MatCorrType::USEMatCorrLUT,
                                  track::TrackLTIntegral* tofInfo = nullptr, int signCorr = 0) const;
@@ -84,6 +88,10 @@ class PropagatorImpl
                            value_type maxSnp = MAX_SIN_PHI, value_type maxStep = MAX_STEP, MatCorrType matCorr = MatCorrType::USEMatCorrLUT,
                            track::TrackLTIntegral* tofInfo = nullptr, int signCorr = 0) const;
 
+  GPUd() bool propagateToX(TrackParCov_t& track, TrackPar_t& linRef, value_type x, value_type bZ,
+                           value_type maxSnp = MAX_SIN_PHI, value_type maxStep = MAX_STEP, MatCorrType matCorr = MatCorrType::USEMatCorrLUT,
+                           track::TrackLTIntegral* tofInfo = nullptr, int signCorr = 0) const;
+
   GPUd() bool propagateToX(TrackPar_t& track, value_type x, value_type bZ,
                            value_type maxSnp = MAX_SIN_PHI, value_type maxStep = MAX_STEP, MatCorrType matCorr = MatCorrType::USEMatCorrLUT,
                            track::TrackLTIntegral* tofInfo = nullptr, int signCorr = 0) const;
@@ -92,9 +100,40 @@ class PropagatorImpl
   GPUd() bool propagateTo(track_T& track, value_type x, bool bzOnly = false, value_type maxSnp = MAX_SIN_PHI, value_type maxStep = MAX_STEP,
                           MatCorrType matCorr = MatCorrType::USEMatCorrLUT, track::TrackLTIntegral* tofInfo = nullptr, int signCorr = 0) const
   {
-    return bzOnly ? propagateToX(track, x, getNominalBz(), maxSnp, maxStep, matCorr, tofInfo, signCorr) : PropagateToXBxByBz(track, x, maxSnp, maxStep, matCorr, tofInfo, signCorr);
+    return bzOnly ? propagateToX(track, x, getBz(track.getXYZGlo()), maxSnp, maxStep, matCorr, tofInfo, signCorr) : PropagateToXBxByBz(track, x, maxSnp, maxStep, matCorr, tofInfo, signCorr);
   }
 
+  GPUd() bool propagateToX(TrackParCov_t& track, TrackPar_t* linRef, value_type x, value_type bZ,
+                           value_type maxSnp = MAX_SIN_PHI, value_type maxStep = MAX_STEP, MatCorrType matCorr = MatCorrType::USEMatCorrLUT,
+                           track::TrackLTIntegral* tofInfo = nullptr, int signCorr = 0) const
+  {
+    return linRef ? propagateToX(track, *linRef, x, bZ, maxSnp, maxStep, matCorr, tofInfo, signCorr) : propagateToX(track, x, bZ, maxSnp, maxStep, matCorr, tofInfo, signCorr);
+  }
+
+  GPUd() bool PropagateToXBxByBz(TrackParCov_t& track, TrackPar_t* linRef, value_type x,
+                                 value_type maxSnp = MAX_SIN_PHI, value_type maxStep = MAX_STEP, MatCorrType matCorr = MatCorrType::USEMatCorrLUT,
+                                 track::TrackLTIntegral* tofInfo = nullptr, int signCorr = 0) const
+  {
+    return linRef ? PropagateToXBxByBz(track, *linRef, x, maxSnp, maxStep, matCorr, tofInfo, signCorr) : PropagateToXBxByBz(track, x, maxSnp, maxStep, matCorr, tofInfo, signCorr);
+  }
+
+  GPUd() bool propagateTo(TrackParCov_t& track, TrackPar_t* linRef, value_type x, bool bzOnly = false, value_type maxSnp = MAX_SIN_PHI, value_type maxStep = MAX_STEP,
+                          MatCorrType matCorr = MatCorrType::USEMatCorrLUT, track::TrackLTIntegral* tofInfo = nullptr, int signCorr = 0) const
+  {
+    return bzOnly ? propagateToX(track, linRef, x, getBz(track.getXYZGlo()), maxSnp, maxStep, matCorr, tofInfo, signCorr) : PropagateToXBxByBz(track, linRef, x, maxSnp, maxStep, matCorr, tofInfo, signCorr);
+  }
+
+  template 
+  GPUd() bool propagateToAlphaX(track_T& track, value_type alpha, value_type x, bool bzOnly = false, value_type maxSnp = MAX_SIN_PHI, value_type maxStep = MAX_STEP, int minSteps = 1,
+                                MatCorrType matCorr = MatCorrType::USEMatCorrLUT, track::TrackLTIntegral* tofInfo = nullptr, int signCorr = 0) const;
+
+  GPUd() bool propagateToAlphaX(TrackParCov_t& track, TrackPar_t* linRef, value_type alpha, value_type x, bool bzOnly = false, value_type maxSnp = MAX_SIN_PHI, value_type maxStep = MAX_STEP, int minSteps = 1,
+                                MatCorrType matCorr = MatCorrType::USEMatCorrLUT, track::TrackLTIntegral* tofInfo = nullptr, int signCorr = 0) const;
+
+  template 
+  GPUd() bool propagateToR(track_T& track, value_type r, bool bzOnly = false, value_type maxSnp = MAX_SIN_PHI, value_type maxStep = MAX_STEP,
+                           MatCorrType matCorr = MatCorrType::USEMatCorrLUT, track::TrackLTIntegral* tofInfo = nullptr, int signCorr = 0) const;
+
   GPUd() bool propagateToDCA(const o2::dataformats::VertexBase& vtx, o2::track::TrackParametrizationWithError& track, value_type bZ,
                              value_type maxStep = MAX_STEP, MatCorrType matCorr = MatCorrType::USEMatCorrLUT,
                              o2::dataformats::DCA* dcaInfo = nullptr, track::TrackLTIntegral* tofInfo = nullptr,
@@ -107,12 +146,12 @@ class PropagatorImpl
 
   GPUd() bool propagateToDCA(const o2::math_utils::Point3D& vtx, o2::track::TrackParametrization& track, value_type bZ,
                              value_type maxStep = MAX_STEP, MatCorrType matCorr = MatCorrType::USEMatCorrLUT,
-                             gpu::gpustd::array* dca = nullptr, track::TrackLTIntegral* tofInfo = nullptr,
+                             std::array* dca = nullptr, track::TrackLTIntegral* tofInfo = nullptr,
                              int signCorr = 0, value_type maxD = 999.f) const;
 
   GPUd() bool propagateToDCABxByBz(const o2::math_utils::Point3D& vtx, o2::track::TrackParametrization& track,
                                    value_type maxStep = MAX_STEP, MatCorrType matCorr = MatCorrType::USEMatCorrLUT,
-                                   gpu::gpustd::array* dca = nullptr, track::TrackLTIntegral* tofInfo = nullptr,
+                                   std::array* dca = nullptr, track::TrackLTIntegral* tofInfo = nullptr,
                                    int signCorr = 0, value_type maxD = 999.f) const;
 
   PropagatorImpl(PropagatorImpl const&) = delete;
@@ -122,16 +161,18 @@ class PropagatorImpl
 
   // Bz at the origin
   GPUd() void updateField();
-  GPUd() value_type getNominalBz() const { return mBz; }
+  GPUd() value_type getNominalBz() const { return mNominalBz; }
   GPUd() void setTGeoFallBackAllowed(bool v) { mTGeoFallBackAllowed = v; }
   GPUd() bool isTGeoFallBackAllowed() const { return mTGeoFallBackAllowed; }
   GPUd() void setMatLUT(const o2::base::MatLayerCylSet* lut) { mMatLUT = lut; }
   GPUd() const o2::base::MatLayerCylSet* getMatLUT() const { return mMatLUT; }
   GPUd() void setGPUField(const o2::gpu::GPUTPCGMPolynomialField* field) { mGPUField = field; }
   GPUd() const o2::gpu::GPUTPCGMPolynomialField* getGPUField() const { return mGPUField; }
-  GPUd() void setBz(value_type bz) { mBz = bz; }
+  GPUd() void setNominalBz(value_type bz) { mNominalBz = bz; }
+  GPUd() bool hasMagFieldSet() const { return mField != nullptr; }
 
-  GPUd() void estimateLTFast(o2::track::TrackLTIntegral& lt, const o2::track::TrackParametrization& trc) const;
+  GPUd() value_type estimateLTFast(o2::track::TrackLTIntegral& lt, const o2::track::TrackParametrization& trc) const;
+  GPUd() float estimateLTIncrement(const o2::track::TrackParametrization& trc, const o2::math_utils::Point3D& postStart, const o2::math_utils::Point3D& posEnd) const;
 
 #ifndef GPUCA_GPUCODE
   static PropagatorImpl* Instance(bool uninitialized = false)
@@ -151,18 +192,24 @@ class PropagatorImpl
 
   GPUd() void getFieldXYZ(const math_utils::Point3D xyz, double* bxyz) const;
 
+  GPUd() float getBz(const math_utils::Point3D xyz) const;
+
+  GPUd() double getBz(const math_utils::Point3D xyz) const;
+
  private:
 #ifndef GPUCA_GPUCODE
   PropagatorImpl(bool uninitialized = false);
   ~PropagatorImpl() = default;
 #endif
-
+  static constexpr value_type Epsilon = 0.00001; // precision of propagation to X
   template 
   GPUd() void getFieldXYZImpl(const math_utils::Point3D xyz, T* bxyz) const;
+  template 
+  GPUd() T getBzImpl(const math_utils::Point3D xyz) const;
 
   const o2::field::MagFieldFast* mFieldFast = nullptr; ///< External fast field map (barrel only for the moment)
   o2::field::MagneticField* mField = nullptr;          ///< External nominal field map
-  value_type mBz = 0;                                  ///< nominal field
+  value_type mNominalBz = 0;                           ///< nominal field
 
   bool mTGeoFallBackAllowed = true;                            ///< allow fall back to TGeo if requested MatLUT is not available
   const o2::base::MatLayerCylSet* mMatLUT = nullptr;           // externally set LUT
diff --git a/Detectors/Base/include/DetectorsBase/Ray.h b/Detectors/Base/include/DetectorsBase/Ray.h
index 1701e65c2d638..a72208c41af0d 100644
--- a/Detectors/Base/include/DetectorsBase/Ray.h
+++ b/Detectors/Base/include/DetectorsBase/Ray.h
@@ -49,7 +49,7 @@ class Ray
   GPUd() Ray() : mP{0.f}, mD{0.f}, mDistXY2(0.f), mDistXY2i(0.f), mDistXYZ(0.f), mXDxPlusYDy(0.f), mXDxPlusYDyRed(0.f), mXDxPlusYDy2(0.f), mR02(0.f), mR12(0.f)
   {
   }
-  GPUdDefault() ~Ray() CON_DEFAULT;
+  GPUdDefault() ~Ray() = default;
 
 #ifndef GPUCA_ALIGPUCODE // this part is unvisible on GPU version
   Ray(const math_utils::Point3D point0, const math_utils::Point3D point1);
@@ -74,8 +74,8 @@ class Ray
   GPUd() float getDist() const { return mDistXYZ; }
   GPUd() float getDist(float deltaT) const { return mDistXYZ * (deltaT > 0 ? deltaT : -deltaT); }
 
-  // for debud only
-  float getPos(float t, int i) const { return mP[i] + t * mD[i]; }
+  // for debug only
+  GPUd() float getPos(float t, int i) const { return mP[i] + t * mD[i]; }
 
   GPUd() float getPhi(float t) const
   {
diff --git a/Detectors/Base/include/DetectorsBase/SimFieldUtils.h b/Detectors/Base/include/DetectorsBase/SimFieldUtils.h
index 2371f41a763d2..0386a1c421d59 100644
--- a/Detectors/Base/include/DetectorsBase/SimFieldUtils.h
+++ b/Detectors/Base/include/DetectorsBase/SimFieldUtils.h
@@ -15,12 +15,15 @@
 #ifndef ALICEO2_BASE_SIMFIELDUTILS_H_
 #define ALICEO2_BASE_SIMFIELDUTILS_H_
 
+class FairField;
+
 namespace o2
 {
-namespace field
-{
-class MagneticField;
-}
+// namespace field
+
+//{
+// class MagneticField;
+//}
 
 namespace base
 {
@@ -30,7 +33,7 @@ class SimFieldUtils
  public:
   // a common entry point to create the mag field for simulation
   // based on the simulation configuration in SimConfig
-  static o2::field::MagneticField* const createMagField();
+  static FairField* const createMagField();
 };
 
 } // namespace base
diff --git a/DataFormats/simulation/include/SimulationDataFormat/Stack.h b/Detectors/Base/include/DetectorsBase/Stack.h
similarity index 95%
rename from DataFormats/simulation/include/SimulationDataFormat/Stack.h
rename to Detectors/Base/include/DetectorsBase/Stack.h
index d6bda808830ec..69d221000e493 100644
--- a/DataFormats/simulation/include/SimulationDataFormat/Stack.h
+++ b/Detectors/Base/include/DetectorsBase/Stack.h
@@ -127,6 +127,7 @@ class Stack : public FairGenericStack
     // the const cast is necessary ... the interface should have been `const TParticle* GetCurrentParticle() const`
     return const_cast(&mCurrentParticle);
   }
+
   /// Get the number of the current track
   /// Declared in TVirtualMCStack
   Int_t GetCurrentTrackNumber() const override { return mIndexOfCurrentTrack; }
@@ -227,6 +228,15 @@ class Stack : public FairGenericStack
   /// update values in the current event header
   void updateEventStats();
 
+  /// returns if current track left a hit
+  bool currentTrackLeftHit() const { return mCurrentParticle.TestBit(ParticleStatus::kHasHits); }
+
+  /// returns if current track left a TrackReference
+  bool currentTrackLeftTrackRef() const { return mCurrentParticle.TestBit(ParticleStatus::kHasTrackRefs); }
+
+  /// set track seeding mode (tracks are seeded individually for increased reproducability)
+  void setTrackSeedingMode(bool m) { mDoTrackSeeding = m; }
+
   typedef std::function& particles)> TransportFcn;
 
  private:
@@ -248,7 +258,7 @@ class Stack : public FairGenericStack
   // (mainly for the PopPrimaryParticleInterface
   std::vector mPrimaryParticles;
 
-  /// vector of reducded tracks written to the output
+  /// vector of reduced/pruned tracks written to the output
   std::vector* mTracks;
 
   /// STL map from particle index to persistent track index
@@ -284,6 +294,7 @@ class Stack : public FairGenericStack
   bool mIsG4Like = false; //! flag indicating if the stack is used in a manner done by Geant4
 
   bool mIsExternalMode = false; // is stack an external factory or directly used inside simulation?
+  bool mDoTrackSeeding = false; // whether to do track based seeding
 
   TransportFcn mTransportPrimary = [](const TParticle& p, const std::vector& particles) { return false; }; //! a function to inhibit the tracking of a particle
 
@@ -325,6 +336,7 @@ inline void Stack::addTrackReference(const o2::TrackReference& ref)
     auto& part = mParticles[iTrack];
     part.setStore(true);
   }
+  mCurrentParticle.SetBit(ParticleStatus::kHasTrackRefs, 1); // mark that this particle has track refs
   mTrackRefs->push_back(ref);
 }
 
diff --git a/Detectors/Base/include/DetectorsBase/TGeoGeometryUtils.h b/Detectors/Base/include/DetectorsBase/TGeoGeometryUtils.h
new file mode 100644
index 0000000000000..5ec85f1c14702
--- /dev/null
+++ b/Detectors/Base/include/DetectorsBase/TGeoGeometryUtils.h
@@ -0,0 +1,38 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file TGeoGeometryUtils.h
+/// \author Sandro Wenzel (CERN)
+/// \brief Collection of utility functions for TGeo
+
+#ifndef ALICEO2_BASE_TGEOGEOMETRYUTILS_H_
+#define ALICEO2_BASE_TGEOGEOMETRYUTILS_H_
+
+class TGeoShape;
+class TGeoTessellated;
+
+namespace o2
+{
+namespace base
+{
+
+/// A few utility functions to operate on TGeo geometries (transformations, printing, ...)
+class TGeoGeometryUtils
+{
+ public:
+  ///< Transform any (primitive) TGeoShape to a tessellated representation
+  static TGeoTessellated* TGeoShapeToTGeoTessellated(TGeoShape const*);
+};
+
+} // namespace base
+} // namespace o2
+
+#endif
diff --git a/Detectors/Base/include/DetectorsBase/VMCSeederService.h b/Detectors/Base/include/DetectorsBase/VMCSeederService.h
new file mode 100644
index 0000000000000..1669c73b39620
--- /dev/null
+++ b/Detectors/Base/include/DetectorsBase/VMCSeederService.h
@@ -0,0 +1,50 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef ALICEO2_VMC_SEEDERSERVICE_H
+#define ALICEO2_VMC_SEEDERSERVICE_H
+
+class TVirtualMC;
+
+#include 
+
+namespace o2
+{
+namespace base
+{
+
+// A simple (singleton) service to propagate random number seeding
+// to VMC engines in a loosely-coupled, just-in-time way (without needing to compile against these libraries).
+class VMCSeederService
+{
+
+ public:
+  static VMCSeederService const& instance()
+  {
+    static VMCSeederService inst;
+    return inst;
+  }
+
+  void setSeed() const; // will propagate seed to the VMC engines
+
+  typedef std::function SeederFcn;
+
+ private:
+  VMCSeederService();
+  void initSeederFunction(TVirtualMC const*);
+
+  SeederFcn mSeederFcn; // the just-in-time compiled function talking to the VMC engines
+};
+
+} // namespace base
+} // namespace o2
+
+#endif
diff --git a/Detectors/Base/src/BaseDPLDigitizer.cxx b/Detectors/Base/src/BaseDPLDigitizer.cxx
index 081c4edc8a163..3b3c5cfd477cb 100644
--- a/Detectors/Base/src/BaseDPLDigitizer.cxx
+++ b/Detectors/Base/src/BaseDPLDigitizer.cxx
@@ -14,8 +14,9 @@
 #include 
 #include 
 #include 
-#include 
+#include 
 #include 
+#include 
 
 using namespace o2::base;
 
@@ -54,6 +55,10 @@ void BaseDPLDigitizer::init(o2::framework::InitContext& ic)
     }
   }
 
+  // initialize the global ROOT random number generator (needed or not)
+  LOG(info) << "Initializing ROOT digitizer random with seed " << o2::conf::DigiParams::Instance().seed;
+  gRandom->SetSeed(o2::conf::DigiParams::Instance().seed);
+
   // finally call specific init
   this->initDigitizerTask(ic);
 }
diff --git a/Detectors/Base/src/CTFCoderBase.cxx b/Detectors/Base/src/CTFCoderBase.cxx
index aec48e8763e24..e29ca9e6d1afe 100644
--- a/Detectors/Base/src/CTFCoderBase.cxx
+++ b/Detectors/Base/src/CTFCoderBase.cxx
@@ -17,6 +17,7 @@
 #include "Framework/ControlService.h"
 #include "Framework/ProcessingContext.h"
 #include "Framework/InputRecord.h"
+#include "Framework/TimingInfo.h"
 
 using namespace o2::ctf;
 using namespace o2::framework;
@@ -45,9 +46,52 @@ void CTFCoderBase::assignDictVersion(CTFDictHeader& h) const
   //  }
 }
 
-void CTFCoderBase::updateTimeDependentParams(ProcessingContext& pc)
+void CTFCoderBase::updateTimeDependentParams(ProcessingContext& pc, bool askTree)
 {
-  if (mLoadDictFromCCDB) {
-    pc.inputs().get*>("ctfdict"); // just to trigger the finaliseCCDB
+  setFirstTFOrbit(pc.services().get().firstTForbit);
+  if (pc.services().get().globalRunNumberChanged) { // this params need to be queried only once
+    if (mOpType == OpType::Decoder) {
+      pc.inputs().get(mTrigOffsBinding); // this is a configurable param
+    }
+    if (mLoadDictFromCCDB) {
+      if (askTree) {
+        pc.inputs().get(mDictBinding); // just to trigger the finaliseCCDB
+      } else {
+        pc.inputs().get*>(mDictBinding); // just to trigger the finaliseCCDB
+      }
+    }
+  }
+}
+
+bool CTFCoderBase::isTreeDictionary(const void* buff) const
+{
+  // heuristic check for the dictionary being a tree
+  const char* patt[] = {"ccdb_object", "ctf_dictionary"};
+  const char* ptr = reinterpret_cast(buff);
+  bool found = false;
+  int i = 0, np = sizeof(patt) / sizeof(char*);
+  while (i < 50 && !found) {
+    for (int ip = 0; ip < np; ip++) {
+      const auto *p = patt[ip], *s = &ptr[i];
+      while (*p && *s == *p) {
+        p++;
+        s++;
+      }
+      if (!*p) {
+        found = true;
+        break;
+      }
+    }
+    i++;
+  }
+  return found;
+}
+
+void CTFCoderBase::reportIRFrames()
+{
+  static bool repDone = false;
+  if (!repDone) {
+    LOGP(info, "IRFrames will be selected with shift {}, forward {} margin and backward {} margin (in BCs)", mIRFrameSelShift, mIRFrameSelMarginBwd, mIRFrameSelMarginFwd);
+    repDone = true;
   }
 }
diff --git a/Detectors/Base/src/Detector.cxx b/Detectors/Base/src/Detector.cxx
index e93295f1fc3f6..d2be9237f6f13 100644
--- a/Detectors/Base/src/Detector.cxx
+++ b/Detectors/Base/src/Detector.cxx
@@ -17,6 +17,7 @@
 #include "DetectorsBase/MaterialManager.h"
 #include "DetectorsCommonDataFormats/DetID.h"
 #include "Field/MagneticField.h"
+#include "Framework/TMessageSerializer.h"
 #include "TString.h" // for TString
 #include "TGeoManager.h"
 
@@ -30,6 +31,7 @@ using namespace o2::base;
 using namespace o2::detectors;
 
 Float_t Detector::mDensityFactor = 1.0;
+std::vector o2::base::Detector::sDetId2HitBitIndex{}; // initialize empty vector
 
 Detector::Detector() : FairDetector(), mMapMaterial(), mMapMedium() {}
 Detector::Detector(const char* name, Bool_t Active)
@@ -169,6 +171,10 @@ void Detector::addAlignableVolumes() const
   LOG(warning) << "Alignable volumes are not yet defined for " << GetName();
 }
 
+void Detector::fillParallelWorld() const
+{
+}
+
 int Detector::registerSensitiveVolumeAndGetVolID(TGeoVolume const* vol)
 {
   // register this volume with FairRoot
@@ -195,16 +201,19 @@ int Detector::registerSensitiveVolumeAndGetVolID(std::string const& name)
 #include 
 #include 
 #include 
-namespace o2
-{
-namespace base
+namespace o2::base
 {
 // this goes into the source
-void attachMessageBufferToParts(fair::mq::Parts& parts, fair::mq::Channel& channel, void* data, size_t size,
-                                void (*free_func)(void* data, void* hint), void* hint)
+void attachMessageBufferToParts(fair::mq::Parts& parts, fair::mq::Channel& channel, void* data, TClass* cl)
 {
-  std::unique_ptr message(channel.NewMessage(data, size, free_func, hint));
-  parts.AddPart(std::move(message));
+  auto msg = channel.Transport()->CreateMessage(4096, fair::mq::Alignment{64});
+  // This will serialize the data directly into the message buffer, without any further
+  // buffer or copying. Notice how the message will have 8 bytes of header and then
+  // the serialized data as TBufferFile. In principle one could construct a serialized TMessage payload
+  // however I did not manage to get it to work for every case.
+  o2::framework::FairOutputTBuffer buffer(*msg);
+  o2::framework::TMessageSerializer::serialize(buffer, data, cl);
+  parts.AddPart(std::move(msg));
 }
 void attachDetIDHeaderMessage(int id, fair::mq::Channel& channel, fair::mq::Parts& parts)
 {
@@ -245,17 +254,14 @@ void* decodeShmCore(fair::mq::Parts& dataparts, int index, bool*& busy)
 
 void* decodeTMessageCore(fair::mq::Parts& dataparts, int index)
 {
-  class TMessageWrapper : public TMessage
-  {
-   public:
-    TMessageWrapper(void* buf, Int_t len) : TMessage(buf, len) { ResetBit(kIsOwner); }
-    ~TMessageWrapper() override = default;
-  };
   auto rawmessage = std::move(dataparts.At(index));
-  auto message = std::make_unique(rawmessage->GetData(), rawmessage->GetSize());
-  return message.get()->ReadObjectAny(message.get()->GetClass());
+  o2::framework::FairInputTBuffer buffer((char*)rawmessage->GetData(), rawmessage->GetSize());
+  buffer.InitMap();
+  auto* cl = buffer.ReadClass();
+  buffer.SetBufferOffset(0);
+  buffer.ResetMap();
+  return buffer.ReadObjectAny(cl);
 }
 
-} // namespace base
-} // namespace o2
+} // namespace o2::base
 ClassImp(o2::base::Detector);
diff --git a/Detectors/Base/src/DetectorsBaseLinkDef.h b/Detectors/Base/src/DetectorsBaseLinkDef.h
index 20495e08ecc6f..8255c143ebb4a 100644
--- a/Detectors/Base/src/DetectorsBaseLinkDef.h
+++ b/Detectors/Base/src/DetectorsBaseLinkDef.h
@@ -26,6 +26,7 @@
 #pragma link C++ class o2::base::GeometryManager::MatBudgetExt + ;
 #pragma link C++ class o2::base::MaterialManager + ;
 #pragma link C++ class o2::MaterialManagerParam + ;
+#pragma link C++ class o2::GeometryManagerParam + ;
 #pragma link C++ class o2::base::SimFieldUtils + ;
 
 #pragma link C++ class o2::base::Ray + ;
@@ -36,4 +37,11 @@
 #pragma link C++ class o2::base::Aligner + ;
 #pragma link C++ class o2::conf::ConfigurableParamHelper < o2::base::Aligner> + ;
 
+#pragma link C++ class o2::GlobalParams + ;
+#pragma link C++ class o2::conf::ConfigurableParamHelper < o2::GlobalParams> + ;
+
+#pragma link C++ class o2::data::Stack + ;
+
+#pragma link C++ class o2::base::O2Tessellated - ;
+
 #endif
diff --git a/Detectors/Base/src/GRPGeomHelper.cxx b/Detectors/Base/src/GRPGeomHelper.cxx
index da54dc3efbfcc..e7e5248493548 100644
--- a/Detectors/Base/src/GRPGeomHelper.cxx
+++ b/Detectors/Base/src/GRPGeomHelper.cxx
@@ -22,6 +22,7 @@
 #include "Framework/DataRefUtils.h"
 #include "Framework/InputSpec.h"
 #include "Framework/InputRecord.h"
+#include "Framework/TimingInfo.h"
 #include "Framework/CCDBParamSpec.h"
 #include "DetectorsBase/MatLayerCylSet.h"
 #include "DetectorsBase/Propagator.h"
@@ -35,21 +36,24 @@ using namespace o2::base;
 using namespace o2::framework;
 namespace o2d = o2::dataformats;
 
-GRPGeomRequest::GRPGeomRequest(bool orbitResetTime, bool GRPECS, bool GRPLHCIF, bool GRPMagField, bool askMatLUT, GeomRequest geom, std::vector& inputs, bool askOnce)
-  : askGRPECS(GRPECS), askGRPLHCIF(GRPLHCIF), askGRPMagField(GRPMagField), askMatLUT(askMatLUT), askTime(orbitResetTime), askOnceAllButField(askOnce)
+GRPGeomRequest::GRPGeomRequest(bool orbitResetTime, bool GRPECS, bool GRPLHCIF, bool GRPMagField, bool askMatLUT, GeomRequest geom, std::vector& inputs, bool askOnce, bool needPropD, std::string detMaskString)
+  : askGRPECS(GRPECS), askGRPLHCIF(GRPLHCIF), askGRPMagField(GRPMagField), askMatLUT(askMatLUT), askTime(orbitResetTime), askOnceAllButField(askOnce), needPropagatorD(needPropD)
 {
   if (geom == Aligned) {
     askGeomAlign = true;
-    addInput({"geomAlp", "GLO", "GEOMALIGN", 0, Lifetime::Condition, ccdbParamSpec("GLO/Config/GeometryAligned")}, inputs);
+    addInput({"geomAlg", "GLO", "GEOMALIGN", 0, Lifetime::Condition, ccdbParamSpec("GLO/Config/GeometryAligned")}, inputs);
   } else if (geom == Ideal || geom == Alignments) {
     askGeomIdeal = true;
     addInput({"geomIdeal", "GLO", "GEOMIDEAL", 0, Lifetime::Condition, ccdbParamSpec("GLO/Config/Geometry")}, inputs);
   }
   if (geom == Alignments) {
     askAlignments = true;
+    o2::detectors::DetID::mask_t algDetMask = DetID::getMask(detMaskString);
     for (auto id = DetID::First; id <= DetID::Last; id++) {
-      std::string binding = fmt::format("align{}", DetID::getName(id));
-      addInput({binding, DetID::getDataOrigin(id), "ALIGNMENT", 0, Lifetime::Condition, ccdbParamSpec(fmt::format("{}/Calib/Align", DetID::getName(id)))}, inputs);
+      if (algDetMask[id]) {
+        std::string binding = fmt::format("align{}", DetID::getName(id));
+        addInput({binding, DetID::getDataOrigin(id), "ALIGNMENT", 0, Lifetime::Condition, ccdbParamSpec(fmt::format("{}/Calib/Align", DetID::getName(id)))}, inputs);
+      }
     }
   }
   if (askMatLUT) {
@@ -59,7 +63,7 @@ GRPGeomRequest::GRPGeomRequest(bool orbitResetTime, bool GRPECS, bool GRPLHCIF,
     addInput({"orbitReset", "CTP", "ORBITRESET", 0, Lifetime::Condition, ccdbParamSpec("CTP/Calib/OrbitReset")}, inputs);
   }
   if (askGRPECS) {
-    addInput({"grpecs", "GLO", "GRPECS", 0, Lifetime::Condition, ccdbParamSpec("GLO/Config/GRPECS", true)}, inputs); // Run dependent !!!
+    addInput({"grpecs", "GLO", "GRPECS", 0, Lifetime::Condition, ccdbParamSpec("GLO/Config/GRPECS", 1)}, inputs); // Run dependent !!!
   }
   if (askGRPLHCIF) {
     addInput({"grplhcif", "GLO", "GRPLHCIF", 0, Lifetime::Condition, ccdbParamSpec("GLO/Config/GRPLHCIF")}, inputs);
@@ -69,6 +73,22 @@ GRPGeomRequest::GRPGeomRequest(bool orbitResetTime, bool GRPECS, bool GRPLHCIF,
   }
 }
 
+void GRPGeomRequest::requireAggregateRunInfo(std::vector& inputs)
+{
+  askAggregateRunInfo = true;
+  // require dependencies
+  if (!askGRPECS) {
+    askGRPECS = true;
+    addInput({"grpecs", "GLO", "GRPECS", 0, Lifetime::Condition, ccdbParamSpec("GLO/Config/GRPECS", 1)}, inputs);
+  }
+  if (!askTime) {
+    askTime = true;
+    addInput({"orbitReset", "CTP", "ORBITRESET", 0, Lifetime::Condition, ccdbParamSpec("CTP/Calib/OrbitReset")}, inputs);
+  }
+  addInput({"RCTRunInfo", "RCT", "RunInfo", 0, Lifetime::Condition, ccdbParamSpec("RCT/Info/RunInformation", 2)}, inputs);
+  addInput({"CTPRunOrbit", "CTP", "RunOrbit", 0, Lifetime::Condition, ccdbParamSpec("CTP/Calib/FirstRunOrbit")}, inputs); // TODO: should become run-depenendent (1) object
+}
+
 void GRPGeomRequest::addInput(const o2::framework::InputSpec&& isp, std::vector& inputs)
 {
   if (std::find(inputs.begin(), inputs.end(), isp) == inputs.end()) {
@@ -94,12 +114,18 @@ bool GRPGeomHelper::finaliseCCDB(ConcreteDataMatcher& matcher, void* obj)
     LOG(info) << "GRP MagField object updated";
     if (needInit) {
       o2::base::Propagator::initFieldFromGRP(mGRPMagField);
+      if (mRequest->needPropagatorD) {
+        o2::base::PropagatorD::initFieldFromGRP(mGRPMagField);
+      }
     } else {
       auto field = TGeoGlobalMagField::Instance()->GetField();
       if (field->InheritsFrom("o2::field::MagneticField")) {
         ((o2::field::MagneticField*)field)->rescaleField(mGRPMagField->getL3Current(), mGRPMagField->getDipoleCurrent(), mGRPMagField->getFieldUniformity());
       }
       o2::base::Propagator::Instance(false)->updateField();
+      if (mRequest->needPropagatorD) {
+        o2::base::PropagatorD::Instance(false)->updateField();
+      }
     }
     return true;
   }
@@ -114,14 +140,17 @@ bool GRPGeomHelper::finaliseCCDB(ConcreteDataMatcher& matcher, void* obj)
     return true;
   }
   if (mRequest->askTime && matcher == ConcreteDataMatcher("CTP", "ORBITRESET", 0)) {
-    mOrbitResetTimeMS = (*(std::vector*)obj)[0] / 1000;
-    LOG(info) << "orbit reset time updated to " << mOrbitResetTimeMS;
+    mOrbitResetTimeMUS = (*(std::vector*)obj)[0];
+    LOG(info) << "orbit reset time updated to " << mOrbitResetTimeMUS;
     return true;
   }
   if (mRequest->askMatLUT && matcher == ConcreteDataMatcher("GLO", "MATLUT", 0)) {
     LOG(info) << "material LUT updated";
     mMatLUT = o2::base::MatLayerCylSet::rectifyPtrFromFile((o2::base::MatLayerCylSet*)obj);
     o2::base::Propagator::Instance(false)->setMatLUT(mMatLUT);
+    if (mRequest->needPropagatorD) {
+      o2::base::PropagatorD::Instance(false)->setMatLUT(mMatLUT);
+    }
     return true;
   }
   if (mRequest->askGeomAlign && matcher == ConcreteDataMatcher("GLO", "GEOMALIGN", 0)) {
@@ -146,38 +175,80 @@ bool GRPGeomHelper::finaliseCCDB(ConcreteDataMatcher& matcher, void* obj)
   return false;
 }
 
-void GRPGeomHelper::checkUpdates(ProcessingContext& pc) const
+void GRPGeomHelper::checkUpdates(ProcessingContext& pc)
 {
   // request input just to trigger finaliseCCDB if there was an update
-  static bool initOnceDone = false;
-  if (mRequest->askGRPMagField) {
-    pc.inputs().get("grpfield");
-  }
-  if (mRequest->askGRPLHCIF && !initOnceDone) {
-    pc.inputs().get("grplhcif");
-  }
-  if (mRequest->askGRPECS && !initOnceDone) {
-    pc.inputs().get("grpecs");
-  }
-  if (mRequest->askTime && !initOnceDone) {
-    pc.inputs().get*>("orbitReset");
-  }
-  if (mRequest->askMatLUT && !initOnceDone) {
-    pc.inputs().get("matLUT");
-  }
-  if (mRequest->askGeomAlign && !initOnceDone) {
-    pc.inputs().get("geomAlp");
-  } else if (mRequest->askGeomIdeal) {
-    pc.inputs().get("geomIdeal");
-  }
-  if (mRequest->askAlignments && !initOnceDone) {
-    for (auto id = DetID::First; id <= DetID::Last; id++) {
-      std::string binding = fmt::format("align{}", DetID::getName(id));
-      pc.inputs().get*>(binding);
+
+  if (mRequest->askGRPMagField) { // always check
+    if (pc.inputs().isValid("grpfield")) {
+      pc.inputs().get("grpfield");
+    } else {
+      return;
     }
   }
-  if (!initOnceDone) {
-    initOnceDone = mRequest->askOnceAllButField;
+
+  bool checkTF = pc.services().get().globalRunNumberChanged || !mRequest->askOnceAllButField;
+
+  if (checkTF) {
+    if (mRequest->askGRPLHCIF) {
+      if (pc.inputs().isValid("grplhcif")) {
+        pc.inputs().get("grplhcif");
+      } else {
+        return;
+      }
+    }
+    if (mRequest->askGRPECS) {
+      if (pc.inputs().isValid("grpecs")) {
+        pc.inputs().get("grpecs");
+      } else {
+        return;
+      }
+    }
+    if (mRequest->askTime) {
+      if (pc.inputs().isValid("orbitReset")) {
+        pc.inputs().get*>("orbitReset");
+      } else {
+        return;
+      }
+    }
+    if (mRequest->askMatLUT) {
+      if (pc.inputs().isValid("matLUT")) {
+        pc.inputs().get("matLUT");
+      } else {
+        return;
+      }
+    }
+    if (mRequest->askGeomAlign) {
+      if (pc.inputs().isValid("geomAlg")) {
+        pc.inputs().get("geomAlg");
+      } else {
+        return;
+      }
+    } else if (mRequest->askGeomIdeal) {
+      if (pc.inputs().isValid("geomIdeal")) {
+        pc.inputs().get("geomIdeal");
+      } else {
+        return;
+      }
+    }
+    if (mRequest->askAlignments) {
+      for (auto id = DetID::First; id <= DetID::Last; id++) {
+        std::string binding = fmt::format("align{}", DetID::getName(id));
+        if (pc.inputs().getPos(binding.c_str()) < 0) {
+          continue;
+        } else {
+          pc.inputs().get*>(binding);
+        }
+      }
+    }
+    if (mRequest->askAggregateRunInfo) {
+      const auto hmap = pc.inputs().get("RCTRunInfo"); // metadata only!
+      auto rl = o2::ccdb::BasicCCDBManager::getRunDuration(hmap);
+      auto ctfFirstRunOrbitVec = pc.inputs().get*>("CTPRunOrbit");
+      mAggregatedRunInfo = o2::parameters::AggregatedRunInfo::buildAggregatedRunInfo(pc.services().get().runNumber, rl.first, rl.second, mOrbitResetTimeMUS, mGRPECS, ctfFirstRunOrbitVec.get());
+      LOGP(debug, "Extracted AggregateRunInfo: runNumber:{}, sor:{}, eor:{}, orbitsPerTF:{}, orbitReset:{}, orbitSOR:{}, orbitEOR:{}",
+           mAggregatedRunInfo.runNumber, mAggregatedRunInfo.sor, mAggregatedRunInfo.eor, mAggregatedRunInfo.orbitsPerTF, mAggregatedRunInfo.orbitReset, mAggregatedRunInfo.orbitSOR, mAggregatedRunInfo.orbitEOR);
+    }
   }
 }
 
diff --git a/Detectors/Base/src/GeometryManager.cxx b/Detectors/Base/src/GeometryManager.cxx
index 300b5a2709868..a067767752a69 100644
--- a/Detectors/Base/src/GeometryManager.cxx
+++ b/Detectors/Base/src/GeometryManager.cxx
@@ -12,8 +12,8 @@
 /// \file GeometryManager.cxx
 /// \brief Implementation of the GeometryManager class
 
-#include   // for LOG
-#include  // for TIter
+#include  // for LOG
+#include        // for TIter
 #include 
 #include        // for TGeoHMatrix
 #include          // for TGeoNode
@@ -24,6 +24,7 @@
 #include 
 
 #include "DetectorsBase/GeometryManager.h"
+#include "DetectorsBase/GeometryManagerParam.h"
 #include "DetectorsCommonDataFormats/AlignParam.h"
 #include "CommonUtils/NameConf.h"
 #include "DetectorsBase/Aligner.h"
@@ -252,11 +253,11 @@ bool GeometryManager::applyAlignment(const std::vector ord(nvols);
   std::iota(std::begin(ord), std::end(ord), 0); // sort to apply alignment in correct hierarchy
-  std::sort(std::begin(ord), std::end(ord), [&algPars](int a, int b) { return algPars[a].getLevel() > algPars[b].getLevel(); });
+  std::sort(std::begin(ord), std::end(ord), [&algPars](int a, int b) { return algPars[a].getLevel() < algPars[b].getLevel(); });
 
   bool res = true;
   for (int i = 0; i < nvols; i++) {
-    if (!algPars[ord[i]].applyToGeometry()) {
+    if (!algPars[ord[i]].applyToGeometry(GeometryManagerParam::Instance().printLevel)) {
       res = false;
       LOG(error) << "Error applying alignment object for volume" << algPars[ord[i]].getSymName();
     }
diff --git a/Detectors/Base/src/GeometryManagerParam.cxx b/Detectors/Base/src/GeometryManagerParam.cxx
new file mode 100644
index 0000000000000..e3c3ff3d687a7
--- /dev/null
+++ b/Detectors/Base/src/GeometryManagerParam.cxx
@@ -0,0 +1,13 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "DetectorsBase/GeometryManagerParam.h"
+O2ParamImpl(o2::GeometryManagerParam);
diff --git a/Detectors/Base/src/GlobalParams.cxx b/Detectors/Base/src/GlobalParams.cxx
new file mode 100644
index 0000000000000..b399a7d009c64
--- /dev/null
+++ b/Detectors/Base/src/GlobalParams.cxx
@@ -0,0 +1,13 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "DetectorsBase/GlobalParams.h"
+O2ParamImpl(o2::GlobalParams);
diff --git a/Detectors/Base/src/MatLayerCyl.cxx b/Detectors/Base/src/MatLayerCyl.cxx
index 1059074666952..29bed111b3584 100644
--- a/Detectors/Base/src/MatLayerCyl.cxx
+++ b/Detectors/Base/src/MatLayerCyl.cxx
@@ -23,10 +23,12 @@
 using namespace o2::base;
 using flatObject = o2::gpu::FlatObject;
 
+#ifndef GPUCA_GPUCODE
 //________________________________________________________________________________
 MatLayerCyl::MatLayerCyl() : mNZBins(0), mNPhiBins(0), mNPhiSlices(0), mZHalf(0.f), mRMin2(0.f), mRMax2(0.f), mDZ(0.f), mDZInv(0.f), mDPhi(0.f), mDPhiInv(0.f), mPhiBin2Slice(nullptr), mSliceCos(nullptr), mSliceSin(nullptr), mCells(nullptr)
 {
 }
+#endif
 
 #ifndef GPUCA_ALIGPUCODE // this part is unvisible on GPU version
 //________________________________________________________________________________
@@ -97,8 +99,8 @@ void MatLayerCyl::initSegmentation(float rMin, float rMax, float zHalfSpan, int
   offs = alignSize(offs + nphi * sizeof(float), getBufferAlignmentBytes());                  // account for alignment
 
   for (int i = nphi; i--;) {
-    mSliceCos[i] = std::cos(getPhiBinMin(i));
-    mSliceSin[i] = std::sin(getPhiBinMin(i));
+    mSliceCos[i] = o2::math_utils::cos(getPhiBinMin(i));
+    mSliceSin[i] = o2::math_utils::sin(getPhiBinMin(i));
   }
 
   o2::gpu::resizeArray(mCells, 0, getNCells(), reinterpret_cast(mFlatBufferPtr + offs));
@@ -193,20 +195,25 @@ void MatLayerCyl::optimizePhiSlices(float maxRelDiff)
     return;
   }
   int newSl = 0;
+  std::vector phi2SlNew(getNPhiBins());
+  for (int i = 0; i < getNPhiBins(); i++) {
+    phi2SlNew[i] = mPhiBin2Slice[i];
+  }
   for (int is = 1; is < getNPhiSlices(); is++) {
     if (!canMergePhiSlices(is - 1, is, maxRelDiff)) {
       newSl++;
+    } else {
+      mPhiBin2Slice[is] = mPhiBin2Slice[is - 1];
     }
-    mPhiBin2Slice[is] = newSl;
+    phi2SlNew[is] = newSl; // new numbering
   }
-  LOG(info) << newSl + 1 << " slices out of " << getNPhiBins();
   if (newSl + 1 == getNPhiSlices()) {
     return;
   }
   newSl = 0;
   int slMin = 0, slMax = 0, is = 0;
   while (is++ < getNPhiSlices()) {
-    while (is < getNPhiSlices() && mPhiBin2Slice[is] == newSl) { // select similar slices
+    while (is < getNPhiSlices() && phi2SlNew[is] == newSl) { // select similar slices
       slMax++;
       is++;
     }
@@ -229,6 +236,9 @@ void MatLayerCyl::optimizePhiSlices(float maxRelDiff)
     newSl++;
     slMin = slMax = is;
   }
+  for (int i = 0; i < getNPhiBins(); i++) {
+    mPhiBin2Slice[i] = phi2SlNew[i];
+  }
   mNPhiSlices = newSl;
 
   // relocate arrays to avoid spaces after optimization
@@ -265,8 +275,8 @@ void MatLayerCyl::getMeanRMS(MatCell& mean, MatCell& rms) const
   rms.meanX2X0 /= nc;
   rms.meanRho -= mean.meanRho * mean.meanRho;
   rms.meanX2X0 -= mean.meanX2X0 * mean.meanX2X0;
-  rms.meanRho = rms.meanRho > 0.f ? std::sqrt(rms.meanRho) : 0.f;
-  rms.meanX2X0 = rms.meanX2X0 > 0.f ? std::sqrt(rms.meanX2X0) : 0.f;
+  rms.meanRho = rms.meanRho > 0.f ? o2::math_utils::sqrt(rms.meanRho) : 0.f;
+  rms.meanX2X0 = rms.meanX2X0 > 0.f ? o2::math_utils::sqrt(rms.meanX2X0) : 0.f;
 }
 
 //________________________________________________________________________________
@@ -309,9 +319,24 @@ void MatLayerCyl::flatten(char* newPtr)
   mConstructionMask = Constructed;
 }
 
+//________________________________________________________________________________
+void MatLayerCyl::scale(float factor, bool _x2x0, bool _rho)
+{
+  LOGP(info, "Scaling layer {:.3f}
 #include "CommonUtils/TreeStreamRedirector.h"
 //#define _DBG_LOC_ // for local debugging only
 
 #endif // !GPUCA_ALIGPUCODE
-
+#undef NDEBUG
 using namespace o2::base;
 
 using flatObject = o2::gpu::FlatObject;
@@ -90,7 +88,15 @@ void MatLayerCylSet::populateFromTGeo(int ntrPerCell)
     get()->mLayers[i].print();
     get()->mLayers[i].populateFromTGeo(ntrPerCell);
   }
+  finalizeStructures();
+}
+
+//________________________________________________________________________________
+void MatLayerCylSet::finalizeStructures()
+{
   // build layer search structures
+  assert(mConstructionMask == InProgress);
+  int nlr = getNLayers();
   int nR2Int = 2 * (nlr + 1);
   o2::gpu::resizeArray(get()->mR2Intervals, 0, nR2Int);
   o2::gpu::resizeArray(get()->mInterval2LrID, 0, nR2Int);
@@ -102,7 +108,7 @@ void MatLayerCylSet::populateFromTGeo(int ntrPerCell)
 
   for (int i = 1; i < nlr; i++) {
     const auto& lr = getLayer(i);
-    if (std::sqrt(lr.getRMin2()) > std::sqrt(get()->mR2Intervals[nRIntervals] + Ray::Tiny)) {
+    if (o2::math_utils::sqrt(lr.getRMin2()) > o2::math_utils::sqrt(get()->mR2Intervals[nRIntervals] + Ray::Tiny)) {
       // register gap
       get()->mInterval2LrID[nRIntervals] = -1;
       get()->mR2Intervals[++nRIntervals] = lr.getRMin2();
@@ -169,6 +175,29 @@ void MatLayerCylSet::writeToFile(const std::string& outFName)
   outf.Close();
 }
 
+void MatLayerCylSet::initLayerVoxelLU()
+{
+  if (mInitializedLayerVoxelLU) {
+    LOG(info) << "Layer voxel already initialized; Aborting";
+    return;
+  }
+  LOG(info) << "Initializing voxel layer lookup";
+  // do some check if voxels are dimensioned correctly
+  if (LayerRMax < get()->mRMax) {
+    LOG(fatal) << "Cannot initialized layer voxel lookup due to dimension problem (fix constants in MatLayerCylSet.h)";
+  }
+  for (int voxel = 0; voxel < NumVoxels; ++voxel) {
+    // check the 2 extremes of this voxel "covering"
+    const auto lowerR = voxel * VoxelRDelta;
+    const auto upperR = lowerR + VoxelRDelta;
+    const auto lowerSegment = searchSegment(lowerR * lowerR);
+    const auto upperSegment = searchSegment(upperR * upperR);
+    mLayerVoxelLU[2 * voxel] = lowerSegment;
+    mLayerVoxelLU[2 * voxel + 1] = upperSegment;
+  }
+  mInitializedLayerVoxelLU = true;
+}
+
 //________________________________________________________________________________
 MatLayerCylSet* MatLayerCylSet::loadFromFile(const std::string& inpFName)
 {
@@ -182,7 +211,8 @@ MatLayerCylSet* MatLayerCylSet::loadFromFile(const std::string& inpFName)
     LOG(error) << "Failed to load mat.LUT from " << inpFName;
     return nullptr;
   }
-  return rectifyPtrFromFile(mb);
+  auto rptr = rectifyPtrFromFile(mb);
+  return rptr;
 }
 
 //________________________________________________________________________________
@@ -192,6 +222,7 @@ MatLayerCylSet* MatLayerCylSet::rectifyPtrFromFile(MatLayerCylSet* ptr)
   if (ptr && !ptr->get()) {
     ptr->fixPointers();
   }
+  ptr->initLayerVoxelLU();
   return ptr;
 }
 
@@ -225,6 +256,33 @@ void MatLayerCylSet::print(bool data) const
          float(getFlatBufferSize()) / 1024 / 1024);
 }
 
+//________________________________________________________________________________
+void MatLayerCylSet::scaleLayersByID(int lrFrom, int lrTo, float factor, bool _x2x0, bool _rho)
+{
+  lrFrom = std::max(0, std::min(lrFrom, get()->mNLayers - 1));
+  lrTo = std::max(0, std::min(lrTo, get()->mNLayers - 1));
+  int dir = lrFrom >= lrTo ? -1 : 1;
+  lrTo += dir;
+  for (int i = lrFrom; i != lrTo; i += dir) {
+    get()->mLayers[i].scale(factor, _x2x0, _rho);
+  }
+}
+
+//________________________________________________________________________________
+void MatLayerCylSet::scaleLayersByR(float rFrom, float rTo, float factor, bool _x2x0, bool _rho)
+{
+  if (rFrom > rTo) {
+    std::swap(rFrom, rTo);
+  }
+  Ray ray(std::max(getRMin(), rFrom), 0., 0., std::min(getRMax(), rTo), 0., 0.);
+  short lmin, lmax;
+  if (!getLayersRange(ray, lmin, lmax)) {
+    LOGP(warn, "No layers found for {} < r < {}", rFrom, rTo);
+    return;
+  }
+  scaleLayersByID(lmin, lmax, factor, _x2x0, _rho);
+}
+
 #endif //!GPUCA_ALIGPUCODE
 
 #ifndef GPUCA_GPUCODE
@@ -259,10 +317,11 @@ GPUd() MatBudget MatLayerCylSet::getMatBudget(float x0, float y0, float z0, floa
   while (lrID >= lmin) { // go from outside to inside
     const auto& lr = getLayer(lrID);
     int nphiSlices = lr.getNPhiSlices();
-    int nc = ray.crossLayer(lr);
+    int nc = ray.crossLayer(lr); // determines how many crossings this ray has with this tubular layer
     for (int ic = nc; ic--;) {
       float cross1, cross2;
       ray.getCrossParams(ic, cross1, cross2); // tmax,tmin of crossing the layer
+
       auto phi0 = ray.getPhi(cross1), phi1 = ray.getPhi(cross2), dPhi = phi0 - phi1;
       auto phiID = lr.getPhiSliceID(phi0), phiIDLast = lr.getPhiSliceID(phi1);
       // account for eventual wrapping around 0
@@ -275,6 +334,7 @@ GPUd() MatBudget MatLayerCylSet::getMatBudget(float x0, float y0, float z0, floa
           phiID += nphiSlices;
         }
       }
+
       int stepPhiID = phiID > phiIDLast ? -1 : 1;
       bool checkMorePhi = true;
       auto tStartPhi = cross1, tEndPhi = 0.f;
@@ -382,21 +442,40 @@ GPUd() bool MatLayerCylSet::getLayersRange(const Ray& ray, short& lmin, short& l
     return false;
   }
   int lmxInt, lmnInt;
-  lmxInt = rmax2 < getRMax2() ? searchSegment(rmax2, 0) : get()->mNRIntervals - 2;
-  lmnInt = rmin2 >= getRMin2() ? searchSegment(rmin2, 0, lmxInt + 1) : 0;
+  if (!mInitializedLayerVoxelLU) {
+    lmxInt = rmax2 < getRMax2() ? searchSegment(rmax2, 0) : get()->mNRIntervals - 2;
+    lmnInt = rmin2 >= getRMin2() ? searchSegment(rmin2, 0, lmxInt + 1) : 0;
+  } else {
+    lmxInt = rmax2 < getRMax2() ? searchLayerFast(rmax2, 0) : get()->mNRIntervals - 2;
+    lmnInt = rmin2 >= getRMin2() ? searchLayerFast(rmin2, 0, lmxInt + 1) : 0;
+  }
+
   const auto* interval2LrID = get()->mInterval2LrID;
   lmax = interval2LrID[lmxInt];
   lmin = interval2LrID[lmnInt];
   // make sure lmnInt and/or lmxInt are not in the gap
   if (lmax < 0) {
-    lmax = interval2LrID[--lmxInt]; // rmax2 is in the gap, take highest layer below rmax2
+    lmax = interval2LrID[lmxInt - 1]; // rmax2 is in the gap, take highest layer below rmax2
   }
   if (lmin < 0) {
-    lmin = interval2LrID[++lmnInt]; // rmin2 is in the gap, take lowest layer above rmin2
+    lmin = interval2LrID[lmnInt + 1]; // rmin2 is in the gap, take lowest layer above rmin2
   }
   return lmin <= lmax; // valid if both are not in the same gap
 }
 
+GPUd() int MatLayerCylSet::searchLayerFast(float r2, int low, int high) const
+{
+  // we can avoid the sqrt .. at the cost of more memory in the lookup
+  const auto index = 2 * int(o2::gpu::CAMath::Sqrt(r2) * InvVoxelRDelta);
+  const auto layersfirst = mLayerVoxelLU[index];
+  const auto layerslast = mLayerVoxelLU[index + 1];
+  if (layersfirst != layerslast) {
+    // this means the voxel is undecided and we revert to search
+    return searchSegment(r2, layersfirst, layerslast + 1);
+  }
+  return layersfirst;
+}
+
 GPUd() int MatLayerCylSet::searchSegment(float val, int low, int high) const
 {
   ///< search segment val belongs to. The val MUST be within the boundaries
@@ -416,6 +495,7 @@ GPUd() int MatLayerCylSet::searchSegment(float val, int low, int high) const
     }
     mid = (low + high) >> 1;
   }
+
   return mid;
 }
 
@@ -525,3 +605,55 @@ void MatLayerCylSet::fixPointers(char* oldPtr, char* newPtr, bool newPtrValid)
   }
 }
 #endif // !GPUCA_GPUCODE
+
+#ifndef GPUCA_ALIGPUCODE // this part is unvisible on GPU version
+
+MatLayerCylSet* MatLayerCylSet::extractCopy(float rmin, float rmax, float tolerance, const MatLayerCylSet* addTo) const
+{
+  // extract layers in the covering rmin-rmax range. If addTo is provided, simply substitute its layers by those from this
+  if (addTo && addTo->getNLayers() != getNLayers()) {
+    LOGP(fatal, "addTo has {} layers, this has {}", addTo->getNLayers(), getNLayers());
+  }
+  Ray ray(std::max(getRMin(), rmin), 0., 0., std::min(getRMax(), rmax), 0., 0.);
+  short lmin, lmax;
+  if (!getLayersRange(ray, lmin, lmax)) {
+    LOGP(warn, "No layers found for {} < r < {}", rmin, rmax);
+    return nullptr;
+  }
+  LOGP(info, "Will extract layers {}:{} (out of {} layers) for {} < r < {}", lmin, lmax, getNLayers(), rmin, rmax);
+  MatLayerCylSet* copy = new MatLayerCylSet();
+  int lrCount = 0, lrCounOld = 0, lrCountTot = 0;
+  auto addLr = [copy, &lrCountTot](const MatLayerCyl& lr) {
+    float drphi = lr.getDPhi() * (lr.getRMin() + lr.getRMax()) / 2. * 0.999;
+    copy->addLayer(lr.getRMin(), lr.getRMax(), lr.getZMax(), lr.getDZ(), drphi);
+    auto& lrNew = copy->getLayer(lrCountTot++);
+    for (int iz = 0; iz < lrNew.getNZBins(); iz++) {
+      for (int ip = 0; ip < lrNew.getNPhiBins(); ip++) {
+        lrNew.getCellPhiBin(ip, iz).set(lr.getCellPhiBin(ip, iz));
+      }
+    }
+  };
+  if (addTo) {
+    for (int il = 0; il < lmin; il++) {
+      addLr(addTo->getLayer(il));
+      lrCounOld++;
+    }
+  }
+  for (int il = lmin; il <= lmax; il++) {
+    addLr(getLayer(il));
+    lrCount++;
+  }
+  if (addTo) {
+    for (int il = lmax + 1; il < getNLayers(); il++) {
+      addLr(addTo->getLayer(il));
+      lrCounOld++;
+    }
+  }
+  copy->finalizeStructures();
+  copy->optimizePhiSlices(tolerance);
+  copy->flatten();
+  LOGP(info, "Added layers {}:{} for {}
 #include 
 #include 
-#include 
+#include 
 #ifdef NDEBUG
 #undef NDEBUG
 #endif
@@ -123,22 +123,51 @@ void MaterialManager::initDensityMap()
   mDensityMapInitialized = true;
 }
 
-float MaterialManager::getDensity(std::string const& modname)
+float MaterialManager::getDensity(std::string const& modname, std::string const& matname)
 {
+  // This function returns the final density for a material of name matname inside module modname.
+  // The priority is
+  // - return density for a specific module + material if it exists in the lookup
+  // - return density for the module if it exists in the the lookup
+  // - return global density factor
+
+  auto debug = getenv("O2SIM_MATMGR_LOCALDENSITY_DEBUG");
+
   if (!mDensityMapInitialized) {
     initDensityMap();
   }
-  if (mDensityMap.find(modname) != mDensityMap.end()) {
-    return mDensityMap[modname];
+  // density on final material level
+  // (this works by a name lookup of pair "modname/matname")
+  std::string lookupstring = modname + "/" + matname;
+  auto iter = mDensityMap.find(lookupstring);
+  if (iter != mDensityMap.end()) {
+    if (debug) {
+      LOG(info) << "MatManager - " << modname << "/" << matname << " : applying density " << iter->second << " from material match";
+    }
+    return iter->second;
   }
-  return o2::conf::SimMaterialParams::Instance().globalDensityFactor;
+  // density on module level
+  iter = mDensityMap.find(modname);
+  if (iter != mDensityMap.end()) {
+    if (debug) {
+      LOG(info) << "MatManager - " << modname << "/" << matname << " : applying density " << iter->second << " from module match";
+    }
+    return iter->second;
+  }
+  // global factor
+  const auto global = o2::conf::SimMaterialParams::Instance().globalDensityFactor;
+  if (debug && global != 1.0) {
+    LOG(info) << "MatManager - " << modname << "/" << matname << " : applying global density " << iter->second;
+  }
+  return global;
 }
 
 void MaterialManager::Material(const char* modname, Int_t imat, const char* name, Float_t a, Float_t z, Float_t dens,
                                Float_t radl, Float_t absl, Float_t* buf, Int_t nwbuf)
 {
   TString uniquename = modname;
-  auto densityFactor = getDensity(modname);
+  auto densityFactor = getDensity(modname, name);
+
   uniquename.Append("_");
   uniquename.Append(name);
   if (TVirtualMC::GetMC()) {
@@ -173,7 +202,7 @@ void MaterialManager::Mixture(const char* modname, Int_t imat, const char* name,
                               Int_t nlmat, Float_t* wmat)
 {
   TString uniquename = modname;
-  auto densityFactor = getDensity(modname);
+  auto densityFactor = getDensity(modname, name);
   uniquename.Append("_");
   uniquename.Append(name);
 
@@ -254,6 +283,12 @@ void MaterialManager::Cut(ESpecial special, int globalindex, ECut cut, Float_t v
   if (val < 0.) {
     return;
   }
+  // if low energy neutron transport is requested setting kCUTNEU will set to 0.005eV
+  if (mLowNeut && cut == ECut::kCUTNEU) {
+    LOG(info) << "Due to low energy neutrons, neutron cut value " << val << " discarded and reset to 5e-12";
+    val = 5.e-12;
+  }
+
   auto it = mCutIDToName.find(cut);
   if (it == mCutIDToName.end()) {
     return;
@@ -423,7 +458,7 @@ void MaterialManager::loadCutsAndProcessesFromJSON(ESpecial special, std::string
   }
   std::ifstream is(filenameIn);
   if (!is.is_open()) {
-    LOG(error) << "Cannot open file " << filenameIn;
+    LOG(fatal) << "Cannot open MC cuts/processes file " << filenameIn;
     return;
   }
   auto digestCutsFromJSON = [this](int globalindex, rj::Value& cuts) {
diff --git a/Detectors/Base/src/O2Tessellated.cxx b/Detectors/Base/src/O2Tessellated.cxx
new file mode 100644
index 0000000000000..256a70e5a697a
--- /dev/null
+++ b/Detectors/Base/src/O2Tessellated.cxx
@@ -0,0 +1,1509 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+// Sandro Wenzel 2026
+
+// An implementation of TGeoTessellated augmented with efficient navigation functions.
+// Asked for integration into ROOT here https://github.com/root-project/root/pull/21045
+// Will be deleted once we get this from ROOT.
+
+#include 
+#include 
+
+#include "TGeoManager.h"
+#include "TGeoMatrix.h"
+#include "TGeoVolume.h"
+#include "TVirtualGeoPainter.h"
+#include "DetectorsBase/O2Tessellated.h"
+#include "TBuffer3D.h"
+#include "TBuffer3DTypes.h"
+#include "TMath.h"
+#include "TBuffer.h"
+
+#include 
+#include 
+
+// THIS IS THIRD PARTY CODE (TO BE PUT IN ROOT) WHICH DOES NOT NEED TO ADHERE TO OUR LINTING
+// NOLINTBEGIN
+
+// include the Third-party BVH headers
+#include "bvh2_third_party.h"
+// some kernels on top of BVH
+#include "bvh2_extra_kernels.h"
+
+#include 
+#include 
+
+using namespace o2::base;
+ClassImp(O2Tessellated);
+
+using Vertex_t = Tessellated::Vertex_t;
+
+////////////////////////////////////////////////////////////////////////////////
+/// Compact consecutive equal vertices
+
+int TGeoFacet::CompactFacet(Vertex_t* vert, int nvertices)
+{
+  // Compact the common vertices and return new facet
+  if (nvertices < 2)
+    return nvertices;
+  int nvert = nvertices;
+  int i = 0;
+  while (i < nvert) {
+    if (vert[(i + 1) % nvert] == vert[i]) {
+      // shift last vertices left by one element
+      for (int j = i + 2; j < nvert; ++j)
+        vert[j - 1] = vert[j];
+      nvert--;
+    }
+    i++;
+  }
+  return nvert;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// Check if a connected neighbour facet has compatible normal
+
+bool TGeoFacet::IsNeighbour(const TGeoFacet& other, bool& flip) const
+{
+
+  // Find a connecting segment
+  bool neighbour = false;
+  int line1[2], line2[2];
+  int npoints = 0;
+  for (int i = 0; i < fNvert; ++i) {
+    auto ivert = fIvert[i];
+    // Check if the other facet has the same vertex
+    for (int j = 0; j < other.GetNvert(); ++j) {
+      if (ivert == other[j]) {
+        line1[npoints] = i;
+        line2[npoints] = j;
+        if (++npoints == 2) {
+          neighbour = true;
+          bool order1 = line1[1] == line1[0] + 1;
+          bool order2 = line2[1] == (line2[0] + 1) % other.GetNvert();
+          flip = (order1 == order2);
+          return neighbour;
+        }
+      }
+    }
+  }
+  return neighbour;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// Constructor. In case nfacets is zero, it is user's responsibility to
+/// call CloseShape once all faces are defined.
+
+O2Tessellated::O2Tessellated(const char* name, int nfacets) : TGeoBBox(name, 0, 0, 0)
+{
+  fNfacets = nfacets;
+  if (nfacets)
+    fFacets.reserve(nfacets);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// Constructor providing directly the array of vertices. Facets have to be added
+/// providing vertex indices rather than coordinates.
+
+O2Tessellated::O2Tessellated(const char* name, const std::vector& vertices) : TGeoBBox(name, 0, 0, 0)
+{
+  fVertices = vertices;
+  fNvert = fVertices.size();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// Construct from TGeoTessellated
+
+O2Tessellated::O2Tessellated(TGeoTessellated const& tsl, bool check) : TGeoBBox(tsl.GetName(), 0, 0, 0)
+{
+  fNfacets = tsl.GetNfacets();
+  fNvert = tsl.GetNvertices();
+  fNseg = tsl.GetNsegments();
+
+  // copy facet and vertex done
+  fVertices.reserve(fNvert);
+  fFacets.reserve(fNfacets);
+  for (int i = 0; i < fNfacets; ++i) {
+    fFacets.push_back(tsl.GetFacet(i));
+  }
+  for (int i = 0; i < fNvert; ++i) {
+    fVertices.push_back(tsl.GetVertex(i));
+  }
+  // finish remaining structures
+  CloseShape(check);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// Add a vertex checking for duplicates, returning the vertex index
+
+int O2Tessellated::AddVertex(Vertex_t const& vert)
+{
+  constexpr double tolerance = 1.e-10;
+  auto vertexHash = [&](Vertex_t const& vertex) {
+    // Compute hash for the vertex
+    long hash = 0;
+    // helper function to generate hash from integer numbers
+    auto hash_combine = [](long seed, const long value) {
+      return seed ^ (std::hash{}(value) + 0x9e3779b9 + (seed << 6) + (seed >> 2));
+    };
+    for (int i = 0; i < 3; i++) {
+      // use tolerance to generate int with the desired precision from a real number for hashing
+      hash = hash_combine(hash, std::roundl(vertex[i] / tolerance));
+    }
+    return hash;
+  };
+
+  auto hash = vertexHash(vert);
+  bool isAdded = false;
+  int ivert = -1;
+  // Get the compatible vertices
+  auto range = fVerticesMap.equal_range(hash);
+  for (auto it = range.first; it != range.second; ++it) {
+    ivert = it->second;
+    if (fVertices[ivert] == vert) {
+      isAdded = true;
+      break;
+    }
+  }
+  if (!isAdded) {
+    ivert = fVertices.size();
+    fVertices.push_back(vert);
+    fVerticesMap.insert(std::make_pair(hash, ivert));
+  }
+  return ivert;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// Adding a triangular facet from vertex positions in absolute coordinates
+
+bool O2Tessellated::AddFacet(const Vertex_t& pt0, const Vertex_t& pt1, const Vertex_t& pt2)
+{
+  if (fDefined) {
+    Error("AddFacet", "Shape %s already fully defined. Not adding", GetName());
+    return false;
+  }
+
+  Vertex_t vert[3];
+  vert[0] = pt0;
+  vert[1] = pt1;
+  vert[2] = pt2;
+  int nvert = TGeoFacet::CompactFacet(vert, 3);
+  if (nvert < 3) {
+    Error("AddFacet", "Triangular facet at index %d degenerated. Not adding.", GetNfacets());
+    return false;
+  }
+  int ind[3];
+  for (auto i = 0; i < 3; ++i)
+    ind[i] = AddVertex(vert[i]);
+  fNseg += 3;
+  fFacets.emplace_back(ind[0], ind[1], ind[2]);
+
+  return true;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// Adding a triangular facet from indices of vertices
+
+bool O2Tessellated::AddFacet(int i0, int i1, int i2)
+{
+  if (fDefined) {
+    Error("AddFacet", "Shape %s already fully defined. Not adding", GetName());
+    return false;
+  }
+  if (fVertices.empty()) {
+    Error("AddFacet", "Shape %s Cannot add facets by indices without vertices. Not adding", GetName());
+    return false;
+  }
+
+  fNseg += 3;
+  fFacets.emplace_back(i0, i1, i2);
+  return true;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// Adding a quadrilateral facet from vertex positions in absolute coordinates
+
+bool O2Tessellated::AddFacet(const Vertex_t& pt0, const Vertex_t& pt1, const Vertex_t& pt2, const Vertex_t& pt3)
+{
+  if (fDefined) {
+    Error("AddFacet", "Shape %s already fully defined. Not adding", GetName());
+    return false;
+  }
+  Vertex_t vert[4];
+  vert[0] = pt0;
+  vert[1] = pt1;
+  vert[2] = pt2;
+  vert[3] = pt3;
+  int nvert = TGeoFacet::CompactFacet(vert, 4);
+  if (nvert < 3) {
+    Error("AddFacet", "Quadrilateral facet at index %d degenerated. Not adding.", GetNfacets());
+    return false;
+  }
+
+  int ind[4];
+  for (auto i = 0; i < nvert; ++i)
+    ind[i] = AddVertex(vert[i]);
+  fNseg += nvert;
+  if (nvert == 3)
+    fFacets.emplace_back(ind[0], ind[1], ind[2]);
+  else
+    fFacets.emplace_back(ind[0], ind[1], ind[2], ind[3]);
+
+  if (fNfacets > 0 && GetNfacets() == fNfacets)
+    CloseShape(false);
+  return true;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// Adding a quadrilateral facet from indices of vertices
+
+bool O2Tessellated::AddFacet(int i0, int i1, int i2, int i3)
+{
+  if (fDefined) {
+    Error("AddFacet", "Shape %s already fully defined. Not adding", GetName());
+    return false;
+  }
+  if (fVertices.empty()) {
+    Error("AddFacet", "Shape %s Cannot add facets by indices without vertices. Not adding", GetName());
+    return false;
+  }
+
+  fNseg += 4;
+  fFacets.emplace_back(i0, i1, i2, i3);
+  return true;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// Compute normal for a given facet
+
+Vertex_t O2Tessellated::FacetComputeNormal(int ifacet, bool& degenerated) const
+{
+  // Compute normal using non-zero segments
+  constexpr double kTolerance = 1.e-20;
+  auto const& facet = fFacets[ifacet];
+  int nvert = facet.GetNvert();
+  degenerated = true;
+  Vertex_t normal;
+  for (int i = 0; i < nvert - 1; ++i) {
+    Vertex_t e1 = fVertices[facet[i + 1]] - fVertices[facet[i]];
+    if (e1.Mag2() < kTolerance)
+      continue;
+    for (int j = i + 1; j < nvert; ++j) {
+      Vertex_t e2 = fVertices[facet[(j + 1) % nvert]] - fVertices[facet[j]];
+      if (e2.Mag2() < kTolerance)
+        continue;
+      normal = Vertex_t::Cross(e1, e2);
+      // e1 and e2 may be colinear
+      if (normal.Mag2() < kTolerance)
+        continue;
+      normal.Normalize();
+      degenerated = false;
+      break;
+    }
+    if (!degenerated)
+      break;
+  }
+  return normal;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// Check validity of facet
+
+bool O2Tessellated::FacetCheck(int ifacet) const
+{
+  constexpr double kTolerance = 1.e-10;
+  auto const& facet = fFacets[ifacet];
+  int nvert = facet.GetNvert();
+  bool degenerated = true;
+  FacetComputeNormal(ifacet, degenerated);
+  if (degenerated) {
+    std::cout << "Facet: " << ifacet << " is degenerated\n";
+    return false;
+  }
+
+  // Compute surface area
+  double surfaceArea = 0.;
+  for (int i = 1; i < nvert - 1; ++i) {
+    Vertex_t e1 = fVertices[facet[i]] - fVertices[facet[0]];
+    Vertex_t e2 = fVertices[facet[i + 1]] - fVertices[facet[0]];
+    surfaceArea += 0.5 * Vertex_t::Cross(e1, e2).Mag();
+  }
+  if (surfaceArea < kTolerance) {
+    std::cout << "Facet: " << ifacet << " has zero surface area\n";
+    return false;
+  }
+
+  return true;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// Close the shape: calculate bounding box and compact vertices
+
+void O2Tessellated::CloseShape(bool check, bool fixFlipped, bool verbose)
+{
+  if (fIsClosed && fBVH) {
+    return;
+  }
+  // Compute bounding box
+  fDefined = true;
+  fNvert = fVertices.size();
+  fNfacets = fFacets.size();
+  ComputeBBox();
+
+  BuildBVH();
+  if (fOutwardNormals.size() == 0) {
+    CalculateNormals();
+  } else {
+    // short check if the normal container is of correct size
+    if (fOutwardNormals.size() != fFacets.size()) {
+      std::cerr << "Inconsistency in normal container";
+    }
+  }
+  fIsClosed = true;
+
+  // Cleanup the vertex map
+  std::multimap().swap(fVerticesMap);
+
+  if (fVertices.size() > 0) {
+    if (!check)
+      return;
+
+    // Check facets
+    for (auto i = 0; i < fNfacets; ++i)
+      FacetCheck(i);
+
+    fClosedBody = CheckClosure(fixFlipped, verbose);
+  }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// Check closure of the solid and check/fix flipped normals
+
+bool O2Tessellated::CheckClosure(bool fixFlipped, bool verbose)
+{
+  int* nn = new int[fNfacets];
+  bool* flipped = new bool[fNfacets];
+  bool hasorphans = false;
+  bool hasflipped = false;
+  for (int i = 0; i < fNfacets; ++i) {
+    nn[i] = 0;
+    flipped[i] = false;
+  }
+
+  for (int icrt = 0; icrt < fNfacets; ++icrt) {
+    // all neighbours checked?
+    if (nn[icrt] >= fFacets[icrt].GetNvert())
+      continue;
+    for (int i = icrt + 1; i < fNfacets; ++i) {
+      bool isneighbour = fFacets[icrt].IsNeighbour(fFacets[i], flipped[i]);
+      if (isneighbour) {
+        if (flipped[icrt])
+          flipped[i] = !flipped[i];
+        if (flipped[i])
+          hasflipped = true;
+        nn[icrt]++;
+        nn[i]++;
+        if (nn[icrt] == fFacets[icrt].GetNvert())
+          break;
+      }
+    }
+    if (nn[icrt] < fFacets[icrt].GetNvert())
+      hasorphans = true;
+  }
+
+  if (hasorphans && verbose) {
+    Error("Check", "Tessellated solid %s has following not fully connected facets:", GetName());
+    for (int icrt = 0; icrt < fNfacets; ++icrt) {
+      if (nn[icrt] < fFacets[icrt].GetNvert())
+        std::cout << icrt << " (" << fFacets[icrt].GetNvert() << " edges, " << nn[icrt] << " neighbours)\n";
+    }
+  }
+  fClosedBody = !hasorphans;
+  int nfixed = 0;
+  if (hasflipped) {
+    if (verbose)
+      Warning("Check", "Tessellated solid %s has following facets with flipped normals:", GetName());
+    for (int icrt = 0; icrt < fNfacets; ++icrt) {
+      if (flipped[icrt]) {
+        if (verbose)
+          std::cout << icrt << "\n";
+        if (fixFlipped) {
+          fFacets[icrt].Flip();
+          nfixed++;
+        }
+      }
+    }
+    if (nfixed && verbose)
+      Info("Check", "Automatically flipped %d facets to match first defined facet", nfixed);
+  }
+  delete[] nn;
+  delete[] flipped;
+
+  return !hasorphans;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// Compute bounding box
+
+void O2Tessellated::ComputeBBox()
+{
+  const double kBig = TGeoShape::Big();
+  double vmin[3] = {kBig, kBig, kBig};
+  double vmax[3] = {-kBig, -kBig, -kBig};
+  for (const auto& facet : fFacets) {
+    for (int i = 0; i < facet.GetNvert(); ++i) {
+      for (int j = 0; j < 3; ++j) {
+        vmin[j] = TMath::Min(vmin[j], fVertices[facet[i]].operator[](j));
+        vmax[j] = TMath::Max(vmax[j], fVertices[facet[i]].operator[](j));
+      }
+    }
+  }
+  fDX = 0.5 * (vmax[0] - vmin[0]);
+  fDY = 0.5 * (vmax[1] - vmin[1]);
+  fDZ = 0.5 * (vmax[2] - vmin[2]);
+  for (int i = 0; i < 3; ++i)
+    fOrigin[i] = 0.5 * (vmax[i] + vmin[i]);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// Returns numbers of vertices, segments and polygons composing the shape mesh.
+
+void O2Tessellated::GetMeshNumbers(int& nvert, int& nsegs, int& npols) const
+{
+  nvert = fNvert;
+  nsegs = fNseg;
+  npols = GetNfacets();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// Creates a TBuffer3D describing *this* shape.
+/// Coordinates are in local reference frame.
+
+TBuffer3D* O2Tessellated::MakeBuffer3D() const
+{
+  const int nvert = fNvert;
+  const int nsegs = fNseg;
+  const int npols = GetNfacets();
+  auto buff = new TBuffer3D(TBuffer3DTypes::kGeneric, nvert, 3 * nvert, nsegs, 3 * nsegs, npols, 6 * npols);
+  if (buff) {
+    SetPoints(buff->fPnts);
+    SetSegsAndPols(*buff);
+  }
+  return buff;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// Prints basic info
+
+void O2Tessellated::Print(Option_t*) const
+{
+  std::cout << "=== Tessellated shape " << GetName() << " having " << GetNvertices() << " vertices and "
+            << GetNfacets() << " facets\n";
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// Fills TBuffer3D structure for segments and polygons.
+
+void O2Tessellated::SetSegsAndPols(TBuffer3D& buff) const
+{
+  const int c = GetBasicColor();
+  int* segs = buff.fSegs;
+  int* pols = buff.fPols;
+
+  int indseg = 0; // segment internal data index
+  int indpol = 0; // polygon internal data index
+  int sind = 0;   // segment index
+  for (const auto& facet : fFacets) {
+    auto nvert = facet.GetNvert();
+    pols[indpol++] = c;
+    pols[indpol++] = nvert;
+    for (auto j = 0; j < nvert; ++j) {
+      int k = (j + 1) % nvert;
+      // segment made by next consecutive points
+      segs[indseg++] = c;
+      segs[indseg++] = facet[j];
+      segs[indseg++] = facet[k];
+      // add segment to current polygon and increment segment index
+      pols[indpol + nvert - j - 1] = sind++;
+    }
+    indpol += nvert;
+  }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// Fill tessellated points to an array.
+
+void O2Tessellated::SetPoints(double* points) const
+{
+  int ind = 0;
+  for (const auto& vertex : fVertices) {
+    vertex.CopyTo(&points[ind]);
+    ind += 3;
+  }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// Fill tessellated points in float.
+
+void O2Tessellated::SetPoints(Float_t* points) const
+{
+  int ind = 0;
+  for (const auto& vertex : fVertices) {
+    points[ind++] = vertex.x();
+    points[ind++] = vertex.y();
+    points[ind++] = vertex.z();
+  }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// Resize the shape by scaling vertices within maxsize and center to origin
+
+void O2Tessellated::ResizeCenter(double maxsize)
+{
+  using Vector3_t = Vertex_t;
+
+  if (!fDefined) {
+    Error("ResizeCenter", "Not all faces are defined");
+    return;
+  }
+  Vector3_t origin(fOrigin[0], fOrigin[1], fOrigin[2]);
+  double maxedge = TMath::Max(TMath::Max(fDX, fDY), fDZ);
+  double scale = maxsize / maxedge;
+  for (size_t i = 0; i < fVertices.size(); ++i) {
+    fVertices[i] = scale * (fVertices[i] - origin);
+  }
+  fOrigin[0] = fOrigin[1] = fOrigin[2] = 0;
+  fDX *= scale;
+  fDY *= scale;
+  fDZ *= scale;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// Fills a static 3D buffer and returns a reference.
+
+const TBuffer3D& O2Tessellated::GetBuffer3D(int reqSections, Bool_t localFrame) const
+{
+  static TBuffer3D buffer(TBuffer3DTypes::kGeneric);
+
+  FillBuffer3D(buffer, reqSections, localFrame);
+
+  const int nvert = fNvert;
+  const int nsegs = fNseg;
+  const int npols = GetNfacets();
+
+  if (reqSections & TBuffer3D::kRawSizes) {
+    if (buffer.SetRawSizes(nvert, 3 * nvert, nsegs, 3 * nsegs, npols, 6 * npols)) {
+      buffer.SetSectionsValid(TBuffer3D::kRawSizes);
+    }
+  }
+  if ((reqSections & TBuffer3D::kRaw) && buffer.SectionsValid(TBuffer3D::kRawSizes)) {
+    SetPoints(buffer.fPnts);
+    if (!buffer.fLocalFrame) {
+      TransformPoints(buffer.fPnts, buffer.NbPnts());
+    }
+
+    SetSegsAndPols(buffer);
+    buffer.SetSectionsValid(TBuffer3D::kRaw);
+  }
+
+  return buffer;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// Reads a single tessellated solid from an .obj file.
+
+O2Tessellated* O2Tessellated::ImportFromObjFormat(const char* objfile, bool check, bool verbose)
+{
+  using std::vector, std::string, std::ifstream, std::stringstream, std::endl;
+
+  vector vertices;
+  vector sfacets;
+
+  struct FacetInd_t {
+    int i0 = -1;
+    int i1 = -1;
+    int i2 = -1;
+    int i3 = -1;
+    int nvert = 0;
+    FacetInd_t(int a, int b, int c)
+    {
+      i0 = a;
+      i1 = b;
+      i2 = c;
+      nvert = 3;
+    };
+    FacetInd_t(int a, int b, int c, int d)
+    {
+      i0 = a;
+      i1 = b;
+      i2 = c;
+      i3 = d;
+      nvert = 4;
+    };
+  };
+
+  vector facets;
+  // List of geometric vertices, with (x, y, z [,w]) coordinates, w is optional and defaults to 1.0.
+  // struct vtx_t { double x = 0; double y = 0; double z = 0; double w = 1; };
+
+  // Texture coordinates in u, [,v ,w]) coordinates, these will vary between 0 and 1. v, w are optional and default to
+  // 0.
+  // struct tex_t { double u; double v; double w; };
+
+  // List of vertex normals in (x,y,z) form; normals might not be unit vectors.
+  // struct vn_t { double x; double y; double z; };
+
+  // Parameter space vertices in ( u [,v] [,w] ) form; free form geometry statement
+  // struct vp_t { double u; double v; double w; };
+
+  // Faces are defined using lists of vertex, texture and normal indices which start at 1.
+  // Polygons such as quadrilaterals can be defined by using more than three vertex/texture/normal indices.
+  //     f v1//vn1 v2//vn2 v3//vn3 ...
+
+  // Records starting with the letter "l" specify the order of the vertices which build a polyline.
+  //     l v1 v2 v3 v4 v5 v6 ...
+
+  string line;
+  int ind[4] = {0};
+  ifstream file(objfile);
+  if (!file.is_open()) {
+    ::Error("O2Tessellated::ImportFromObjFormat", "Unable to open %s", objfile);
+    return nullptr;
+  }
+
+  while (getline(file, line)) {
+    stringstream ss(line);
+    string tag;
+
+    // We ignore everything which is not a vertex or a face
+    if (line.rfind('v', 0) == 0 && line.rfind("vt", 0) != 0 && line.rfind("vn", 0) != 0 && line.rfind("vn", 0) != 0) {
+      // Decode the vertex
+      double pos[4] = {0, 0, 0, 1};
+      ss >> tag >> pos[0] >> pos[1] >> pos[2] >> pos[3];
+      vertices.emplace_back(pos[0] * pos[3], pos[1] * pos[3], pos[2] * pos[3]);
+    }
+
+    else if (line.rfind('f', 0) == 0) {
+      // Decode the face
+      ss >> tag;
+      string word;
+      sfacets.clear();
+      while (ss >> word)
+        sfacets.push_back(word);
+      if (sfacets.size() > 4 || sfacets.size() < 3) {
+        ::Error("O2Tessellated::ImportFromObjFormat", "Detected face having unsupported %zu vertices",
+                sfacets.size());
+        return nullptr;
+      }
+      int nvert = 0;
+      for (auto& sword : sfacets) {
+        stringstream ssword(sword);
+        string token;
+        getline(ssword, token, '/'); // just need the vertex index, which is the first token
+        // Convert string token to integer
+
+        ind[nvert++] = stoi(token) - 1;
+        if (ind[nvert - 1] < 0) {
+          ::Error("O2Tessellated::ImportFromObjFormat", "Unsupported relative vertex index definition in %s",
+                  objfile);
+          return nullptr;
+        }
+      }
+      if (nvert == 3)
+        facets.emplace_back(ind[0], ind[1], ind[2]);
+      else
+        facets.emplace_back(ind[0], ind[1], ind[2], ind[3]);
+    }
+  }
+
+  int nvertices = (int)vertices.size();
+  int nfacets = (int)facets.size();
+  if (nfacets < 3) {
+    ::Error("O2Tessellated::ImportFromObjFormat", "Not enough faces detected in %s", objfile);
+    return nullptr;
+  }
+
+  string sobjfile(objfile);
+  if (verbose)
+    std::cout << "Read " << nvertices << " vertices and " << nfacets << " facets from " << sobjfile << endl;
+
+  auto tsl = new O2Tessellated(sobjfile.erase(sobjfile.find_last_of('.')).c_str(), vertices);
+
+  for (int i = 0; i < nfacets; ++i) {
+    auto facet = facets[i];
+    if (facet.nvert == 3)
+      tsl->AddFacet(facet.i0, facet.i1, facet.i2);
+    else
+      tsl->AddFacet(facet.i0, facet.i1, facet.i2, facet.i3);
+  }
+  tsl->CloseShape(check, true, verbose);
+  tsl->Print();
+  return tsl;
+}
+
+// implementation of some geometry helper functions in anonymous namespace
+namespace
+{
+
+using Vertex_t = Tessellated::Vertex_t;
+// The classic Moeller-Trumbore ray triangle-intersection kernel:
+// - Compute triangle edges e1, e2
+// - Compute determinant det
+// - Reject parallel rays
+// - Compute barycentric coordinates u, v
+// - Compute ray parameter t
+double rayTriangle(const Vertex_t& orig, const Vertex_t& dir, const Vertex_t& v0, const Vertex_t& v1,
+                   const Vertex_t& v2, double rayEPS = 1e-8)
+{
+  constexpr double EPS = 1e-8;
+  const double INF = std::numeric_limits::infinity();
+  Vertex_t e1{v1[0] - v0[0], v1[1] - v0[1], v1[2] - v0[2]};
+  Vertex_t e2{v2[0] - v0[0], v2[1] - v0[1], v2[2] - v0[2]};
+  auto p = Vertex_t::Cross(dir, e2);
+  auto det = e1.Dot(p);
+  if (std::abs(det) <= EPS) {
+    return INF;
+  }
+
+  Vertex_t tvec{orig[0] - v0[0], orig[1] - v0[1], orig[2] - v0[2]};
+  auto invDet = 1.0 / det;
+  auto u = tvec.Dot(p) * invDet;
+  if (u < 0.0 || u > 1.0) {
+    return INF;
+  }
+  auto q = Vertex_t::Cross(tvec, e1);
+  auto v = dir.Dot(q) * invDet;
+  if (v < 0.0 || u + v > 1.0) {
+    return INF;
+  }
+  auto t = e2.Dot(q) * invDet;
+  return (t > rayEPS) ? t : INF;
+}
+
+template 
+struct Vec3f {
+  T x, y, z;
+};
+
+template 
+inline Vec3f operator-(const Vec3f& a, const Vec3f& b)
+{
+  return {a.x - b.x, a.y - b.y, a.z - b.z};
+}
+
+template 
+inline Vec3f cross(const Vec3f& a, const Vec3f& b)
+{
+  return {a.y * b.z - a.z * b.y, a.z * b.x - a.x * b.z, a.x * b.y - a.y * b.x};
+}
+
+template 
+inline T dot(const Vec3f& a, const Vec3f& b)
+{
+  return a.x * b.x + a.y * b.y + a.z * b.z;
+}
+
+// Kernel to get closest/shortest distance between a point and a triangl (a,b,c).
+// Performed by default in float since Safety can be approximate.
+// Project point onto triangle plane
+// If projection lies inside → distance to plane
+// Otherwise compute min distance to the three edges
+// Return squared distance
+template 
+T pointTriangleDistSq(const Vec3f& p, const Vec3f& a, const Vec3f& b, const Vec3f& c)
+{
+  // Edges
+  Vec3f ab = b - a;
+  Vec3f ac = c - a;
+  Vec3f ap = p - a;
+
+  auto d1 = dot(ab, ap);
+  auto d2 = dot(ac, ap);
+  if (d1 <= T(0.0) && d2 <= T(0.0)) {
+    return dot(ap, ap); // barycentric (1,0,0)
+  }
+
+  Vec3f bp = p - b;
+  auto d3 = dot(ab, bp);
+  auto d4 = dot(ac, bp);
+  if (d3 >= T(0.0) && d4 <= d3) {
+    return dot(bp, bp); // (0,1,0)
+  }
+
+  T vc = d1 * d4 - d3 * d2;
+  if (vc <= 0.0f && d1 >= 0.0f && d3 <= 0.0f) {
+    T v = d1 / (d1 - d3);
+    Vec3f proj = {a.x + v * ab.x, a.y + v * ab.y, a.z + v * ab.z};
+    Vec3f d = p - proj;
+    return dot(d, d); // edge AB
+  }
+
+  Vec3f cp = p - c;
+  T d5 = dot(ab, cp);
+  T d6 = dot(ac, cp);
+  if (d6 >= T(0.0f) && d5 <= d6) {
+    return dot(cp, cp); // (0,0,1)
+  }
+
+  T vb = d5 * d2 - d1 * d6;
+  if (vb <= 0.0f && d2 >= 0.0f && d6 <= 0.0f) {
+    T w = d2 / (d2 - d6);
+    Vec3f proj = {a.x + w * ac.x, a.y + w * ac.y, a.z + w * ac.z};
+    Vec3f d = p - proj;
+    return dot(d, d); // edge AC
+  }
+
+  T va = d3 * d6 - d5 * d4;
+  if (va <= 0.0f && (d4 - d3) >= 0.0f && (d5 - d6) >= 0.0f) {
+    T w = (d4 - d3) / ((d4 - d3) + (d5 - d6));
+    Vec3f proj = {b.x + w * (c.x - b.x), b.y + w * (c.y - b.y), b.z + w * (c.z - b.z)};
+    Vec3f d = p - proj;
+    return dot(d, d); // edge BC
+  }
+
+  // Inside face region
+  T denom = T(1.0f) / (va + vb + vc);
+  T v = vb * denom;
+  T w = vc * denom;
+
+  Vec3f proj = {a.x + ab.x * v + ac.x * w, a.y + ab.y * v + ac.y * w, a.z + ab.z * v + ac.z * w};
+
+  Vec3f d = p - proj;
+  return dot(d, d);
+}
+
+template 
+inline Vec3f normalize(const Vec3f& v)
+{
+  T len2 = dot(v, v);
+  if (len2 == T(0.0f)) {
+    std::cerr << "Degnerate triangle. Cannot determine normal";
+    return {0, 0, 0};
+  }
+  T invLen = T(1.0f) / std::sqrt(len2);
+  return {v.x * invLen, v.y * invLen, v.z * invLen};
+}
+
+template 
+inline Vec3f triangleNormal(const Vec3f& a, const Vec3f& b, const Vec3f& c)
+{
+  const Vec3f e1 = b - a;
+  const Vec3f e2 = c - a;
+  return normalize(cross(e1, e2));
+}
+
+} // end anonymous namespace
+
+////////////////////////////////////////////////////////////////////////////////
+/// DistFromOutside
+
+Double_t O2Tessellated::DistFromOutside(const Double_t* point, const Double_t* dir, Int_t /*iact*/, Double_t stepmax,
+                                        Double_t* /*safe*/) const
+{
+  // use the BVH intersector in combination with leaf ray-triangle testing
+  double local_step = Big(); // we need this otherwise the lambda get's confused
+
+  using Scalar = float;
+  using Vec3 = bvh::v2::Vec;
+  using Node = bvh::v2::Node;
+  using Bvh = bvh::v2::Bvh;
+  using Ray = bvh::v2::Ray;
+
+  // let's fetch the bvh
+  auto mybvh = (Bvh*)fBVH;
+  if (!mybvh) {
+    assert(false);
+    return -1.;
+  }
+
+  auto truncate_roundup = [](double orig) {
+    float epsilon = std::numeric_limits::epsilon() * std::fabs(orig);
+    // Add the bias to x before assigning it to y
+    return static_cast(orig + epsilon);
+  };
+
+  // let's do very quick checks against the top node
+  const auto topnode_bbox = mybvh->get_root().get_bbox();
+  if ((-point[0] + topnode_bbox.min[0]) > stepmax) {
+    return Big();
+  }
+  if ((-point[1] + topnode_bbox.min[1]) > stepmax) {
+    return Big();
+  }
+  if ((-point[2] + topnode_bbox.min[2]) > stepmax) {
+    return Big();
+  }
+  if ((point[0] - topnode_bbox.max[0]) > stepmax) {
+    return Big();
+  }
+  if ((point[1] - topnode_bbox.max[1]) > stepmax) {
+    return Big();
+  }
+  if ((point[2] - topnode_bbox.max[2]) > stepmax) {
+    return Big();
+  }
+
+  // the ray used for bvh interaction
+  Ray ray(Vec3(point[0], point[1], point[2]), // origin
+          Vec3(dir[0], dir[1], dir[2]),       // direction
+          0.0f,                               // minimum distance (could give stepmax ?)
+          truncate_roundup(local_step));
+
+  static constexpr bool use_robust_traversal = true;
+
+  Vertex_t dir_v{dir[0], dir[1], dir[2]};
+  // Traverse the BVH and apply concrete object intersection in BVH leafs
+  bvh::v2::GrowingStack stack;
+  mybvh->intersect(ray, mybvh->get_root().index, stack, [&](size_t begin, size_t end) {
+    for (size_t prim_id = begin; prim_id < end; ++prim_id) {
+      auto objectid = mybvh->prim_ids[prim_id];
+      const auto& facet = fFacets[objectid];
+      const auto& n = fOutwardNormals[objectid];
+
+      // quick normal test. Coming from outside, the dot product must be negative
+      if (n.Dot(dir_v) > 0.) {
+        continue;
+      }
+
+      auto thisdist = rayTriangle(Vertex_t(point[0], point[1], point[2]), dir_v,
+                                  fVertices[facet[0]], fVertices[facet[1]], fVertices[facet[2]], 0.);
+
+      if (thisdist < local_step) {
+        local_step = thisdist;
+      }
+    }
+    return false; // go on after this
+  });
+
+  return local_step;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// DistFromOutside
+
+Double_t O2Tessellated::DistFromInside(const Double_t* point, const Double_t* dir, Int_t /*iact*/, Double_t /*stepmax*/,
+                                       Double_t* /*safe*/) const
+{
+  // use the BVH intersector in combination with leaf ray-triangle testing
+  double local_step = Big(); // we need this otherwise the lambda get's confused
+
+  using Scalar = float;
+  using Vec3 = bvh::v2::Vec;
+  using Node = bvh::v2::Node;
+  using Bvh = bvh::v2::Bvh;
+  using Ray = bvh::v2::Ray;
+
+  // let's fetch the bvh
+  auto mybvh = (Bvh*)fBVH;
+  if (!mybvh) {
+    assert(false);
+    return -1.;
+  }
+
+  auto truncate_roundup = [](double orig) {
+    float epsilon = std::numeric_limits::epsilon() * std::fabs(orig);
+    // Add the bias to x before assigning it to y
+    return static_cast(orig + epsilon);
+  };
+
+  // the ray used for bvh interaction
+  Ray ray(Vec3(point[0], point[1], point[2]), // origin
+          Vec3(dir[0], dir[1], dir[2]),       // direction
+          0.,                                 // minimum distance (could give stepmax ?)
+          truncate_roundup(local_step));
+
+  static constexpr bool use_robust_traversal = true;
+
+  Vertex_t dir_v{dir[0], dir[1], dir[2]};
+  // Traverse the BVH and apply concrete object intersection in BVH leafs
+  bvh::v2::GrowingStack stack;
+  mybvh->intersect(ray, mybvh->get_root().index, stack, [&](size_t begin, size_t end) {
+    for (size_t prim_id = begin; prim_id < end; ++prim_id) {
+      auto objectid = mybvh->prim_ids[prim_id];
+      auto facet = fFacets[objectid];
+      const auto& n = fOutwardNormals[objectid];
+
+      // Only exiting surfaces are relevant (from inside--> dot product must be positive)
+      if (n.Dot(dir_v) <= 0.) {
+        continue;
+      }
+
+      const auto& v0 = fVertices[facet[0]];
+      const auto& v1 = fVertices[facet[1]];
+      const auto& v2 = fVertices[facet[2]];
+
+      const double t =
+        rayTriangle(Vertex_t{point[0], point[1], point[2]}, dir_v, v0, v1, v2, 0.);
+      if (t < local_step) {
+        local_step = t;
+      }
+    }
+    return false; // go on after this
+  });
+
+  return local_step;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// Capacity
+
+Double_t O2Tessellated::Capacity() const
+{
+  // For explanation of the following algorithm see:
+  // https://en.wikipedia.org/wiki/Polyhedron#Volume
+  // http://wwwf.imperial.ac.uk/~rn/centroid.pdf
+
+  double vol = 0.0;
+  for (size_t i = 0; i < fFacets.size(); ++i) {
+    auto& facet = fFacets[i];
+    auto a = fVertices[facet[0]];
+    auto b = fVertices[facet[1]];
+    auto c = fVertices[facet[2]];
+    vol +=
+      a[0] * (b[1] * c[2] - b[2] * c[1]) + b[0] * (c[1] * a[2] - c[2] * a[1]) + c[0] * (a[1] * b[2] - a[2] * b[1]);
+  }
+  return vol / 6.0;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// BuildBVH
+
+void O2Tessellated::BuildBVH()
+{
+  using Scalar = float;
+  using BBox = bvh::v2::BBox;
+  using Vec3 = bvh::v2::Vec;
+  using Node = bvh::v2::Node;
+  using Bvh = bvh::v2::Bvh;
+
+  // helper determining axis aligned bounding box from a facet;
+  auto GetBoundingBox = [this](TGeoFacet const& facet) {
+#ifndef NDEBUG
+    const auto nvertices = facet.GetNvert();
+    assert(nvertices == 3); // for now only triangles
+#endif
+    const auto& v1 = fVertices[facet[0]];
+    const auto& v2 = fVertices[facet[1]];
+    const auto& v3 = fVertices[facet[2]];
+    BBox bbox;
+    bbox.min[0] = std::min(std::min(v1[0], v2[0]), v3[0]) - 0.001f;
+    bbox.min[1] = std::min(std::min(v1[1], v2[1]), v3[1]) - 0.001f;
+    bbox.min[2] = std::min(std::min(v1[2], v2[2]), v3[2]) - 0.001f;
+    bbox.max[0] = std::max(std::max(v1[0], v2[0]), v3[0]) + 0.001f;
+    bbox.max[1] = std::max(std::max(v1[1], v2[1]), v3[1]) + 0.001f;
+    bbox.max[2] = std::max(std::max(v1[2], v2[2]), v3[2]) + 0.001f;
+    return bbox;
+  };
+
+  // we need bounding boxes enclosing the primitives and centers of primitives
+  // (replaced here by centers of bounding boxes) to build the bvh
+  std::vector bboxes;
+  std::vector centers;
+
+  // loop over all the triangles/Facets;
+  int nd = fFacets.size();
+  for (int i = 0; i < nd; ++i) {
+    auto& facet = fFacets[i];
+
+    // fetch the bounding box of this node and add to the vector of bounding boxes
+    (bboxes).push_back(GetBoundingBox(facet));
+    centers.emplace_back((bboxes).back().get_center());
+  }
+
+  // check if some previous object is registered and delete if necessary
+  if (fBVH) {
+    delete (Bvh*)fBVH;
+    fBVH = nullptr;
+  }
+
+  // create the bvh
+  typename bvh::v2::DefaultBuilder::Config config;
+  config.quality = bvh::v2::DefaultBuilder::Quality::High;
+  auto bvh = bvh::v2::DefaultBuilder::build(bboxes, centers, config);
+  auto bvhptr = new Bvh;
+  *bvhptr = std::move(bvh); // copy structure
+  fBVH = (void*)(bvhptr);
+
+  return;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// Contains
+
+bool O2Tessellated::Contains(Double_t const* point) const
+{
+  // we do the parity test
+  using Scalar = float;
+  using Vec3 = bvh::v2::Vec;
+  using Node = bvh::v2::Node;
+  using Bvh = bvh::v2::Bvh;
+  using Ray = bvh::v2::Ray;
+
+  // let's fetch the bvh
+  auto mybvh = (Bvh*)fBVH;
+  if (!mybvh) {
+    assert(false);
+    return false;
+  }
+
+  auto truncate_roundup = [](double orig) {
+    float epsilon = std::numeric_limits::epsilon() * std::fabs(orig);
+    // Add the bias to x before assigning it to y
+    return static_cast(orig + epsilon);
+  };
+
+  // let's do very quick checks against the top node
+  if (!TGeoBBox::Contains(point)) {
+    return false;
+  }
+
+  // An arbitrary test direction.
+  // Doesn't need to be normalized and probes all normals. Also ensuring to be skewed somewhat
+  // without evident symmetries.
+  Vertex_t test_dir{1.0, 1.41421356237, 1.73205080757};
+
+  double local_step = Big();
+  // the ray used for bvh interaction
+  Ray ray(Vec3(point[0], point[1], point[2]),          // origin
+          Vec3(test_dir[0], test_dir[1], test_dir[2]), // direction
+          0.0f,                                        // minimum distance (could give stepmax ?)
+          truncate_roundup(local_step));
+
+  static constexpr bool use_robust_traversal = true;
+
+  // Traverse the BVH and apply concrete object intersection in BVH leafs
+  bvh::v2::GrowingStack stack;
+  size_t crossings = 0;
+  mybvh->intersect(ray, mybvh->get_root().index, stack, [&](size_t begin, size_t end) {
+    for (size_t prim_id = begin; prim_id < end; ++prim_id) {
+      auto objectid = mybvh->prim_ids[prim_id];
+      auto& facet = fFacets[objectid];
+
+      // for the parity test, we probe all crossing surfaces
+      const auto& v0 = fVertices[facet[0]];
+      const auto& v1 = fVertices[facet[1]];
+      const auto& v2 = fVertices[facet[2]];
+
+      const double t = rayTriangle(Vertex_t(point[0], point[1], point[2]),
+                                   test_dir, v0, v1, v2, 0.);
+
+      if (t != std::numeric_limits::infinity()) {
+        ++crossings;
+      }
+    }
+    return false;
+  });
+
+  return crossings & 1;
+}
+
+namespace
+{
+
+// Helper classes/structs used for priority queue - BVH traversal
+// structure keeping cost (value) for a BVH index
+struct BVHPrioElement {
+  size_t bvh_node_id;
+  float value;
+};
+
+// A priority queue for BVHPrioElement with an additional clear method
+// for quick reset. We intentionally derive from std::priority_queue here to expose a
+// clear() convenience method via access to the protected container `c`.
+// This is internal, non-polymorphic code and relies on standard-library
+// implementation details that are stable across supported platforms.
+template 
+class BVHPrioQueue : public std::priority_queue, Comparator>
+{
+ public:
+  using std::priority_queue,
+                            Comparator>::priority_queue; // constructor inclusion
+
+  // convenience method to quickly clear/reset the queue (instead of having to pop one by one)
+  void clear() { this->c.clear(); }
+};
+
+} // namespace
+
+/// a reusable safety kernel, which optionally returns the closest face
+template 
+inline Double_t O2Tessellated::SafetyKernel(const Double_t* point, bool in, int* closest_facet_id) const
+{
+  // This is the classic traversal/pruning of a BVH based on priority queue search
+
+  float smallest_safety_sq = TGeoShape::Big();
+
+  using Scalar = float;
+  using Vec3 = bvh::v2::Vec;
+  using Node = bvh::v2::Node;
+  using Bvh = bvh::v2::Bvh;
+
+  // let's fetch the bvh
+  auto mybvh = (Bvh*)fBVH;
+
+  // testpoint object in float for quick BVH interaction
+  Vec3 testpoint(point[0], point[1], point[2]);
+
+  auto currnode = mybvh->nodes[0]; // we start from the top BVH node
+  // we do a quick check on the top node (in case we are outside shape)
+  bool outside_top = false;
+  if (!in) {
+    outside_top = !bvh::v2::extra::contains(currnode.get_bbox(), testpoint);
+    if (outside_top) {
+      const auto safety_sq_to_top = bvh::v2::extra::SafetySqToNode(currnode.get_bbox(), testpoint);
+      // we simply return safety to the outer bounding box as an estimate
+      return std::sqrt(safety_sq_to_top);
+    }
+  }
+
+  // comparator bringing out "smallest" value on top
+  auto cmp = [](BVHPrioElement a, BVHPrioElement b) { return a.value > b.value; };
+  static thread_local BVHPrioQueue queue(cmp);
+  queue.clear();
+
+  // algorithm is based on standard iterative tree traversal with priority queues
+  float current_safety_to_node_sq = 0.f;
+
+  if (returnFace) {
+    *closest_facet_id = -1;
+  }
+
+  do {
+    if (currnode.is_leaf()) {
+      // we are in a leaf node and actually talk to a face/triangular primitive
+      const auto begin_prim_id = currnode.index.first_id();
+      const auto end_prim_id = begin_prim_id + currnode.index.prim_count();
+
+      for (auto p_id = begin_prim_id; p_id < end_prim_id; p_id++) {
+        const auto object_id = mybvh->prim_ids[p_id];
+
+        const auto& facet = fFacets[object_id];
+        const auto& v1 = fVertices[facet[0]];
+        const auto& v2 = fVertices[facet[1]];
+        const auto& v3 = fVertices[facet[2]];
+
+        auto thissafetySQ = pointTriangleDistSq(Vec3f{point[0], point[1], point[2]}, Vec3f{v1[0], v1[1], v1[2]},
+                                                Vec3f{v2[0], v2[1], v2[2]}, Vec3f{v3[0], v3[1], v3[2]});
+
+        if (thissafetySQ < smallest_safety_sq) {
+          smallest_safety_sq = thissafetySQ;
+          if (returnFace) {
+            *closest_facet_id = object_id;
+          }
+        }
+      }
+    } else {
+      // not a leave node ... for further traversal,
+      // we inject the children into priority queue based on distance to it's bounding box
+      const auto leftchild_id = currnode.index.first_id();
+      const auto rightchild_id = leftchild_id + 1;
+
+      for (size_t childid : {leftchild_id, rightchild_id}) {
+        if (childid >= mybvh->nodes.size()) {
+          continue;
+        }
+
+        const auto& node = mybvh->nodes[childid];
+        const auto inside = bvh::v2::extra::contains(node.get_bbox(), testpoint);
+
+        if (inside) {
+          // this must be further considered because we are inside the bounding box
+          queue.push(BVHPrioElement{childid, -1.});
+        } else {
+          auto safety_to_node_square = bvh::v2::extra::SafetySqToNode(node.get_bbox(), testpoint);
+          if (safety_to_node_square <= smallest_safety_sq) {
+            // this should be further considered
+            queue.push(BVHPrioElement{childid, safety_to_node_square});
+          }
+        }
+      }
+    }
+
+    if (queue.size() > 0) {
+      auto currElement = queue.top();
+      currnode = mybvh->nodes[currElement.bvh_node_id];
+      current_safety_to_node_sq = currElement.value;
+      queue.pop();
+    } else {
+      break;
+    }
+  } while (current_safety_to_node_sq <= smallest_safety_sq);
+
+  return std::nextafter(std::sqrt(smallest_safety_sq), 0.0f);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// Safety
+
+Double_t O2Tessellated::Safety(const Double_t* point, Bool_t in) const
+{
+  // we could use some caching here (in future) since queries to the solid will likely
+  // be made with some locality
+
+  // fall-back to precise safety kernel
+  return SafetyKernel(point, in);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// ComputeNormal interface
+
+void O2Tessellated::ComputeNormal(const Double_t* point, const Double_t* dir, Double_t* norm) const
+{
+  // We take the approach to identify closest facet to the point via safety
+  // and returning the normal from this face.
+
+  // TODO: Before doing that we could check for cached points from other queries
+
+  // use safety kernel
+  int closest_face_id = -1;
+  SafetyKernel(point, true, &closest_face_id);
+
+  if (closest_face_id < 0) {
+    norm[0] = 1.;
+    norm[1] = 0.;
+    norm[2] = 0.;
+    return;
+  }
+
+  const auto& n = fOutwardNormals[closest_face_id];
+  norm[0] = n[0];
+  norm[1] = n[1];
+  norm[2] = n[2];
+
+  // change sign depending on dir
+  if (norm[0] * dir[0] + norm[1] * dir[1] + norm[2] * dir[2] < 0) {
+    norm[0] = -norm[0];
+    norm[1] = -norm[1];
+    norm[2] = -norm[2];
+  }
+  return;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// trivial (non-BVH) DistFromInside function
+
+Double_t O2Tessellated::DistFromInside_Loop(const Double_t* point, const Double_t* dir) const
+{
+  Vertex_t p(point[0], point[1], point[2]);
+  Vertex_t d(dir[0], dir[1], dir[2]);
+
+  double dist = Big();
+  for (size_t i = 0; i < fFacets.size(); ++i) {
+    const auto& facet = fFacets[i];
+    const auto& n = fOutwardNormals[i];
+
+    // Only exiting surfaces are relevant (from inside--> dot product must be positive)
+    if (n.Dot(d) <= 0.0) {
+      continue;
+    }
+
+    const auto& v0 = fVertices[facet[0]];
+    const auto& v1 = fVertices[facet[1]];
+    const auto& v2 = fVertices[facet[2]];
+
+    const double t = rayTriangle(p, d, v0, v1, v2, 0.);
+
+    if (t < dist) {
+      dist = t;
+    }
+  }
+  return dist;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// trivial (non-BVH) DistFromOutside function
+
+Double_t O2Tessellated::DistFromOutside_Loop(const Double_t* point, const Double_t* dir) const
+{
+  Vertex_t p(point[0], point[1], point[2]);
+  Vertex_t d(dir[0], dir[1], dir[2]);
+
+  double dist = Big();
+  for (size_t i = 0; i < fFacets.size(); ++i) {
+    const auto& facet = fFacets[i];
+    const auto& n = fOutwardNormals[i];
+
+    // Only exiting surfaces are relevant (from outside, the dot product must be negative)
+    if (n.Dot(d) > 0.0) {
+      continue;
+    }
+
+    const auto& v0 = fVertices[facet[0]];
+    const auto& v1 = fVertices[facet[1]];
+    const auto& v2 = fVertices[facet[2]];
+
+    const double t = rayTriangle(p, d, v0, v1, v2, 0.);
+
+    if (t < dist) {
+      dist = t;
+    }
+  }
+  return dist;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// trivial (non-BVH) Contains
+
+bool O2Tessellated::Contains_Loop(const Double_t* point) const
+{
+  // Fixed ray direction
+  const Vertex_t test_dir{1.0, 1.41421356237, 1.73205080757};
+
+  Vertex_t p(point[0], point[1], point[2]);
+
+  int crossings = 0;
+  for (size_t i = 0; i < fFacets.size(); ++i) {
+    const auto& facet = fFacets[i];
+
+    const auto& v0 = fVertices[facet[0]];
+    const auto& v1 = fVertices[facet[1]];
+    const auto& v2 = fVertices[facet[2]];
+
+    const double t = rayTriangle(p, test_dir, v0, v1, v2, 0.);
+    if (t != std::numeric_limits::infinity()) {
+      ++crossings;
+    }
+  }
+  return (crossings & 1);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// Custom streamer which performs Closing on read.
+/// Recalculation of BVH and normals is fast
+
+void O2Tessellated::Streamer(TBuffer& b)
+{
+  if (b.IsReading()) {
+    b.ReadClassBuffer(O2Tessellated::Class(), this);
+    CloseShape(false); // close shape but do not re-perform checks
+  } else {
+    b.WriteClassBuffer(O2Tessellated::Class(), this);
+  }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+/// Calculate the normals
+
+void O2Tessellated::CalculateNormals()
+{
+  fOutwardNormals.clear();
+  for (auto& facet : fFacets) {
+    auto& v1 = fVertices[facet[0]];
+    auto& v2 = fVertices[facet[1]];
+    auto& v3 = fVertices[facet[2]];
+    using Vec3d = Vec3f;
+    auto norm = triangleNormal(Vec3d{v1[0], v1[1], v1[2]}, Vec3d{v2[0], v2[1], v2[2]}, Vec3d{v3[0], v3[1], v3[2]});
+    fOutwardNormals.emplace_back(Vertex_t{norm.x, norm.y, norm.z});
+  }
+}
+
+// NOLINTEND
\ No newline at end of file
diff --git a/Detectors/Base/src/Propagator.cxx b/Detectors/Base/src/Propagator.cxx
index f965dc5faf875..a5983cab8e257 100644
--- a/Detectors/Base/src/Propagator.cxx
+++ b/Detectors/Base/src/Propagator.cxx
@@ -15,6 +15,7 @@
 #include "GPUCommonMath.h"
 #include "GPUTPCGMPolynomialField.h"
 #include "MathUtils/Utils.h"
+#include "ReconstructionDataFormats/HelixHelper.h"
 #include "ReconstructionDataFormats/Vertex.h"
 
 using namespace o2::base;
@@ -62,9 +63,9 @@ void PropagatorImpl::updateField()
   }
   const value_type xyz[3] = {0.};
   if (mFieldFast) {
-    mFieldFast->GetBz(xyz, mBz);
+    mFieldFast->GetBz(xyz, mNominalBz);
   } else {
-    mBz = mField->GetBz(xyz[0], xyz[1], xyz[2]);
+    mNominalBz = mField->GetBz(xyz[0], xyz[1], xyz[2]);
   }
 }
 
@@ -164,14 +165,13 @@ GPUd() bool PropagatorImpl::PropagateToXBxByBz(TrackParCov_t& track, va
   //
   // matCorr  - material correction type, it is up to the user to make sure the pointer is attached (if LUT is requested)
   //----------------------------------------------------------------
-  const value_type Epsilon = 0.00001;
   auto dx = xToGo - track.getX();
   int dir = dx > 0.f ? 1 : -1;
   if (!signCorr) {
     signCorr = -dir; // sign of eloss correction is not imposed
   }
 
-  gpu::gpustd::array b;
+  std::array b{};
   while (math_utils::detail::abs(dx) > Epsilon) {
     auto step = math_utils::detail::min(math_utils::detail::abs(dx), maxStep);
     if (dir < 0) {
@@ -181,28 +181,105 @@ GPUd() bool PropagatorImpl::PropagateToXBxByBz(TrackParCov_t& track, va
     auto xyz0 = track.getXYZGlo();
     getFieldXYZ(xyz0, &b[0]);
 
+    auto correct = [&track, &xyz0, tofInfo, matCorr, signCorr, this]() {
+      bool res = true;
+      if (matCorr != MatCorrType::USEMatCorrNONE) {
+        auto xyz1 = track.getXYZGlo();
+        auto mb = this->getMatBudget(matCorr, xyz0, xyz1);
+        if (!track.correctForMaterial(mb.meanX2X0, mb.getXRho(signCorr))) {
+          res = false;
+        }
+        if (tofInfo) {
+          tofInfo->addStep(mb.length, track.getQ2P2()); // fill L,ToF info using already calculated step length
+          tofInfo->addX2X0(mb.meanX2X0);
+          tofInfo->addXRho(mb.getXRho(signCorr));
+        }
+      } else if (tofInfo) { // if tofInfo filling was requested w/o material correction, we need to calculate the step lenght
+        auto xyz1 = track.getXYZGlo();
+        math_utils::Vector3D stepV(xyz1.X() - xyz0.X(), xyz1.Y() - xyz0.Y(), xyz1.Z() - xyz0.Z());
+        tofInfo->addStep(stepV.R(), track.getQ2P2());
+      }
+      return res;
+    };
+
     if (!track.propagateTo(x, b)) {
       return false;
     }
     if (maxSnp > 0 && math_utils::detail::abs(track.getSnp()) >= maxSnp) {
+      correct();
       return false;
     }
-    if (matCorr != MatCorrType::USEMatCorrNONE) {
-      auto xyz1 = track.getXYZGlo();
-      auto mb = getMatBudget(matCorr, xyz0, xyz1);
-      if (!track.correctForMaterial(mb.meanX2X0, mb.getXRho(signCorr))) {
-        return false;
-      }
+    if (!correct()) {
+      return false;
+    }
+    dx = xToGo - track.getX();
+  }
+  track.setX(xToGo);
+  return true;
+}
 
-      if (tofInfo) {
-        tofInfo->addStep(mb.length, track.getP2Inv()); // fill L,ToF info using already calculated step length
-        tofInfo->addX2X0(mb.meanX2X0);
-        tofInfo->addXRho(mb.getXRho(signCorr));
+//_______________________________________________________________________
+template 
+GPUd() bool PropagatorImpl::PropagateToXBxByBz(TrackParCov_t& track, TrackPar_t& linRef, value_type xToGo, value_type maxSnp, value_type maxStep,
+                                                        PropagatorImpl::MatCorrType matCorr, track::TrackLTIntegral* tofInfo, int signCorr) const
+{
+  //----------------------------------------------------------------
+  //
+  // Propagates the track to the plane X=xk (cm), using linRef as a Kalman linearisation point.
+  // taking into account all the three components of the magnetic field
+  // and correcting for the crossed material.
+  //
+  // maxStep  - maximal step for propagation
+  // tofInfo  - optional container for track length and PID-dependent TOF integration
+  //
+  // matCorr  - material correction type, it is up to the user to make sure the pointer is attached (if LUT is requested)
+  //----------------------------------------------------------------
+  auto dx = xToGo - track.getX();
+  int dir = dx > 0.f ? 1 : -1;
+  if (!signCorr) {
+    signCorr = -dir; // sign of eloss correction is not imposed
+  }
+
+  std::array b{};
+  while (math_utils::detail::abs(dx) > Epsilon) {
+    auto step = math_utils::detail::min(math_utils::detail::abs(dx), maxStep);
+    if (dir < 0) {
+      step = -step;
+    }
+    auto x = track.getX() + step;
+    auto xyz0 = linRef.getXYZGlo();
+    getFieldXYZ(xyz0, &b[0]);
+
+    auto correct = [&track, &linRef, &xyz0, tofInfo, matCorr, signCorr, this]() {
+      bool res = true;
+      if (matCorr != MatCorrType::USEMatCorrNONE) {
+        auto xyz1 = linRef.getXYZGlo();
+        auto mb = this->getMatBudget(matCorr, xyz0, xyz1);
+        if (!track.correctForMaterial(linRef, mb.meanX2X0, mb.getXRho(signCorr))) {
+          res = false;
+        }
+        if (tofInfo) {
+          tofInfo->addStep(mb.length, linRef.getQ2P2()); // fill L,ToF info using already calculated step length
+          tofInfo->addX2X0(mb.meanX2X0);
+          tofInfo->addXRho(mb.getXRho(signCorr));
+        }
+      } else if (tofInfo) { // if tofInfo filling was requested w/o material correction, we need to calculate the step lenght
+        auto xyz1 = linRef.getXYZGlo();
+        math_utils::Vector3D stepV(xyz1.X() - xyz0.X(), xyz1.Y() - xyz0.Y(), xyz1.Z() - xyz0.Z());
+        tofInfo->addStep(stepV.R(), linRef.getQ2P2());
       }
-    } else if (tofInfo) { // if tofInfo filling was requested w/o material correction, we need to calculate the step lenght
-      auto xyz1 = track.getXYZGlo();
-      math_utils::Vector3D stepV(xyz1.X() - xyz0.X(), xyz1.Y() - xyz0.Y(), xyz1.Z() - xyz0.Z());
-      tofInfo->addStep(stepV.R(), track.getP2Inv());
+      return res;
+    };
+
+    if (!track.propagateTo(x, linRef, b)) {
+      return false;
+    }
+    if (maxSnp > 0 && math_utils::detail::abs(track.getSnp()) >= maxSnp) {
+      correct();
+      return false;
+    }
+    if (!correct()) {
+      return false;
     }
     dx = xToGo - track.getX();
   }
@@ -226,14 +303,13 @@ GPUd() bool PropagatorImpl::PropagateToXBxByBz(TrackPar_t& track, value
   //
   // matCorr  - material correction type, it is up to the user to make sure the pointer is attached (if LUT is requested)
   //----------------------------------------------------------------
-  const value_type Epsilon = 0.00001;
   auto dx = xToGo - track.getX();
   int dir = dx > 0.f ? 1 : -1;
   if (!signCorr) {
     signCorr = -dir; // sign of eloss correction is not imposed
   }
 
-  gpu::gpustd::array b;
+  std::array b{};
   while (math_utils::detail::abs(dx) > Epsilon) {
     auto step = math_utils::detail::min(math_utils::detail::abs(dx), maxStep);
     if (dir < 0) {
@@ -243,26 +319,36 @@ GPUd() bool PropagatorImpl::PropagateToXBxByBz(TrackPar_t& track, value
     auto xyz0 = track.getXYZGlo();
     getFieldXYZ(xyz0, &b[0]);
 
+    auto correct = [&track, &xyz0, tofInfo, matCorr, signCorr, this]() {
+      bool res = true;
+      if (matCorr != MatCorrType::USEMatCorrNONE) {
+        auto xyz1 = track.getXYZGlo();
+        auto mb = this->getMatBudget(matCorr, xyz0, xyz1);
+        if (!track.correctForELoss(((signCorr < 0) ? -mb.length : mb.length) * mb.meanRho)) {
+          res = false;
+        }
+        if (tofInfo) {
+          tofInfo->addStep(mb.length, track.getQ2P2()); // fill L,ToF info using already calculated step length
+          tofInfo->addX2X0(mb.meanX2X0);
+          tofInfo->addXRho(mb.getXRho(signCorr));
+        }
+      } else if (tofInfo) { // if tofInfo filling was requested w/o material correction, we need to calculate the step lenght
+        auto xyz1 = track.getXYZGlo();
+        math_utils::Vector3D stepV(xyz1.X() - xyz0.X(), xyz1.Y() - xyz0.Y(), xyz1.Z() - xyz0.Z());
+        tofInfo->addStep(stepV.R(), track.getQ2P2());
+      }
+      return res;
+    };
+
     if (!track.propagateParamTo(x, b)) {
       return false;
     }
     if (maxSnp > 0 && math_utils::detail::abs(track.getSnp()) >= maxSnp) {
+      correct();
       return false;
     }
-    if (matCorr != MatCorrType::USEMatCorrNONE) {
-      auto xyz1 = track.getXYZGlo();
-      auto mb = getMatBudget(matCorr, xyz0, xyz1);
-      if (!track.correctForELoss(((signCorr < 0) ? -mb.length : mb.length) * mb.meanRho)) {
-        return false;
-      }
-      if (tofInfo) {
-        tofInfo->addStep(mb.length, track.getP2Inv()); // fill L,ToF info using already calculated step length
-        tofInfo->addX2X0(mb.meanX2X0);
-      }
-    } else if (tofInfo) { // if tofInfo filling was requested w/o material correction, we need to calculate the step lenght
-      auto xyz1 = track.getXYZGlo();
-      math_utils::Vector3D stepV(xyz1.X() - xyz0.X(), xyz1.Y() - xyz0.Y(), xyz1.Z() - xyz0.Z());
-      tofInfo->addStep(stepV.R(), track.getP2Inv());
+    if (!correct()) {
+      return false;
     }
     dx = xToGo - track.getX();
   }
@@ -278,15 +364,13 @@ GPUd() bool PropagatorImpl::propagateToX(TrackParCov_t& track, value_ty
   //----------------------------------------------------------------
   //
   // Propagates the track to the plane X=xk (cm)
-  // taking into account all the three components of the magnetic field
-  // and correcting for the crossed material.
+  // Use bz only and correct for the crossed material.
   //
   // maxStep  - maximal step for propagation
   // tofInfo  - optional container for track length and PID-dependent TOF integration
   //
   // matCorr  - material correction type, it is up to the user to make sure the pointer is attached (if LUT is requested)
   //----------------------------------------------------------------
-  const value_type Epsilon = 0.00001;
   auto dx = xToGo - track.getX();
   int dir = dx > 0.f ? 1 : -1;
   if (!signCorr) {
@@ -300,29 +384,101 @@ GPUd() bool PropagatorImpl::propagateToX(TrackParCov_t& track, value_ty
     }
     auto x = track.getX() + step;
     auto xyz0 = track.getXYZGlo();
-
+    auto correct = [&track, &xyz0, tofInfo, matCorr, signCorr, this]() {
+      bool res = true;
+      if (matCorr != MatCorrType::USEMatCorrNONE) {
+        auto xyz1 = track.getXYZGlo();
+        auto mb = this->getMatBudget(matCorr, xyz0, xyz1);
+        if (!track.correctForMaterial(mb.meanX2X0, mb.getXRho(signCorr))) {
+          res = false;
+        }
+        if (tofInfo) {
+          tofInfo->addStep(mb.length, track.getQ2P2()); // fill L,ToF info using already calculated step length
+          tofInfo->addX2X0(mb.meanX2X0);
+          tofInfo->addXRho(mb.getXRho(signCorr));
+        }
+      } else if (tofInfo) { // if tofInfo filling was requested w/o material correction, we need to calculate the step lenght
+        auto xyz1 = track.getXYZGlo();
+        math_utils::Vector3D stepV(xyz1.X() - xyz0.X(), xyz1.Y() - xyz0.Y(), xyz1.Z() - xyz0.Z());
+        tofInfo->addStep(stepV.R(), track.getQ2P2());
+      }
+      return res;
+    };
     if (!track.propagateTo(x, bZ)) {
       return false;
     }
     if (maxSnp > 0 && math_utils::detail::abs(track.getSnp()) >= maxSnp) {
+      correct();
       return false;
     }
-    if (matCorr != MatCorrType::USEMatCorrNONE) {
-      auto xyz1 = track.getXYZGlo();
-      auto mb = getMatBudget(matCorr, xyz0, xyz1);
-      //
-      if (!track.correctForMaterial(mb.meanX2X0, mb.getXRho(signCorr))) {
-        return false;
-      }
+    if (!correct()) {
+      return false;
+    }
+    dx = xToGo - track.getX();
+  }
+  track.setX(xToGo);
+  return true;
+}
+
+//_______________________________________________________________________
+template 
+GPUd() bool PropagatorImpl::propagateToX(TrackParCov_t& track, TrackPar_t& linRef, value_type xToGo, value_type bZ, value_type maxSnp, value_type maxStep,
+                                                  PropagatorImpl::MatCorrType matCorr, track::TrackLTIntegral* tofInfo, int signCorr) const
+{
+  //----------------------------------------------------------------
+  //
+  // Propagates the track to the plane X=xk (cm), using linRef as a Kalman linearisation point.
+  // Use bz only and correct for the crossed material if requested.
+  //
+  // maxStep  - maximal step for propagation
+  // tofInfo  - optional container for track length and PID-dependent TOF integration
+  //
+  // matCorr  - material correction type, it is up to the user to make sure the pointer is attached (if LUT is requested)
+  //----------------------------------------------------------------
+  auto dx = xToGo - track.getX();
+  int dir = dx > 0.f ? 1 : -1;
+  if (!signCorr) {
+    signCorr = -dir; // sign of eloss correction is not imposed
+  }
+
+  while (math_utils::detail::abs(dx) > Epsilon) {
+    auto step = math_utils::detail::min(math_utils::detail::abs(dx), maxStep);
+    if (dir < 0) {
+      step = -step;
+    }
+    auto x = track.getX() + step;
+    auto xyz0 = linRef.getXYZGlo();
 
-      if (tofInfo) {
-        tofInfo->addStep(mb.length, track.getP2Inv()); // fill L,ToF info using already calculated step length
-        tofInfo->addX2X0(mb.meanX2X0);
+    auto correct = [&track, &linRef, &xyz0, tofInfo, matCorr, signCorr, this]() {
+      bool res = true;
+      if (matCorr != MatCorrType::USEMatCorrNONE) {
+        auto xyz1 = linRef.getXYZGlo();
+        auto mb = this->getMatBudget(matCorr, xyz0, xyz1);
+        if (!track.correctForMaterial(linRef, mb.meanX2X0, mb.getXRho(signCorr))) {
+          res = false;
+        }
+        if (tofInfo) {
+          tofInfo->addStep(mb.length, linRef.getQ2P2()); // fill L,ToF info using already calculated step length
+          tofInfo->addX2X0(mb.meanX2X0);
+          tofInfo->addXRho(mb.getXRho(signCorr));
+        }
+      } else if (tofInfo) { // if tofInfo filling was requested w/o material correction, we need to calculate the step lenght
+        auto xyz1 = linRef.getXYZGlo();
+        math_utils::Vector3D stepV(xyz1.X() - xyz0.X(), xyz1.Y() - xyz0.Y(), xyz1.Z() - xyz0.Z());
+        tofInfo->addStep(stepV.R(), linRef.getQ2P2());
       }
-    } else if (tofInfo) { // if tofInfo filling was requested w/o material correction, we need to calculate the step lenght
-      auto xyz1 = track.getXYZGlo();
-      math_utils::Vector3D stepV(xyz1.X() - xyz0.X(), xyz1.Y() - xyz0.Y(), xyz1.Z() - xyz0.Z());
-      tofInfo->addStep(stepV.R(), track.getP2Inv());
+      return res;
+    };
+
+    if (!track.propagateTo(x, linRef, bZ)) { // linRef also updated
+      return false;
+    }
+    if (maxSnp > 0 && math_utils::detail::abs(track.getSnp()) >= maxSnp) {
+      correct();
+      return false;
+    }
+    if (!correct()) {
+      return false;
     }
     dx = xToGo - track.getX();
   }
@@ -346,7 +502,6 @@ GPUd() bool PropagatorImpl::propagateToX(TrackPar_t& track, value_type
   //
   // matCorr  - material correction type, it is up to the user to make sure the pointer is attached (if LUT is requested)
   //----------------------------------------------------------------
-  const value_type Epsilon = 0.00001;
   auto dx = xToGo - track.getX();
   int dir = dx > 0.f ? 1 : -1;
   if (!signCorr) {
@@ -361,33 +516,186 @@ GPUd() bool PropagatorImpl::propagateToX(TrackPar_t& track, value_type
     auto x = track.getX() + step;
     auto xyz0 = track.getXYZGlo();
 
+    auto correct = [&track, &xyz0, tofInfo, matCorr, signCorr, this]() {
+      bool res = true;
+      if (matCorr != MatCorrType::USEMatCorrNONE) {
+        auto xyz1 = track.getXYZGlo();
+        auto mb = this->getMatBudget(matCorr, xyz0, xyz1);
+        if (!track.correctForELoss(mb.getXRho(signCorr))) {
+          res = false;
+        }
+        if (tofInfo) {
+          tofInfo->addStep(mb.length, track.getQ2P2()); // fill L,ToF info using already calculated step length
+          tofInfo->addX2X0(mb.meanX2X0);
+          tofInfo->addXRho(mb.getXRho(signCorr));
+        }
+      } else if (tofInfo) { // if tofInfo filling was requested w/o material correction, we need to calculate the step lenght
+        auto xyz1 = track.getXYZGlo();
+        math_utils::Vector3D stepV(xyz1.X() - xyz0.X(), xyz1.Y() - xyz0.Y(), xyz1.Z() - xyz0.Z());
+        tofInfo->addStep(stepV.R(), track.getQ2P2());
+      }
+      return res;
+    };
+
     if (!track.propagateParamTo(x, bZ)) {
       return false;
     }
     if (maxSnp > 0 && math_utils::detail::abs(track.getSnp()) >= maxSnp) {
+      correct();
+      return false;
+    }
+    if (!correct()) {
+      return false;
+    }
+    dx = xToGo - track.getX();
+  }
+  track.setX(xToGo);
+  return true;
+}
+
+//_______________________________________________________________________
+template 
+template 
+GPUd() bool PropagatorImpl::propagateToR(track_T& track, value_type r, bool bzOnly, value_type maxSnp, value_type maxStep,
+                                                  MatCorrType matCorr, track::TrackLTIntegral* tofInfo, int signCorr) const
+{
+  const value_T MaxPhiLoc = math_utils::detail::asin(maxSnp), MaxPhiLocSafe = 0.95 * MaxPhiLoc;
+  auto bz = getNominalBz();
+  if (math_utils::detail::abs(bz) > constants::math::Almost0) {
+    o2::track::TrackAuxPar traux(track, bz);
+    o2::track::TrackAuxPar crad;
+    value_type r0 = math_utils::detail::sqrt(track.getX() * track.getX() + track.getY() * track.getY());
+    value_type dr = (r - r0);
+    value_type rTmp = r - (math_utils::detail::abs(dr) > 1. ? (dr > 0 ? 0.5 : -0.5) : 0.5 * dr); // 1st propagate a few mm short of the targer R
+    crad.rC = rTmp;
+    crad.c = crad.cc = 1.f;
+    crad.s = crad.ss = crad.cs = 0.f;
+    o2::track::CrossInfo cross;
+    cross.circlesCrossInfo(crad, traux, 0.);
+    if (cross.nDCA < 1) {
       return false;
     }
-    if (matCorr != MatCorrType::USEMatCorrNONE) {
-      auto xyz1 = track.getXYZGlo();
-      auto mb = getMatBudget(matCorr, xyz0, xyz1);
-      //
-      if (!track.correctForELoss(mb.getXRho(signCorr))) {
+    double phiCross[2] = {}, dphi[2] = {};
+    auto curv = track.getCurvature(bz);
+    bool clockwise = curv < 0; // q+ in B+ or q- in B- goes clockwise
+    auto phiLoc = math_utils::detail::asin(track.getSnp());
+    auto phi0 = phiLoc + track.getAlpha();
+    o2::math_utils::detail::bringTo02Pi(phi0);
+    for (int i = 0; i < cross.nDCA; i++) {
+      // track pT direction angle at crossing points:
+      // == angle of the tangential to track circle at the crossing point X,Y
+      // == normal to the radial vector from the track circle center {X-cX, Y-cY}
+      // i.e. the angle of the vector {Y-cY, -(X-cx)}
+      auto normX = double(cross.yDCA[i]) - double(traux.yC), normY = -(double(cross.xDCA[i]) - double(traux.xC));
+      if (!clockwise) {
+        normX = -normX;
+        normY = -normY;
+      }
+      phiCross[i] = math_utils::detail::atan2(normY, normX);
+      o2::math_utils::detail::bringTo02Pi(phiCross[i]);
+      dphi[i] = phiCross[i] - phi0;
+      if (dphi[i] > o2::constants::math::PI) {
+        dphi[i] -= o2::constants::math::TwoPI;
+      } else if (dphi[i] < -o2::constants::math::PI) {
+        dphi[i] += o2::constants::math::TwoPI;
+      }
+    }
+    int sel = cross.nDCA == 1 ? 0 : (clockwise ? (dphi[0] < dphi[1] ? 0 : 1) : (dphi[1] < dphi[0] ? 0 : 1));
+    auto deltaPhi = dphi[sel];
+
+    while (1) {
+      auto phiLocFin = phiLoc + deltaPhi;
+      // case1
+      if (math_utils::detail::abs(phiLocFin) < MaxPhiLocSafe) { // just 1 step propagation
+        auto deltaX = (math_utils::detail::sin(phiLocFin) - track.getSnp()) / track.getCurvature(bz);
+        if (!track.propagateTo(track.getX() + deltaX, bz)) {
+          return false;
+        }
+        break;
+      }
+      if (math_utils::detail::abs(deltaPhi) < (2 * MaxPhiLocSafe)) { // still can go in 1 step with one extra rotation
+        auto rot = phiLoc + 0.5 * deltaPhi;
+        if (!track.rotate(track.getAlpha() + rot)) {
+          return false;
+        }
+        phiLoc -= rot;
+        continue; // should be ok for the case 1 now.
+      }
+
+      auto rot = phiLoc + (deltaPhi > 0 ? MaxPhiLocSafe : -MaxPhiLocSafe);
+      if (!track.rotate(track.getAlpha() + rot)) {
         return false;
       }
+      phiLoc -= rot; // = +- MaxPhiLocSafe
 
-      if (tofInfo) {
-        tofInfo->addStep(mb.length, track.getP2Inv()); // fill L,ToF info using already calculated step length
-        tofInfo->addX2X0(mb.meanX2X0);
+      // propagate to phiLoc = +-MaxPhiLocSafe
+      auto tgtPhiLoc = deltaPhi > 0 ? MaxPhiLocSafe : -MaxPhiLocSafe;
+      auto deltaX = (math_utils::detail::sin(tgtPhiLoc) - track.getSnp()) / track.getCurvature(bz);
+      if (!track.propagateTo(track.getX() + deltaX, bz)) {
+        return false;
       }
-    } else if (tofInfo) { // if tofInfo filling was requested w/o material correction, we need to calculate the step lenght
-      auto xyz1 = track.getXYZGlo();
-      math_utils::Vector3D stepV(xyz1.X() - xyz0.X(), xyz1.Y() - xyz0.Y(), xyz1.Z() - xyz0.Z());
-      tofInfo->addStep(stepV.R(), track.getP2Inv());
+      deltaPhi -= tgtPhiLoc - phiLoc;
+      phiLoc = deltaPhi > 0 ? MaxPhiLocSafe : -MaxPhiLocSafe;
+      continue; // should be of for the case 1 now.
     }
-    dx = xToGo - track.getX();
+    bz = getBz(math_utils::Point3D{value_type(cross.xDCA[sel]), value_type(cross.yDCA[sel]), value_type(track.getZ())});
   }
-  track.setX(xToGo);
-  return true;
+  // do final step till target R, also covers Bz = 0;
+  value_type xfin;
+  if (!track.getXatLabR(r, xfin, bz)) {
+    return false;
+  }
+  return propagateToX(track, xfin, bzOnly, maxSnp, maxStep, matCorr, tofInfo, signCorr);
+}
+
+template 
+GPUd() bool PropagatorImpl::propagateToAlphaX(TrackParCov_t& track, TrackPar_t* linRef, value_type alpha, value_type x, bool bzOnly, value_type maxSnp, value_type maxStep, int minSteps,
+                                                       MatCorrType matCorr, track::TrackLTIntegral* tofInfo, int signCorr) const
+{
+  // propagate to alpha,X, if needed in a few steps
+  auto snp = track.getSnpAt(alpha, x, getNominalBz());
+  // apply safety factor 0.9 for crude rotation estimate
+  if (math_utils::detail::abs(snp) < maxSnp * 0.9 && (linRef ? track.rotate(alpha, *linRef, getNominalBz()) : track.rotate(alpha))) {
+    auto dx = math_utils::detail::abs(x - track.getX());
+    if (dx < Epsilon) {
+      return true;
+    }
+    return propagateTo(track, linRef, x, bzOnly, maxSnp, math_utils::detail::min(dx / minSteps, maxStep), matCorr, tofInfo, signCorr);
+  }
+  return false;
+}
+
+//_______________________________________________________________________
+template 
+template 
+GPUd() bool PropagatorImpl::propagateToAlphaX(track_T& track, value_type alpha, value_type x, bool bzOnly, value_type maxSnp, value_type maxStep, int minSteps,
+                                                       MatCorrType matCorr, track::TrackLTIntegral* tofInfo, int signCorr) const
+{
+  // propagate to alpha,X, if needed in a few steps
+  auto snp = track.getSnpAt(alpha, x, getNominalBz());
+  // apply safety factor 0.9 for crude rotation estimate
+  if (math_utils::detail::abs(snp) < maxSnp * 0.9 && track.rotate(alpha)) {
+    auto dx = math_utils::detail::abs(x - track.getX());
+    if (dx < Epsilon) {
+      return true;
+    }
+    return propagateTo(track, x, bzOnly, maxSnp, math_utils::detail::min(dx / minSteps, maxStep), matCorr, tofInfo, signCorr);
+  }
+  return false;
+  /*
+  // try to go in a few steps with intermediate rotations
+
+
+  auto alphaTrg = alpha;
+  math_utils::detail::bringToPMPi(alphaTrg);
+  auto alpCurr = track.getAlpha();
+  math_utils::detail::bringToPMPi(alpCurr);
+  int nsteps = minSteps > 2 ? minSteps : 2;
+  auto dalp = math_utils::detail::deltaPhiSmall(alpCurr, alphaTrg) / nsteps; // effective  (alpha - alphaCurr)/nsteps
+  auto xtmp = (track.getX() + x) / nsteps;
+  return track.rotate(alpCurr + dalp) && propagateTo(track, xtmp, bzOnly, maxSnp, maxStep, matCorr, tofInfo, signCorr) &&
+         track.rotate(alpha) && propagateTo(track, x, bzOnly, maxSnp, maxStep, matCorr, tofInfo, signCorr);
+  */
 }
 
 //_______________________________________________________________________
@@ -404,9 +712,13 @@ GPUd() bool PropagatorImpl::propagateToDCA(const o2::dataformats::Verte
   value_type xv = vtx.getX() * cs + vtx.getY() * sn, yv = -vtx.getX() * sn + vtx.getY() * cs, zv = vtx.getZ();
   x -= xv;
   y -= yv;
-  //Estimate the impact parameter neglecting the track curvature
+  // Estimate the impact parameter neglecting the track curvature
   value_type d = math_utils::detail::abs(x * snp - y * csp);
   if (d > maxD) {
+    if (dca) { // provide default DCA for failed propag
+      dca->set(o2::track::DefaultDCA, o2::track::DefaultDCA,
+               o2::track::DefaultDCACov, o2::track::DefaultDCACov, o2::track::DefaultDCACov);
+    }
     return false;
   }
   value_type crv = track.getCurvature(bZ);
@@ -427,6 +739,10 @@ GPUd() bool PropagatorImpl::propagateToDCA(const o2::dataformats::Verte
 #elif !defined(GPUCA_NO_FMT)
     LOG(debug) << "failed to propagate to alpha=" << alp << " X=" << xv << vtx;
 #endif
+    if (dca) { // provide default DCA for failed propag
+      dca->set(o2::track::DefaultDCA, o2::track::DefaultDCA,
+               o2::track::DefaultDCACov, o2::track::DefaultDCACov, o2::track::DefaultDCACov);
+    }
     return false;
   }
   track = tmpT;
@@ -453,12 +769,16 @@ GPUd() bool PropagatorImpl::propagateToDCABxByBz(const o2::dataformats:
   value_type xv = vtx.getX() * cs + vtx.getY() * sn, yv = -vtx.getX() * sn + vtx.getY() * cs, zv = vtx.getZ();
   x -= xv;
   y -= yv;
-  //Estimate the impact parameter neglecting the track curvature
+  // Estimate the impact parameter neglecting the track curvature
   value_type d = math_utils::detail::abs(x * snp - y * csp);
   if (d > maxD) {
+    if (dca) { // provide default DCA for failed propag
+      dca->set(o2::track::DefaultDCA, o2::track::DefaultDCA,
+               o2::track::DefaultDCACov, o2::track::DefaultDCACov, o2::track::DefaultDCACov);
+    }
     return false;
   }
-  value_type crv = track.getCurvature(mBz);
+  value_type crv = track.getCurvature(mNominalBz);
   value_type tgfv = -(crv * x - snp) / (crv * y + csp);
   sn = tgfv / math_utils::detail::sqrt(1.f + tgfv * tgfv);
   cs = math_utils::detail::sqrt((1. - sn) * (1. + sn));
@@ -476,6 +796,10 @@ GPUd() bool PropagatorImpl::propagateToDCABxByBz(const o2::dataformats:
 #elif !defined(GPUCA_NO_FMT)
     LOG(debug) << "failed to propagate to alpha=" << alp << " X=" << xv << vtx;
 #endif
+    if (dca) { // provide default DCA for failed propag
+      dca->set(o2::track::DefaultDCA, o2::track::DefaultDCA,
+               o2::track::DefaultDCACov, o2::track::DefaultDCACov, o2::track::DefaultDCACov);
+    }
     return false;
   }
   track = tmpT;
@@ -492,7 +816,7 @@ GPUd() bool PropagatorImpl::propagateToDCABxByBz(const o2::dataformats:
 template 
 GPUd() bool PropagatorImpl::propagateToDCA(const math_utils::Point3D& vtx, TrackPar_t& track, value_type bZ,
                                                     value_type maxStep, PropagatorImpl::MatCorrType matCorr,
-                                                    gpu::gpustd::array* dca, track::TrackLTIntegral* tofInfo,
+                                                    std::array* dca, track::TrackLTIntegral* tofInfo,
                                                     int signCorr, value_type maxD) const
 {
   // propagate track to DCA to the vertex
@@ -502,9 +826,13 @@ GPUd() bool PropagatorImpl::propagateToDCA(const math_utils::Point3D(x * snp - y * csp);
   if (d > maxD) {
+    if (dca) { // provide default DCA for failed propag
+      (*dca)[0] = o2::track::DefaultDCA;
+      (*dca)[1] = o2::track::DefaultDCA;
+    }
     return false;
   }
   value_type crv = track.getCurvature(bZ);
@@ -526,6 +854,10 @@ GPUd() bool PropagatorImpl::propagateToDCA(const math_utils::Point3D::propagateToDCA(const math_utils::Point3D
 GPUd() bool PropagatorImpl::propagateToDCABxByBz(const math_utils::Point3D& vtx, TrackPar_t& track,
                                                           value_type maxStep, PropagatorImpl::MatCorrType matCorr,
-                                                          gpu::gpustd::array* dca, track::TrackLTIntegral* tofInfo,
+                                                          std::array* dca, track::TrackLTIntegral* tofInfo,
                                                           int signCorr, value_type maxD) const
 {
   // propagate track to DCA to the vertex
@@ -550,12 +882,16 @@ GPUd() bool PropagatorImpl::propagateToDCABxByBz(const math_utils::Poin
   value_type xv = vtx.X() * cs + vtx.Y() * sn, yv = -vtx.X() * sn + vtx.Y() * cs, zv = vtx.Z();
   x -= xv;
   y -= yv;
-  //Estimate the impact parameter neglecting the track curvature
+  // Estimate the impact parameter neglecting the track curvature
   value_type d = math_utils::detail::abs(x * snp - y * csp);
   if (d > maxD) {
+    if (dca) { // provide default DCA for failed propag
+      (*dca)[0] = o2::track::DefaultDCA;
+      (*dca)[1] = o2::track::DefaultDCA;
+    }
     return false;
   }
-  value_type crv = track.getCurvature(mBz);
+  value_type crv = track.getCurvature(mNominalBz);
   value_type tgfv = -(crv * x - snp) / (crv * y + csp);
   sn = tgfv / math_utils::detail::sqrt(1.f + tgfv * tgfv);
   cs = math_utils::detail::sqrt((1. - sn) * (1. + sn));
@@ -574,6 +910,10 @@ GPUd() bool PropagatorImpl::propagateToDCABxByBz(const math_utils::Poin
 #else
     LOG(debug) << "failed to propagate to alpha=" << alp << " X=" << xv << " for vertex " << vtx.X() << ' ' << vtx.Y() << ' ' << vtx.Z();
 #endif
+    if (dca) { // provide default DCA for failed propag
+      (*dca)[0] = o2::track::DefaultDCA;
+      (*dca)[1] = o2::track::DefaultDCA;
+    }
     return false;
   }
   track = tmpT;
@@ -586,9 +926,30 @@ GPUd() bool PropagatorImpl::propagateToDCABxByBz(const math_utils::Poin
 
 //____________________________________________________________
 template 
-GPUd() void PropagatorImpl::estimateLTFast(o2::track::TrackLTIntegral& lt, const o2::track::TrackParametrization& trc) const
+GPUd() float PropagatorImpl::estimateLTIncrement(const o2::track::TrackParametrization& trc,
+                                                          const o2::math_utils::Point3D& posStart,
+                                                          const o2::math_utils::Point3D& posEnd) const
 {
-  value_T xdca = 0., ydca = 0., length = 0.;          // , zdca = 0. // zdca might be used in future
+  // estimate helical step increment between 2 point
+  float dX = posEnd.X() - posStart.X(), dY = posEnd.Y() - posStart.Y(), dZ = posEnd.Z() - posStart.Z(), d2XY = dX * dX + dY * dY;
+  if (getNominalBz() != 0) { // circular arc = 2*R*asin(dXY/2R)
+    float b[3];
+    o2::math_utils::Point3D posAv(0.5 * (posEnd.X() + posStart.X()), 0.5 * (posEnd.Y() + posStart.Y()), 0.5 * (posEnd.Z() + posStart.Z()));
+    getFieldXYZ(posAv, b);
+    float curvH = math_utils::detail::abs(0.5f * trc.getCurvature(b[2])), asArg = curvH * math_utils::detail::sqrt(d2XY);
+    if (curvH > 0.f) {
+      d2XY = asArg < 1.f ? math_utils::detail::asin(asArg) / curvH : o2::constants::math::PIHalf / curvH;
+      d2XY *= d2XY;
+    }
+  }
+  return math_utils::detail::sqrt(d2XY + dZ * dZ);
+}
+
+//____________________________________________________________
+template 
+GPUd() value_T PropagatorImpl::estimateLTFast(o2::track::TrackLTIntegral& lt, const o2::track::TrackParametrization& trc) const
+{
+  value_T xdca = 0., ydca = 0., length = 0.; // , zdca = 0. // zdca might be used in future
   o2::math_utils::CircleXY c;
   constexpr float TinyF = 1e-9;
   auto straigh_line_approx = [&]() {
@@ -605,7 +966,7 @@ GPUd() void PropagatorImpl::estimateLTFast(o2::track::TrackLTIntegral&
       return math_utils::detail::abs(trc.getY() * math_utils::detail::sqrt(1. + trc.getTgl() * trc.getTgl())); // distance from the current point to DCA
     }
   };
-  trc.getCircleParamsLoc(mBz, c);
+  trc.getCircleParamsLoc(mNominalBz, c);
   if (c.rC != 0.) {                                                     // helix
     auto distC = math_utils::detail::sqrt(c.getCenterD2()); // distance from the circle center to origin
     if (distC > 1.e-3) {
@@ -616,7 +977,7 @@ GPUd() void PropagatorImpl::estimateLTFast(o2::track::TrackLTIntegral&
       auto angcos = (v0x * v1x + v0y * v1y) / (c.rC * c.rC);
       if (math_utils::detail::abs(angcos) < 1.f) {
         auto ang = math_utils::detail::acos(angcos);
-        if ((trc.getSign() > 0.f) == (mBz > 0.f)) {
+        if ((trc.getSign() > 0.f) == (mNominalBz > 0.f)) {
           ang = -ang;   // we need signeg angle
           c.rC = -c.rC; // we need signed curvature for zdca
         }
@@ -633,8 +994,10 @@ GPUd() void PropagatorImpl::estimateLTFast(o2::track::TrackLTIntegral&
     length = straigh_line_approx();
   }
   // since we assume the track or its parent comes from the beam-line or decay, add XY(?) distance to it
-  length += math_utils::detail::sqrt(xdca * xdca + ydca * ydca);
-  lt.addStep(length, trc.getP2Inv());
+  value_T dcaT = math_utils::detail::sqrt(xdca * xdca + ydca * ydca);
+  length += dcaT;
+  lt.addStep(length, trc.getQ2P2());
+  return dcaT;
 }
 
 //____________________________________________________________
@@ -666,9 +1029,9 @@ GPUd() void PropagatorImpl::getFieldXYZImpl(const math_utils::Point3DGetField(xyz.X(), xyz.Y(), xyz.Z(), bxyzF);
-    //copy and convert
+    // copy and convert
     constexpr value_type kCLight1 = 1. / o2::gpu::gpu_common_constants::kCLight;
     for (uint i = 0; i < 3; ++i) {
       bxyz[i] = static_cast(bxyzF[i]) * kCLight1;
@@ -688,6 +1051,35 @@ GPUd() void PropagatorImpl::getFieldXYZImpl(const math_utils::Point3D
+template 
+GPUd() T PropagatorImpl::getBzImpl(const math_utils::Point3D xyz) const
+{
+  T bz = 0;
+  if (mGPUField) {
+#if defined(GPUCA_GPUCODE_DEVICE) && defined(GPUCA_HAS_GLOBAL_SYMBOL_CONSTANT_MEM)
+    const auto* f = &GPUCA_CONSMEM.param.polynomialField; // Access directly from constant memory on GPU (copied here to avoid complicated header dependencies)
+#else
+    const auto* f = mGPUField;
+#endif
+    constexpr value_type kCLight1 = 1. / o2::gpu::gpu_common_constants::kCLight;
+    bz = f->GetFieldBz(xyz.X(), xyz.Y(), xyz.Z()) * kCLight1;
+  } else {
+#ifndef GPUCA_GPUCODE
+    if (mFieldFast) {
+      mFieldFast->GetBz(xyz, bz); // Must not call the host-only function in GPU compilation
+    } else {
+#ifdef GPUCA_STANDALONE
+      LOG(fatal) << "Normal field cannot be used in standalone benchmark";
+#else
+      bz = mField->GetBz(xyz.X(), xyz.Y(), xyz.Z());
+#endif
+    }
+#endif
+  }
+  return bz;
+}
+
 template 
 GPUd() void PropagatorImpl::getFieldXYZ(const math_utils::Point3D xyz, float* bxyz) const
 {
@@ -700,10 +1092,30 @@ GPUd() void PropagatorImpl::getFieldXYZ(const math_utils::Point3D(xyz, bxyz);
 }
 
+template 
+GPUd() float PropagatorImpl::getBz(const math_utils::Point3D xyz) const
+{
+  return getBzImpl(xyz);
+}
+
+template 
+GPUd() double PropagatorImpl::getBz(const math_utils::Point3D xyz) const
+{
+  return getBzImpl(xyz);
+}
+
 namespace o2::base
 {
+#if !defined(GPUCA_GPUCODE) || defined(GPUCA_GPUCODE_DEVICE) // FIXME: DR: WORKAROUND to avoid CUDA bug creating host symbols for device code.
 template class PropagatorImpl;
-#ifndef GPUCA_GPUCODE_DEVICE
+template bool GPUdni() PropagatorImpl::propagateToAlphaX::TrackPar_t>(PropagatorImpl::TrackPar_t&, float, float, bool, float, float, int, PropagatorImpl::MatCorrType matCorr, track::TrackLTIntegral*, int) const;
+template bool GPUdni() PropagatorImpl::propagateToAlphaX::TrackParCov_t>(PropagatorImpl::TrackParCov_t&, float, float, bool, float, float, int, PropagatorImpl::MatCorrType matCorr, track::TrackLTIntegral*, int) const;
+template bool GPUdni() PropagatorImpl::propagateToR::TrackPar_t>(PropagatorImpl::TrackPar_t&, float, bool, float, float, PropagatorImpl::MatCorrType matCorr, track::TrackLTIntegral*, int) const;
+template bool GPUdni() PropagatorImpl::propagateToR::TrackParCov_t>(PropagatorImpl::TrackParCov_t&, float, bool, float, float, PropagatorImpl::MatCorrType matCorr, track::TrackLTIntegral*, int) const;
+#endif
+#ifndef GPUCA_GPUCODE
 template class PropagatorImpl;
+template bool PropagatorImpl::propagateToAlphaX::TrackPar_t>(PropagatorImpl::TrackPar_t&, double, double, bool, double, double, int, PropagatorImpl::MatCorrType matCorr, track::TrackLTIntegral*, int) const;
+template bool PropagatorImpl::propagateToAlphaX::TrackParCov_t>(PropagatorImpl::TrackParCov_t&, double, double, bool, double, double, int, PropagatorImpl::MatCorrType matCorr, track::TrackLTIntegral*, int) const;
 #endif
 } // namespace o2::base
diff --git a/Detectors/Base/src/SimFieldUtils.cxx b/Detectors/Base/src/SimFieldUtils.cxx
index 6516679efdfa2..9673e39bf1b07 100644
--- a/Detectors/Base/src/SimFieldUtils.cxx
+++ b/Detectors/Base/src/SimFieldUtils.cxx
@@ -11,14 +11,19 @@
 
 #include 
 #include 
+#include 
 #include 
 #include 
 #include 
 
 using namespace o2::base;
 
-o2::field::MagneticField* const SimFieldUtils::createMagField()
+FairField* const SimFieldUtils::createMagField()
 {
+  if (getenv("ALICE3_SIM_FIELD")) {
+    return new o2::field::ALICE3MagneticField();
+  }
+
   auto& confref = o2::conf::SimConfig::Instance();
   // a) take field from CDDB
   const auto fieldmode = confref.getConfigData().mFieldMode;
diff --git a/DataFormats/simulation/src/Stack.cxx b/Detectors/Base/src/Stack.cxx
similarity index 97%
rename from DataFormats/simulation/src/Stack.cxx
rename to Detectors/Base/src/Stack.cxx
index 72f0ae344eaa6..de69c866e7b82 100644
--- a/DataFormats/simulation/src/Stack.cxx
+++ b/Detectors/Base/src/Stack.cxx
@@ -13,14 +13,14 @@
 /// \brief Implementation of the Stack class
 /// \author M. Al-Turany, S. Wenzel - June 2014
 
-#include "SimulationDataFormat/Stack.h"
+#include "DetectorsBase/Stack.h"
 #include "DetectorsBase/Detector.h"
+#include 
 #include "DetectorsCommonDataFormats/DetID.h"
 #include "SimulationDataFormat/MCTrack.h"
-#include "SimConfig/SimParams.h"
 
-#include "FairDetector.h" // for FairDetector
-#include "FairLogger.h"   // for FairLogger
+#include "FairDetector.h"      // for FairDetector
+#include  // for FairLogger
 #include "FairRootManager.h"
 #include "SimulationDataFormat/BaseHits.h"
 #include "SimulationDataFormat/StackParam.h"
@@ -83,7 +83,6 @@ Stack::Stack(Int_t size)
   }
 
   auto& param = o2::sim::StackParam::Instance();
-  LOG(info) << param;
   TransportFcn transportPrimary;
   if (param.transportPrimary.compare("none") == 0) {
     transportPrimary = [](const TParticle& p, const std::vector& particles) {
@@ -216,7 +215,7 @@ void Stack::PushTrack(Int_t toBeDone, Int_t parentId, Int_t pdgCode, Double_t px
   TParticle p(pdgCode, iStatus, parentId, secondparentId, daughter1Id, daughter2Id, px, py, pz, e, vx, vy, vz, time);
   p.SetPolarisation(polx, poly, polz);
   p.SetWeight(weight);
-  p.SetUniqueID(proc); // using the unique ID to transfer process ID
+  p.SetUniqueID(proc);                                           // using the unique ID to transfer process ID
   p.SetBit(ParticleStatus::kPrimary, proc == kPPrimary ? 1 : 0); // set primary bit
   p.SetBit(ParticleStatus::kToBeDone, toBeDone == 1 ? 1 : 0);    // set to be done bit
   mNumberOfEntriesInParticles++;
@@ -358,11 +357,12 @@ TParticle* Stack::PopNextTrack(Int_t& iTrack)
         mIndexOfCurrentTrack = mCurrentParticle.GetStatusCode();
       }
       iTrack = mIndexOfCurrentTrack;
-      if (o2::conf::SimCutParams::Instance().trackSeed) {
+      if (mDoTrackSeeding) {
         auto hash = getHash(mCurrentParticle);
         // LOG(info) << "SEEDING NEW TRACK USING HASH" << hash;
         // init seed per track
         gRandom->SetSeed(hash);
+        o2::base::VMCSeederService::instance().setSeed();
         // NOTE: THE BETTER PLACE WOULD BE IN PRETRACK HOOK BUT THIS DOES NOT SEEM TO WORK
         // WORKS ONLY WITH G3 SINCE G4 DOES NOT CALL THIS FUNCTION
       } // .trackSeed ?
@@ -637,22 +637,29 @@ void Stack::Print(Option_t* option) const
 
 void Stack::addHit(int iDet)
 {
+  // translate detector ID to bit id
+  const auto bitID = o2::base::Detector::getDetId2HitBitIndex()[iDet];
+
   if (mIndexOfCurrentTrack < mNumberOfPrimaryParticles) {
     auto& part = mTracks->at(mIndexOfCurrentTrack);
-    part.setHit(iDet);
+    part.setHit(bitID);
 
   } else {
     Int_t iTrack = mTrackIDtoParticlesEntry[mIndexOfCurrentTrack];
     auto& part = mParticles[iTrack];
-    part.setHit(iDet);
+    part.setHit(bitID);
   }
+  mCurrentParticle.SetBit(ParticleStatus::kHasHits, 1);
   mHitCounter++;
 }
 void Stack::addHit(int iDet, Int_t iTrack)
 {
   mHitCounter++;
   auto& part = mParticles[iTrack];
-  part.setHit(iDet);
+  // fetch the bit encoding for hits
+  const auto bitID = o2::base::Detector::getDetId2HitBitIndex()[iDet];
+  part.setHit(bitID);
+  mCurrentParticle.SetBit(ParticleStatus::kHasHits, 1);
 }
 
 Int_t Stack::GetCurrentParentTrackNumber() const
@@ -758,7 +765,7 @@ bool Stack::isPrimary(const MCTrack& part)
 
 bool Stack::isFromPrimaryDecayChain(const MCTrack& part)
 {
-  /** check if the particle is from the 
+  /** check if the particle is from the
       decay chain of a primary particle **/
 
   /** check if from decay **/
@@ -777,7 +784,7 @@ bool Stack::isFromPrimaryDecayChain(const MCTrack& part)
 
 bool Stack::isFromPrimaryPairProduction(const MCTrack& part)
 {
-  /** check if the particle is from 
+  /** check if the particle is from
       pair production from a particle
       belonging to the primary decay chain **/
 
diff --git a/Detectors/Base/src/TGeoGeometryUtils.cxx b/Detectors/Base/src/TGeoGeometryUtils.cxx
new file mode 100644
index 0000000000000..6f06eff17a6d7
--- /dev/null
+++ b/Detectors/Base/src/TGeoGeometryUtils.cxx
@@ -0,0 +1,144 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file TGeoGeometryUtils.cxx
+/// \author Sandro Wenzel (CERN)
+/// \brief Collection of utility functions for TGeo
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+namespace o2
+{
+namespace base
+{
+
+namespace
+{
+// some helpers to interpret TGeo TBuffer3D output
+// and convert it to surface triangles (reengineered from TGeo code)
+
+std::vector BuildVertexLoop(const TBuffer3D& buf,
+                                 const std::vector& segs)
+{
+  // adjacency list
+  std::unordered_map> adj;
+
+  for (int s : segs) {
+    int a = buf.fSegs[3 * s + 1];
+    int b = buf.fSegs[3 * s + 2];
+    adj[a].push_back(b);
+    adj[b].push_back(a);
+  }
+
+  // start from any vertex
+  int start = adj.begin()->first;
+  int prev = -1;
+  int curr = start;
+
+  std::vector loop;
+
+  while (true) {
+    loop.push_back(curr);
+
+    const auto& nbrs = adj[curr];
+    int next = -1;
+
+    for (int n : nbrs) {
+      if (n != prev) {
+        next = n;
+        break;
+      }
+    }
+
+    if (next == -1 || next == start) {
+      break;
+    }
+
+    prev = curr;
+    curr = next;
+  }
+  return loop;
+}
+
+std::vector> ExtractPolygons(const TBuffer3D& buf)
+{
+  std::vector> polys;
+  Int_t idx = 0;
+
+  for (Int_t ip = 0; ip < buf.NbPols(); ++ip) {
+
+    idx++; // color
+    Int_t nseg = buf.fPols[idx++];
+
+    std::vector segs(nseg);
+    for (Int_t i = 0; i < nseg; ++i) {
+      segs[i] = buf.fPols[idx++];
+    }
+
+    auto verts = BuildVertexLoop(buf, segs);
+    if (verts.size() >= 3) {
+      polys.push_back(std::move(verts));
+    }
+  }
+
+  return polys;
+}
+
+std::vector>
+  Triangulate(const std::vector>& polys)
+{
+  std::vector> tris;
+  for (const auto& poly : polys) {
+    int nv = poly.size();
+    if (nv < 3) {
+      continue;
+    }
+
+    int v0 = poly[0];
+    for (int i = 1; i < nv - 1; ++i) {
+      tris.push_back({{v0, poly[i], poly[i + 1]}});
+    }
+  }
+  return tris;
+}
+
+TGeoTessellated* MakeTessellated(const TBuffer3D& buf)
+{
+  auto polys = ExtractPolygons(buf);
+  auto tris = Triangulate(polys);
+  int i = 0;
+  auto* tess = new TGeoTessellated("tess");
+  const Double_t* p = buf.fPnts;
+  for (auto& t : tris) {
+    tess->AddFacet(
+      TGeoTessellated::Vertex_t{p[3 * t[0]], p[3 * t[0] + 1], p[3 * t[0] + 2]},
+      TGeoTessellated::Vertex_t{p[3 * t[1]], p[3 * t[1] + 1], p[3 * t[1] + 2]},
+      TGeoTessellated::Vertex_t{p[3 * t[2]], p[3 * t[2] + 1], p[3 * t[2] + 2]});
+  }
+  tess->CloseShape();
+  return tess;
+}
+} // end anonymous namespace
+
+///< Transform any (primitive) TGeoShape to a TGeoTessellated
+TGeoTessellated* TGeoGeometryUtils::TGeoShapeToTGeoTessellated(TGeoShape const* shape)
+{
+  auto& buf = shape->GetBuffer3D(TBuffer3D::kRawSizes | TBuffer3D::kRaw | TBuffer3D::kCore, false);
+  auto tes = MakeTessellated(buf);
+  return tes;
+}
+
+} // namespace base
+} // namespace o2
diff --git a/Detectors/Base/src/VMCSeederService.cxx b/Detectors/Base/src/VMCSeederService.cxx
new file mode 100644
index 0000000000000..5bf4e1ed5641b
--- /dev/null
+++ b/Detectors/Base/src/VMCSeederService.cxx
@@ -0,0 +1,53 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "DetectorsBase/VMCSeederService.h"
+#include "TVirtualMC.h"
+#include                     // for FairLogger
+#include  // for ROOT JIT helpers
+
+using namespace o2::base;
+
+void VMCSeederService::initSeederFunction(TVirtualMC const* vmc)
+{
+  if (strcmp(vmc->GetName(), "TGeant3TGeo") == 0) {
+    // Geant3 doesn't need anything special in our context
+    mSeederFcn = []() {};
+  } else if (strcmp(vmc->GetName(), "TGeant4") == 0) {
+    // dynamically get access to the Geant4_VMC seeding function (without this function linking against Geant4)
+    std::string G4func("std::function G4func() { gSystem->Load(\"libgeant4vmc\"); return [](){ ((TGeant4*)TVirtualMC::GetMC())->SetRandomSeed(); };}");
+    mSeederFcn = o2::conf::JITAndEvalFunction(G4func, "G4func()", "std::function", "VMCSEEDERFUNC123");
+  } else {
+    LOG(warn) << "Unknown VMC engine or unimplemented VMC seeding function";
+    mSeederFcn = []() {};
+  }
+}
+
+// constructor
+VMCSeederService::VMCSeederService()
+{
+  auto vmc = TVirtualMC::GetMC();
+  if (vmc) {
+    LOG(info) << "Seeder initializing for " << vmc->GetName();
+    initSeederFunction(vmc);
+  } else {
+    LOG(fatal) << " Seeder could not be initialized (no VMC instance found)";
+  }
+}
+
+void VMCSeederService::setSeed() const
+{
+  // Make sure gRandom is sufficiently well initialized
+  // by calling Rndm() once.
+  // This is ok since in any case gRandom->SetSeed(seed); gRandom->GetSeed() != seed;
+  gRandom->Rndm();
+  mSeederFcn();
+}
diff --git a/Detectors/Base/src/bvh2_extra_kernels.h b/Detectors/Base/src/bvh2_extra_kernels.h
new file mode 100644
index 0000000000000..70e43202a53c4
--- /dev/null
+++ b/Detectors/Base/src/bvh2_extra_kernels.h
@@ -0,0 +1,79 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+// Sandro Wenzel 2026
+
+#ifndef ROOT_GEOM_BVH2_EXTRA
+
+namespace bvh::v2::extra
+{
+
+// reusable geometry kernels used in multiple places
+// for interaction with BVH2 structures
+
+// determines if a point is inside the bounding box
+template 
+bool contains(bvh::v2::BBox const& box, bvh::v2::Vec const& p)
+{
+  auto min = box.min;
+  auto max = box.max;
+  return (p[0] >= min[0] && p[0] <= max[0]) && (p[1] >= min[1] && p[1] <= max[1]) &&
+         (p[2] >= min[2] && p[2] <= max[2]);
+}
+
+// determines the largest squared distance of point to any of the bounding box corners
+template 
+auto RmaxSqToNode(bvh::v2::BBox const& box, bvh::v2::Vec const& p)
+{
+  // construct the 8 corners to get the maximal distance
+  const auto minCorner = box.min;
+  const auto maxCorner = box.max;
+  using Vec3 = bvh::v2::Vec;
+  // these are the corners of the bounding box
+  const std::array, 8> corners{
+    Vec3{minCorner[0], minCorner[1], minCorner[2]}, Vec3{minCorner[0], minCorner[1], maxCorner[2]},
+    Vec3{minCorner[0], maxCorner[1], minCorner[2]}, Vec3{minCorner[0], maxCorner[1], maxCorner[2]},
+    Vec3{maxCorner[0], minCorner[1], minCorner[2]}, Vec3{maxCorner[0], minCorner[1], maxCorner[2]},
+    Vec3{maxCorner[0], maxCorner[1], minCorner[2]}, Vec3{maxCorner[0], maxCorner[1], maxCorner[2]}};
+
+  T Rmax_sq{0};
+  for (const auto& corner : corners) {
+    float R_sq = 0.;
+    const auto dx = corner[0] - p[0];
+    R_sq += dx * dx;
+    const auto dy = corner[1] - p[1];
+    R_sq += dy * dy;
+    const auto dz = corner[2] - p[2];
+    R_sq += dz * dz;
+    Rmax_sq = std::max(Rmax_sq, R_sq);
+  }
+  return Rmax_sq;
+};
+
+// determines the minimum squared distance of point to a bounding box ("safey square")
+template 
+auto SafetySqToNode(bvh::v2::BBox const& box, bvh::v2::Vec const& p)
+{
+  T sqDist{0.0};
+  for (int i = 0; i < 3; i++) {
+    T v = p[i];
+    if (v < box.min[i]) {
+      sqDist += (box.min[i] - v) * (box.min[i] - v);
+    } else if (v > box.max[i]) {
+      sqDist += (v - box.max[i]) * (v - box.max[i]);
+    }
+  }
+  return sqDist;
+};
+
+} // namespace bvh::v2::extra
+
+#endif
\ No newline at end of file
diff --git a/Detectors/Base/src/bvh2_third_party.h b/Detectors/Base/src/bvh2_third_party.h
new file mode 100644
index 0000000000000..5cf7772269642
--- /dev/null
+++ b/Detectors/Base/src/bvh2_third_party.h
@@ -0,0 +1,49 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+// Sandro Wenzel 2026
+
+#ifndef ROOT_GEOM_BVH2_THIRD_PARTY
+
+// A single entry header into third-party BVH2 installed in ROOT
+// Good place to manage compiler warnings etc.
+
+#if defined(__clang__)
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wshadow"
+#pragma clang diagnostic ignored "-Wpsabi"
+#elif defined(__GNUC__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wshadow"
+#pragma GCC diagnostic ignored "-Wpsabi"
+#pragma GCC diagnostic ignored "-Wall"
+#pragma GCC diagnostic ignored "-Wshadow"
+#pragma GCC diagnostic ignored "-Wunknown-pragmas"
+#pragma GCC diagnostic ignored "-Wattributes"
+#elif defined(_MSC_VER)
+#pragma warning(push)
+#pragma warning(disable : 5051)
+#endif
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#if defined(__clang__)
+#pragma clang diagnostic pop
+#elif defined(__GNUC__)
+#pragma GCC diagnostic pop
+#endif
+
+#endif
\ No newline at end of file
diff --git a/Detectors/Base/test/README.md b/Detectors/Base/test/README.md
index a545d6091846d..f5f9fd4c04b29 100644
--- a/Detectors/Base/test/README.md
+++ b/Detectors/Base/test/README.md
@@ -30,3 +30,5 @@ auto mb = mbl.getMatBudget(xyz0[0],xyz0[1],xyz0[2], xyz1[0],xyz1[1],xyz1[2]);
 
 std::cout << "= " << mb.meanRho << " = " << mb.meanX2X0 << "\n";
 ```
+
+Macro `extractLUTLayers.C` can be used to extract layers covering certain radius range to obtain more compact LUT.
diff --git a/Detectors/Base/test/buildMatBudLUT.C b/Detectors/Base/test/buildMatBudLUT.C
index fe6b6b9fd7fd4..85f8343a2d35d 100644
--- a/Detectors/Base/test/buildMatBudLUT.C
+++ b/Detectors/Base/test/buildMatBudLUT.C
@@ -23,13 +23,13 @@
 #include 
 #endif
 
-#ifndef GPUCA_ALIGPUCODE // this part is unvisible on GPU version
+#ifndef GPUCA_ALIGPUCODE // this part is invisible on GPU version
 
 o2::base::MatLayerCylSet mbLUT;
 
 bool testMBLUT(const std::string& lutFile = "matbud.root");
 
-bool buildMatBudLUT(int nTst = 30, int maxLr = -1, const std::string& outFile = "matbud.root", const std::string& geomName = "");
+bool buildMatBudLUT(int nTst = 30, int maxLr = -1, const std::string& outFile = "matbud.root", const std::string& geomNamePrefix = "o2sim", const std::string& opts = "");
 
 struct LrData {
   float rMin = 0.f;
@@ -44,14 +44,17 @@ struct LrData {
 std::vector lrData;
 void configLayers();
 
-bool buildMatBudLUT(int nTst, int maxLr, const std::string& outFile, const std::string& geomNameInput)
+bool buildMatBudLUT(int nTst, int maxLr, const std::string& outFile, const std::string& geomNamePrefix, const std::string& opts)
 {
-  auto geomName = o2::base::NameConf::getGeomFileName(geomNameInput);
+  auto geomName = o2::base::NameConf::getGeomFileName(geomNamePrefix);
   if (gSystem->AccessPathName(geomName.c_str())) { // if needed, create geometry
-    std::cout << geomName << " does not exist. Will create it\n";
-    gSystem->Exec("$O2_ROOT/bin/o2-sim -n 0");
+    std::cout << geomName << " does not exist. Will create it on the fly\n";
+    std::stringstream str;
+    // constructing an **unaligned** geom (Geant3 used since faster initialization) --> can be avoided by passing an existing geometry
+    str << "${O2_ROOT}/bin/o2-sim-serial -n 0 -e TGeant3 --configKeyValues \"" << opts << "\" --field 0  -o " << geomNamePrefix;
+    gSystem->Exec(str.str().c_str());
   }
-  o2::base::GeometryManager::loadGeometry(geomNameInput);
+  o2::base::GeometryManager::loadGeometry(geomNamePrefix);
   configLayers();
 
   if (maxLr < 1) {
@@ -187,11 +190,12 @@ void configLayers()
   o2::itsmft::ChipMappingITS mp;
   int nStave = 0;
 
-  //                           rMin    rMax   zHalf
-  lrData.emplace_back(LrData(0.0f, 1.8f, 30.f));
+  //                        rMin    rMax   zHalf
+  lrData.emplace_back(LrData(0.0f, 1.8f, 50.f));
 
   // beam pipe
-  lrData.emplace_back(LrData(lrData.back().rMax, 2.0f, 30.f));
+  lrData.emplace_back(LrData(lrData.back().rMax, 1.92f, 50.f));
+  lrData.emplace_back(LrData(lrData.back().rMax, 2.2f, 50.f));
 
   // ITS Inner Barrel
   drStep = 0.1;
@@ -200,7 +204,7 @@ void configLayers()
   zBin = 0.5;
   do {
     lrData.emplace_back(LrData(lrData.back().rMax, lrData.back().rMax + drStep, zSpanH, zBin, rphiBin));
-  } while (lrData.back().rMax < 5.2 + kToler);
+  } while (lrData.back().rMax < 5 - kToler);
 
   // air space between Inner and Middle Barrels
   zSpanH = 40.;
@@ -213,19 +217,19 @@ void configLayers()
   nStave = mp.getNStavesOnLr(3); // Lr 3
   zSpanH = 55.;
   zBin = 0.5;
-  drStep = 0.2;
+  drStep = 0.3;
   do {
     auto rmean = lrData.back().rMax + drStep / 2;
     rphiBin = rmean * TMath::Pi() * 2 / (nStave * 10);
     lrData.emplace_back(LrData(lrData.back().rMax, lrData.back().rMax + drStep, zSpanH, zBin, rphiBin));
-  } while (lrData.back().rMax < 20.5 + kToler);
+  } while (lrData.back().rMax < 21.4 - kToler);
 
   drStep = 0.5;
   do {
     auto rmean = lrData.back().rMax + drStep / 2;
     rphiBin = rmean * TMath::Pi() * 2 / (nStave * 10);
     lrData.emplace_back(LrData(lrData.back().rMax, lrData.back().rMax + drStep, zSpanH, zBin, rphiBin));
-  } while (lrData.back().rMax < 24. + kToler);
+  } while (lrData.back().rMax < 23.4 - kToler);
 
   nStave = mp.getNStavesOnLr(3); // Lr 4
   drStep = 0.2;
@@ -233,19 +237,21 @@ void configLayers()
     auto rmean = lrData.back().rMax + drStep / 2;
     rphiBin = rmean * TMath::Pi() * 2 / (nStave * 10);
     lrData.emplace_back(LrData(lrData.back().rMax, lrData.back().rMax + drStep, zSpanH, zBin, rphiBin));
-  } while (lrData.back().rMax < 25.6 + kToler);
+  } while (lrData.back().rMax < 26.2 - kToler);
   drStep = 0.5;
   do {
     auto rmean = lrData.back().rMax + drStep / 2;
     rphiBin = rmean * TMath::Pi() * 2 / (nStave * 10);
     lrData.emplace_back(LrData(lrData.back().rMax, lrData.back().rMax + drStep, zSpanH, zBin, rphiBin));
-  } while (lrData.back().rMax < 29. + kToler);
+  } while (lrData.back().rMax < 29. - kToler);
 
   //===================================================================================
 
   // air space between Middle and Outer Barrels
   zSpanH = 80.f;
-  lrData.emplace_back(LrData(lrData.back().rMax, 33.5, zSpanH));
+  zBin = 10.;
+  rphiBin = lrData.back().rMax * TMath::Pi() * 2 / 18;
+  lrData.emplace_back(LrData(lrData.back().rMax, 33.5, zSpanH, zBin, rphiBin));
 
   //===================================================================================
   // ITS Outer barrel
@@ -255,55 +261,64 @@ void configLayers()
   zBin = 1.;
   do {
     auto rmean = lrData.back().rMax + drStep / 2;
-    rphiBin = rmean * TMath::Pi() * 2 / (nStave * 10);
+    rphiBin = rmean * TMath::Pi() * 2 / (nStave * 15);
     lrData.emplace_back(LrData(lrData.back().rMax, lrData.back().rMax + drStep, zSpanH, zBin, rphiBin));
-  } while (lrData.back().rMax < 36. + kToler);
+  } while (lrData.back().rMax < 36. - kToler);
 
   drStep = 1.;
   do {
     auto rmean = lrData.back().rMax + drStep / 2;
-    rphiBin = rmean * TMath::Pi() * 2 / (nStave * 10);
+    rphiBin = rmean * TMath::Pi() * 2 / (nStave * 15);
     lrData.emplace_back(LrData(lrData.back().rMax, lrData.back().rMax + drStep, zSpanH, zBin, rphiBin));
-  } while (lrData.back().rMax < 38.5 + kToler);
+  } while (lrData.back().rMax < 38.5 - kToler);
 
   nStave = mp.getNStavesOnLr(6); // Lr 6
   drStep = 0.25;
   do {
     auto rmean = lrData.back().rMax + drStep / 2;
-    rphiBin = rmean * TMath::Pi() * 2 / (nStave * 10);
+    rphiBin = rmean * TMath::Pi() * 2 / (nStave * 15);
     lrData.emplace_back(LrData(lrData.back().rMax, lrData.back().rMax + drStep, zSpanH, zBin, rphiBin));
-  } while (lrData.back().rMax < 41. + kToler);
+  } while (lrData.back().rMax < 41. - kToler);
 
   drStep = 1.;
   do {
     auto rmean = lrData.back().rMax + drStep / 2;
-    rphiBin = rmean * TMath::Pi() * 2 / (nStave * 10);
+    rphiBin = rmean * TMath::Pi() * 2 / (nStave * 15);
     lrData.emplace_back(LrData(lrData.back().rMax, lrData.back().rMax + drStep, zSpanH, zBin, rphiBin));
-  } while (lrData.back().rMax < 44. + kToler);
+  } while (lrData.back().rMax < 44. - kToler);
 
   //===================================================================================
 
   zSpanH = 100.f;
   zBin = 5.;
-  lrData.emplace_back(LrData(lrData.back().rMax, 47., zSpanH, zBin));
+  lrData.emplace_back(LrData(lrData.back().rMax, 44.8, zSpanH, zBin));
+  lrData.emplace_back(LrData(lrData.back().rMax, 46.2, zSpanH, zBin));
+  lrData.emplace_back(LrData(lrData.back().rMax, 47.0, zSpanH, zBin));
 
   drStep = 2.;
   zBin = 5.;
   rphiBin = 2.;
   do {
     lrData.emplace_back(LrData(lrData.back().rMax, lrData.back().rMax + drStep, zSpanH, zBin, rphiBin));
-  } while (lrData.back().rMax < 53. + kToler);
+  } while (lrData.back().rMax < 55. - kToler);
 
   zSpanH = 120.f;
-  lrData.emplace_back(LrData(lrData.back().rMax, 56.5, zSpanH));
+  zBin = 10.;
+  rphiBin = lrData.back().rMax * TMath::Pi() * 2 / 18;
+  lrData.emplace_back(LrData(lrData.back().rMax, 56.5, zSpanH, zBin, rphiBin));
+  rphiBin = lrData.back().rMax * TMath::Pi() * 2 / 18;
+  lrData.emplace_back(LrData(lrData.back().rMax, 60.5, zSpanH, zBin, rphiBin));
+  rphiBin = lrData.back().rMax * TMath::Pi() * 2 / 18;
+  lrData.emplace_back(LrData(lrData.back().rMax, 61.5, zSpanH, zBin, rphiBin));
 
   zSpanH = 150.f;
-  drStep = 4.;
+  drStep = 3.5;
   zBin = 15.;
-  rphiBin = 10;
   do {
+    auto rmean = lrData.back().rMax + drStep / 2;
+    rphiBin = rmean * TMath::Pi() * 2 / (NSect * 2);
     lrData.emplace_back(LrData(lrData.back().rMax, lrData.back().rMax + drStep, zSpanH, zBin, rphiBin));
-  } while (lrData.back().rMax < 68.5 + kToler);
+  } while (lrData.back().rMax < 68.5 - kToler);
 
   zSpanH = 250.f;
   zBin = 25.;
@@ -320,14 +335,14 @@ void configLayers()
   {
     auto rmean = (lrData.back().rMax + 78.5) / 2;
     rphiBin = rmean * TMath::Pi() * 2 / (NSect * 12);
-    lrData.emplace_back(LrData(lrData.back().rMax, 78.5, zSpanH, zBin, rphiBin));
+    lrData.emplace_back(LrData(lrData.back().rMax, 78.8, zSpanH, zBin, rphiBin));
   }
   //
   zSpanH = 250.f;
   zBin = 2;
   {
     auto rmean = (lrData.back().rMax + 78.5) / 2;
-    rphiBin = rmean * TMath::Pi() * 2 / (NSect * 12);
+    rphiBin = rmean * TMath::Pi() * 2 / (NSect * 24);
     lrData.emplace_back(LrData(lrData.back().rMax, 84.5, zSpanH, zBin, rphiBin));
   }
 
diff --git a/Detectors/Base/test/extractLUTLayers.C b/Detectors/Base/test/extractLUTLayers.C
new file mode 100644
index 0000000000000..113a12e6a07a5
--- /dev/null
+++ b/Detectors/Base/test/extractLUTLayers.C
@@ -0,0 +1,55 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#if !defined(__CLING__) || defined(__ROOTCLING__)
+#include "DetectorsBase/MatLayerCylSet.h"
+#include "Framework/Logger.h"
+#include "CCDB/BasicCCDBManager.h"
+#include 
+#endif
+
+// Macro to extract layers covering selected radial range into the separate LUT file.
+
+o2::base::MatLayerCylSet* extractLUTLayers(const o2::base::MatLayerCylSet* src, const std::string& outName, float rmin = 0., float rmax = 84., float toler = 1e-6)
+{
+  auto* cp = src->extractCopy(rmin, rmax, toler);
+  if (!cp) {
+    LOGP(error, "failed to extract layers for {} < r < {}", rmin, rmax);
+  }
+  if (outName.size()) {
+    cp->writeToFile(outName);
+  }
+  return cp;
+}
+
+void extractLUTLayers(const std::string& fname, float rmin = 0., float rmax = 84., float toler = 1e-6)
+{
+  const auto src = o2::base::MatLayerCylSet::loadFromFile(fname);
+  if (!src) {
+    LOGP(error, "failed to open source LUT from {}", fname);
+    return;
+  }
+  auto fnameOut = std::regex_replace(fname, std::regex(R"(.root)"), fmt::format("_r{}_{}.root", rmin, rmax));
+  auto cp = extractLUTLayers(src, fnameOut, rmin, rmax, toler);
+}
+
+void extractLUTLayers(long timestamp, float rmin = 0., float rmax = 84., float toler = 1e-6)
+{
+  auto& mg = o2::ccdb::BasicCCDBManager::instance();
+  mg.setTimestamp(timestamp);
+  const auto src = o2::base::MatLayerCylSet::rectifyPtrFromFile(mg.get("GLO/Param/MatLUT"));
+  if (!src) {
+    LOGP(error, "failed to open load LUT from CCDB for timestamp {}", timestamp);
+    return;
+  }
+  auto fnameOut = fmt::format("matbudLUT_ts{}_r{}_{}.root", timestamp, rmin, rmax);
+  auto cp = extractLUTLayers(src, fnameOut, rmin, rmax, toler);
+}
diff --git a/Detectors/Base/test/rescaleLUT.C b/Detectors/Base/test/rescaleLUT.C
new file mode 100644
index 0000000000000..9e25c796e43d1
--- /dev/null
+++ b/Detectors/Base/test/rescaleLUT.C
@@ -0,0 +1,82 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#if !defined(__CLING__) || defined(__ROOTCLING__)
+#include "DetectorsBase/MatLayerCylSet.h"
+#include "Framework/Logger.h"
+#include "CCDB/BasicCCDBManager.h"
+#include 
+#endif
+
+// Macro to extract layers covering selected radial range into the separate LUT file.
+
+void rescaleLUT(o2::base::MatLayerCylSet* src, const std::string& outName)
+{
+  struct RescRange {
+    float rMin, rMax, factor;
+  };
+  std::vector task = {
+    // put here radial ranges in increasing order with corresponding factors to rescale
+    {0.1f, 6.f, 1.05},  // e.g. rescale layers covering 0.1 rmax) {
+      LOGP(error, "rMax={:.2f} of range {} is larger then rMin={:.2f} of range {}, must be in increasing order", rmin, il - 1, rmax, il);
+      return;
+    }
+    o2::base::Ray ray(std::max(src->getRMin(), rmin), 0., 0., std::min(src->getRMax(), rmax), 0., 0.);
+    if (!src->getLayersRange(ray, lmin, lmax)) {
+      LOGP(error, "No layers found for {:.2f} < r < {:.2f}", rmin, rmax);
+      return;
+    }
+    if (lmin == lmax) {
+      LOGP(error, "rMax={:.2f} of range {} and rMin={:.2f} of range {}, correspond to the same slice {} with {:.2f}getLayer(lmin).getRMin(), src->getLayer(lmin).getRMax());
+      return;
+    }
+  }
+
+  for (size_t il = 0; il < task.size(); il++) {
+    src->scaleLayersByR(task[il].rMin, task[il].rMax, task[il].factor);
+  }
+  if (outName.size()) {
+    src->writeToFile(outName);
+  }
+}
+
+void rescaleLUT(const std::string& fname)
+{
+  auto src = o2::base::MatLayerCylSet::loadFromFile(fname);
+  if (!src) {
+    LOGP(error, "failed to open source LUT from {}", fname);
+    return;
+  }
+  auto fnameOut = std::regex_replace(fname, std::regex(R"(.root)"), "_rescaled.root");
+  rescaleLUT(src, fnameOut);
+}
+
+void rescaleLUT(long timestamp = -1)
+{
+  auto& mg = o2::ccdb::BasicCCDBManager::instance();
+  mg.setTimestamp(timestamp);
+  auto src = o2::base::MatLayerCylSet::rectifyPtrFromFile(mg.get("GLO/Param/MatLUT"));
+  if (!src) {
+    LOGP(error, "failed to open load LUT from CCDB for timestamp {}", timestamp);
+    return;
+  }
+  auto fnameOut = fmt::format("matbudLUT_ts{}_rescaled.root", timestamp);
+  rescaleLUT(src, fnameOut);
+}
diff --git a/Detectors/Base/test/testDCAFitter.cxx b/Detectors/Base/test/testDCAFitter.cxx
deleted file mode 100644
index 60e0f08f4a5a1..0000000000000
--- a/Detectors/Base/test/testDCAFitter.cxx
+++ /dev/null
@@ -1,114 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-#define BOOST_TEST_MODULE Test DCAFitter class
-#define BOOST_TEST_MAIN
-#define BOOST_TEST_DYN_LINK
-#include 
-
-#include "DetectorsBase/DCAFitter.h"
-#include 
-
-namespace o2
-{
-namespace base
-{
-
-void checkResults(const DCAFitter& fitter, const std::array xyz)
-{
-  int nCand = fitter.getNCandidates();
-  BOOST_CHECK(nCand > 0);
-  for (int ic = 0; ic < nCand; ic++) {
-    const auto& vtx = fitter.getPCACandidate(ic);
-    float dx = vtx.x - xyz[0], dy = vtx.y - xyz[1], dz = vtx.z - xyz[2];
-    float dst = TMath::Sqrt(dx * dx + dy * dy + dz * dz);
-
-    const auto &trc0 = fitter.getTrack0(ic), &trc1 = fitter.getTrack1(ic); // track parameters at V0
-    printf("Candidate %d: DCA:%+e Vtx: %+e %+e %+e [Diff to true: %+e %+e %+e -> %+e]\n",
-           ic, fitter.getChi2AtPCACandidate(ic), vtx.x, vtx.y, vtx.z, dx, dy, dz, dst);
-    printf("Track X-parameters at PCA: %+e %+e\n", trc0.getX(), trc1.getX());
-    trc0.print();
-    trc1.print();
-    BOOST_CHECK(dst < 5e-2);
-  }
-}
-
-BOOST_AUTO_TEST_CASE(PairDCAFitter)
-{
-  double bz = 5.0;
-
-  // create V0
-  std::array cv = {1e-6, 0, 1e-6, 0, 0, 1e-6, 0, 0, 0, 1e-6, 0, 0, 0, 0, 1e-5};
-  std::array pr = {0., 0., -0.2, 0.6, 1.};
-  DCAFitter::Track t0(0., 0., pr, cv);
-  t0.propagateTo(10, bz);
-
-  std::array xyz;
-  t0.getXYZGlo(xyz);
-  printf("true vertex : %+e %+e %+e\n", xyz[0], xyz[1], xyz[2]);
-
-  DCAFitter::Track tA(t0), tB(t0);
-  tB.setParam(-tA.getParam(4), 4);
-  tB.setParam(-0.5 * tA.getParam(3), 3);
-  tB.setParam(-0.2 * tA.getParam(2), 2);
-
-  tA.rotate(-0.3);
-  tB.rotate(0.3);
-
-  printf("True track params at PCA:\n");
-  tA.print();
-  tB.print();
-
-  tA.propagateTo(tA.getX() + 5., bz);
-  tB.propagateTo(tB.getX() + 8., bz);
-
-  DCAFitter df(bz, 10.);
-
-  // check with abs DCA minimization
-  {
-    df.setUseAbsDCA(true);
-    // we may supply track directly to the fitter (they are not modified)
-    int nCand = df.process(tA, tB);
-    printf("\n\nTesting with abs DCA minimization: %d candidates found\n", nCand);
-    // we can have up to 2 candidates
-    checkResults(df, xyz);
-  }
-
-  // check with weighted DCA minimization
-  {
-    df.setUseAbsDCA(false);
-    // we may supply track directly to the fitter (they are not modified)
-    DCAFitter::TrcAuxPar trcAAux(tA, bz), trcBAux(tB, bz);
-    int nCand = df.process(tA, trcAAux, tB, trcBAux);
-    printf("\n\nTesting with abs DCA minimization: %d candidates found\n", nCand);
-    // we can have up to 2 candidates
-    checkResults(df, xyz);
-  }
-
-  {
-    // direct minimization w/o preliminary propagation to XY crossing
-    df.setUseAbsDCA(true);
-    int nCand = df.processAsIs(tA, tB);
-    printf("\n\nTesting with abs DCA minimization w/o taking to XY crossing: %d candidates found\n", nCand);
-    checkResults(df, xyz);
-  }
-
-  {
-    // direct minimization w/o preliminary propagation to XY crossing (errors unreliable)
-    df.setUseAbsDCA(false);
-    int nCand = df.processAsIs(tA, tB);
-    printf("\n\nTesting with weighted DCA minimization w/o taking to XY crossing: %d candidates found\n", nCand);
-    checkResults(df, xyz);
-  }
-}
-
-} // namespace base
-} // namespace o2
diff --git a/Detectors/Base/test/testMatBudLUT.cxx b/Detectors/Base/test/testMatBudLUT.cxx
index d332422035cd6..4daa992368dba 100644
--- a/Detectors/Base/test/testMatBudLUT.cxx
+++ b/Detectors/Base/test/testMatBudLUT.cxx
@@ -13,6 +13,7 @@
 #define BOOST_TEST_MAIN
 #define BOOST_TEST_DYN_LINK
 #include 
+#include 
 
 #include "buildMatBudLUT.C"
 
@@ -22,8 +23,13 @@ BOOST_AUTO_TEST_CASE(MatBudLUT)
 {
 #ifndef GPUCA_ALIGPUCODE // this part is unvisible on GPU version
 
-  BOOST_CHECK(buildMatBudLUT(2, 20)); // generate LUT
-  BOOST_CHECK(testMBLUT());           // test LUT manipulations
+  // using process specific geometry names in order
+  // to avoid race/conditions with other tests accessing geometry
+  std::string geomPrefix("matBudGeom");
+  std::string matBudFile("matbud");
+  matBudFile += std::to_string(getpid()) + ".root";
+  BOOST_CHECK(buildMatBudLUT(2, 20, matBudFile, geomPrefix + std::to_string(getpid()), "align-geom.mDetectors=none")); // generate LUT
+  BOOST_CHECK(testMBLUT(matBudFile));                                                    // test LUT manipulations
 
 #endif //!GPUCA_ALIGPUCODE
 }
diff --git a/Detectors/Base/test/testStack.cxx b/Detectors/Base/test/testStack.cxx
new file mode 100644
index 0000000000000..150fb9515c5f1
--- /dev/null
+++ b/Detectors/Base/test/testStack.cxx
@@ -0,0 +1,46 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#define BOOST_TEST_MODULE Test MCStack class
+#define BOOST_TEST_MAIN
+#define BOOST_TEST_DYN_LINK
+#include 
+#include "DetectorsBase/Stack.h"
+#include "TFile.h"
+#include "TMCProcess.h"
+
+using namespace o2;
+
+// unit tests on MC stack
+BOOST_AUTO_TEST_CASE(Stack_test)
+{
+  o2::data::Stack st;
+  int a;
+  TMCProcess proc{kPPrimary};
+  // add a 2 primary particles
+  st.PushTrack(1, -1, 0, 0, 0., 0., 10., 5., 5., 5., 0.1, 0., 0., 0., proc, a, 1., 1);
+  st.PushTrack(1, -1, 0, 0, 0., 0., 10., 5., 5., 5., 0.1, 0., 0., 0., proc, a, 1., 1);
+  BOOST_CHECK(st.getPrimaries().size() == 2);
+
+  {
+    // serialize it
+    TFile f("StackOut.root", "RECREATE");
+    f.WriteObject(&st, "Stack");
+    f.Close();
+  }
+
+  {
+    o2::data::Stack* inst = nullptr;
+    TFile f("StackOut.root", "OPEN");
+    f.GetObject("Stack", inst);
+    BOOST_CHECK(inst->getPrimaries().size() == 2);
+  }
+}
diff --git a/Detectors/CMakeLists.txt b/Detectors/CMakeLists.txt
index 60a8adfb6465e..eef692ff18ca7 100644
--- a/Detectors/CMakeLists.txt
+++ b/Detectors/CMakeLists.txt
@@ -31,11 +31,15 @@ add_subdirectory(MUON)
 
 add_subdirectory(TPC)
 
+add_subdirectory(FOCAL)
+
 add_subdirectory(GlobalTracking)
 add_subdirectory(GlobalTrackingWorkflow)
 add_subdirectory(Vertexing)
+add_subdirectory(GLOQC)
+
 if(BUILD_ANALYSIS)
-add_subdirectory(AOD)
+  add_subdirectory(AOD)
 endif()
 
 add_subdirectory(Filtering)
@@ -44,12 +48,18 @@ add_subdirectory(Calibration)
 add_subdirectory(DCS)
 
 add_subdirectory(Align)
+add_subdirectory(ForwardAlign)
+
 
 if(BUILD_SIMULATION)
   add_subdirectory(gconfig)
   o2_data_file(COPY gconfig DESTINATION Detectors)
 endif()
 
+if(BUILD_SIMULATION)
+  add_subdirectory(O2TrivialMC)
+endif()
+
 o2_data_file(COPY Geometry DESTINATION Detectors)
 
 if(ENABLE_UPGRADES)
diff --git a/Detectors/CPV/base/src/Geometry.cxx b/Detectors/CPV/base/src/Geometry.cxx
index 0633af4c51680..08bb5504e01da 100644
--- a/Detectors/CPV/base/src/Geometry.cxx
+++ b/Detectors/CPV/base/src/Geometry.cxx
@@ -10,7 +10,7 @@
 // or submit itself to any jurisdiction.
 
 #include "CPVBase/Geometry.h"
-#include "FairLogger.h"
+#include 
 
 using namespace o2::cpv;
 
@@ -68,7 +68,8 @@ short Geometry::areNeighbours(unsigned short absId1, unsigned short absId2)
     short rowdiff = TMath::Abs(relId1[1] - relId2[1]);
     short coldiff = TMath::Abs(relId1[2] - relId2[2]);
 
-    if ((coldiff <= 1) && (rowdiff <= 1)) { // At least common vertex
+    // if ((coldiff <= 1) && (rowdiff <= 1)) { // At least common vertex
+    if (coldiff + rowdiff <= 1) { // Common side
       return 1;
     } else {
       if ((relId2[1] > relId1[1]) && (relId2[2] > relId1[2] + 1)) {
diff --git a/Detectors/CPV/calib/CPVCalibWorkflow/src/CPVBadMapCalibDevice.cxx b/Detectors/CPV/calib/CPVCalibWorkflow/src/CPVBadMapCalibDevice.cxx
index 42d0629b7313b..f2a08e280f954 100644
--- a/Detectors/CPV/calib/CPVCalibWorkflow/src/CPVBadMapCalibDevice.cxx
+++ b/Detectors/CPV/calib/CPVCalibWorkflow/src/CPVBadMapCalibDevice.cxx
@@ -14,7 +14,7 @@
 #include "CCDB/CcdbObjectInfo.h"
 #include "DetectorsCalibration/Utils.h"
 #include 
-#include "FairLogger.h"
+#include 
 #include "CommonDataFormat/InteractionRecord.h"
 #include "Framework/ConfigParamRegistry.h"
 #include "Framework/ControlService.h"
@@ -174,7 +174,7 @@ void CPVBadMapCalibDevice::sendOutput(DataAllocator& output)
     output.snapshot(Output{o2::calibration::Utils::gDataOriginCDBWrapper, "CPV_BadChanMap", subSpec}, info);
   }
 
-  output.snapshot(o2::framework::Output{"CPV", "BADMAPCHANGE", 0, o2::framework::Lifetime::Timeframe}, mMapDiff);
+  output.snapshot(o2::framework::Output{"CPV", "BADMAPCHANGE", 0}, mMapDiff);
 }
 
 bool CPVBadMapCalibDevice::differFromCurrent()
diff --git a/Detectors/CPV/calib/CPVCalibWorkflow/src/CPVGainCalibDevice.cxx b/Detectors/CPV/calib/CPVCalibWorkflow/src/CPVGainCalibDevice.cxx
index 512a4d4e864e8..d7f187e3a88cc 100644
--- a/Detectors/CPV/calib/CPVCalibWorkflow/src/CPVGainCalibDevice.cxx
+++ b/Detectors/CPV/calib/CPVCalibWorkflow/src/CPVGainCalibDevice.cxx
@@ -14,7 +14,7 @@
 #include "CCDB/CcdbObjectInfo.h"
 #include 
 #include 
-#include "FairLogger.h"
+#include 
 #include "CommonDataFormat/InteractionRecord.h"
 #include "Framework/ConfigParamRegistry.h"
 #include "Framework/ControlService.h"
@@ -143,7 +143,7 @@ void CPVGainCalibDevice::sendOutput(DataAllocator& output)
     fout.Close();
   }
   // Anyway send change to QC
-  output.snapshot(o2::framework::Output{"CPV", "GAINDIFF", 0, o2::framework::Lifetime::Timeframe}, mGainRatio);
+  output.snapshot(o2::framework::Output{"CPV", "GAINDIFF", 0}, mGainRatio);
 }
 
 void CPVGainCalibDevice::calculateGains()
diff --git a/Detectors/CPV/calib/CPVCalibWorkflow/src/CPVPedestalCalibDevice.cxx b/Detectors/CPV/calib/CPVCalibWorkflow/src/CPVPedestalCalibDevice.cxx
index a5b8e819e43ba..5dd414ab7aeb8 100644
--- a/Detectors/CPV/calib/CPVCalibWorkflow/src/CPVPedestalCalibDevice.cxx
+++ b/Detectors/CPV/calib/CPVCalibWorkflow/src/CPVPedestalCalibDevice.cxx
@@ -13,7 +13,7 @@
 #include "CCDB/CcdbApi.h"
 #include "CCDB/CcdbObjectInfo.h"
 #include 
-#include "FairLogger.h"
+#include 
 #include "CommonDataFormat/InteractionRecord.h"
 #include "Framework/ConfigParamRegistry.h"
 #include "Framework/ControlService.h"
@@ -118,7 +118,7 @@ void CPVPedestalCalibDevice::sendOutput(DataAllocator& output)
   }
   // Anyway send change to QC
   LOG(info) << "[CPVPedestalCalibDevice - run] Writing ";
-  output.snapshot(o2::framework::Output{"CPV", "PEDDIFF", 0, o2::framework::Lifetime::Timeframe}, mPedDiff);
+  output.snapshot(o2::framework::Output{"CPV", "PEDDIFF", 0}, mPedDiff);
 
   // Write pedestal distributions to calculate bad map
   std::string filename = mPath + "CPVPedestals.root";
diff --git a/Detectors/CPV/calib/include/CPVCalibration/GainCalibrator.h b/Detectors/CPV/calib/include/CPVCalibration/GainCalibrator.h
index 3a67b71240e11..8ecfa64190d0c 100644
--- a/Detectors/CPV/calib/include/CPVCalibration/GainCalibrator.h
+++ b/Detectors/CPV/calib/include/CPVCalibration/GainCalibrator.h
@@ -42,7 +42,8 @@ class AmplitudeSpectrum
   double getRMS() const { return (mSumA2 / mNEntries) - ((mSumA * mSumA) / (mNEntries * mNEntries)); }; // return RMS of distribution
   uint32_t getNEntries() const { return mNEntries; }
   const uint32_t* getBinContent() { return mBinContent.data(); } // since C++17 std::array::data is constexpr
-  void fillBinData(TH1F* h);
+  void dumpToHisto(TH1F* h);
+  int nEventsInRange(float lR, float rR);
 
  private:
   uint32_t mNEntries;
@@ -69,7 +70,7 @@ class GainCalibData
 }; // end GainCalibData
 //=============================================================================
 using GainTimeSlot = o2::calibration::TimeSlot;
-class GainCalibrator final : public o2::calibration::TimeSlotCalibration
+class GainCalibrator final : public o2::calibration::TimeSlotCalibration
 {
  public:
   GainCalibrator();
diff --git a/Detectors/CPV/calib/include/CPVCalibration/NoiseCalibrator.h b/Detectors/CPV/calib/include/CPVCalibration/NoiseCalibrator.h
index 431cb9a3c613f..92802c7f60d2b 100644
--- a/Detectors/CPV/calib/include/CPVCalibration/NoiseCalibrator.h
+++ b/Detectors/CPV/calib/include/CPVCalibration/NoiseCalibrator.h
@@ -40,7 +40,7 @@ struct NoiseCalibData {
 //=========================================================================
 
 using NoiseTimeSlot = o2::calibration::TimeSlot;
-class NoiseCalibrator final : public o2::calibration::TimeSlotCalibration
+class NoiseCalibrator final : public o2::calibration::TimeSlotCalibration
 {
  public:
   NoiseCalibrator();
@@ -48,12 +48,15 @@ class NoiseCalibrator final : public o2::calibration::TimeSlotCalibration& getCcdbInfoBadChannelMapVector() { return mCcdbInfoBadChannelMapVec; }
   const std::vector& getBadChannelMapVector() const { return mBadChannelMapVec; }
 
-  void setPedEfficiencies(std::vector* pedEffs) { mPedEfficiencies.reset(pedEffs); }
-  void setDeadChannels(std::vector* deadChs) { mDeadChannels.reset(deadChs); }
-  void setHighPedChannels(std::vector* highPeds) { mHighPedChannels.reset(highPeds); }
-  bool isSettedPedEfficiencies() { return mPedEfficiencies.get() == nullptr ? false : true; }
-  bool isSettedDeadChannels() { return mDeadChannels.get() == nullptr ? false : true; }
-  bool isSettedHighPedChannels() { return mHighPedChannels.get() == nullptr ? false : true; }
+  void setPedEfficiencies(std::vector* pedEffs) { mPedEfficiencies = pedEffs; }
+  void setDeadChannels(std::vector* deadChs) { mDeadChannels = deadChs; }
+  void setHighPedChannels(std::vector* highPeds) { mHighPedChannels = highPeds; }
+  void setPersistentBadChannels(std::vector* persBadChs) { mPersistentBadChannels = persBadChs; }
+
+  bool isSettedPedEfficiencies() { return mPedEfficiencies == nullptr ? false : true; }
+  bool isSettedDeadChannels() { return mDeadChannels == nullptr ? false : true; }
+  bool isSettedHighPedChannels() { return mHighPedChannels == nullptr ? false : true; }
+  bool isSettedPersistentBadChannels() { return mPersistentBadChannels == nullptr ? false : true; }
 
   bool hasEnoughData(const NoiseTimeSlot& slot) const final
   {
@@ -66,9 +69,10 @@ class NoiseCalibrator final : public o2::calibration::TimeSlotCalibration> mPedEfficiencies = nullptr;
-  std::unique_ptr> mDeadChannels = nullptr;
-  std::unique_ptr> mHighPedChannels = nullptr;
+  std::vector* mPedEfficiencies = nullptr;
+  std::vector* mDeadChannels = nullptr;
+  std::vector* mHighPedChannels = nullptr;
+  std::vector* mPersistentBadChannels = nullptr;
   uint32_t mMinEvents = 100;
   float mNoiseFrequencyCriteria = 0.5; // how often channel should appear to be considered as noisy
   float mToleratedChannelEfficiencyLow = 0.9;
@@ -82,4 +86,4 @@ class NoiseCalibrator final : public o2::calibration::TimeSlotCalibration;
 //===================================================================
-class PedestalCalibrator final : public o2::calibration::TimeSlotCalibration
+class PedestalCalibrator final : public o2::calibration::TimeSlotCalibration
 {
  public:
   PedestalCalibrator();
diff --git a/Detectors/CPV/calib/macros/makeBadMapFromPedestalRun.C b/Detectors/CPV/calib/macros/makeBadMapFromPedestalRun.C
index 3b5af09190fe7..9d62cf1a13baa 100644
--- a/Detectors/CPV/calib/macros/makeBadMapFromPedestalRun.C
+++ b/Detectors/CPV/calib/macros/makeBadMapFromPedestalRun.C
@@ -75,7 +75,7 @@ void makeBadMapFromPedestalRun(const char* ccdbURI = "http://ccdb-test.cern.ch:8
   }
 
   o2::ccdb::CcdbApi api;
-  map metadata; // can be empty
+  std::map metadata; // can be empty
   api.init(ccdbURI);            // or http://localhost:8080 for a local installation
   api.storeAsTFileAny(&badMap, "CPV/Calib/BadChannelMap", metadata, timeStamp, timeStamp + 31536000000);
 }
diff --git a/Detectors/CPV/calib/src/CPVCalibrationLinkDef.h b/Detectors/CPV/calib/src/CPVCalibrationLinkDef.h
index eb2baeb4f41a2..381da0029937c 100644
--- a/Detectors/CPV/calib/src/CPVCalibrationLinkDef.h
+++ b/Detectors/CPV/calib/src/CPVCalibrationLinkDef.h
@@ -18,18 +18,18 @@
 #pragma link C++ class o2::cpv::PedestalSpectrum + ;
 #pragma link C++ class o2::cpv::PedestalCalibData + ;
 #pragma link C++ class o2::calibration::TimeSlot < o2::cpv::PedestalCalibData> + ;
-#pragma link C++ class o2::calibration::TimeSlotCalibration < o2::cpv::Digit, o2::cpv::PedestalCalibData> + ;
+#pragma link C++ class o2::calibration::TimeSlotCalibration < o2::cpv::PedestalCalibData> + ;
 #pragma link C++ class o2::cpv::PedestalCalibrator + ;
 
 #pragma link C++ class o2::cpv::NoiseCalibData + ;
 #pragma link C++ class o2::calibration::TimeSlot < o2::cpv::NoiseCalibData> + ;
-#pragma link C++ class o2::calibration::TimeSlotCalibration < o2::cpv::Digit, o2::cpv::NoiseCalibData> + ;
+#pragma link C++ class o2::calibration::TimeSlotCalibration < o2::cpv::NoiseCalibData> + ;
 #pragma link C++ class o2::cpv::NoiseCalibrator + ;
 
 #pragma link C++ class o2::cpv::AmplitudeSpectrum + ;
 #pragma link C++ class o2::cpv::GainCalibData + ;
 #pragma link C++ class o2::calibration::TimeSlot < o2::cpv::GainCalibData> + ;
-#pragma link C++ class o2::calibration::TimeSlotCalibration < o2::cpv::Digit, o2::cpv::GainCalibData> + ;
+#pragma link C++ class o2::calibration::TimeSlotCalibration < o2::cpv::GainCalibData> + ;
 #pragma link C++ class o2::cpv::GainCalibrator + ;
 
 #endif
diff --git a/Detectors/CPV/calib/src/GainCalibrator.cxx b/Detectors/CPV/calib/src/GainCalibrator.cxx
index a310a41aaf4a6..36d5ddcf8956d 100644
--- a/Detectors/CPV/calib/src/GainCalibrator.cxx
+++ b/Detectors/CPV/calib/src/GainCalibrator.cxx
@@ -58,7 +58,7 @@ AmplitudeSpectrum& AmplitudeSpectrum::operator+=(const AmplitudeSpectrum& rhs)
 void AmplitudeSpectrum::fill(float amplitude)
 {
   if ((lRange <= amplitude) && (amplitude < rRange)) {
-    int bin = (amplitude - lRange) / nBins;
+    int bin = ((amplitude - lRange) / (rRange - lRange)) * nBins;
     mBinContent[bin]++;
     mNEntries++;
     mSumA += amplitude;
@@ -66,13 +66,35 @@ void AmplitudeSpectrum::fill(float amplitude)
   }
 }
 //_____________________________________________________________________________
-void AmplitudeSpectrum::fillBinData(TH1F* h)
+void AmplitudeSpectrum::dumpToHisto(TH1F* h)
 {
-  for (uint16_t i = 0; i < nBins; i++) {
-    h->SetBinContent(i + 1, float(mBinContent[i]));
-    h->SetBinError(i + 1, sqrt(float(mBinContent[i])));
+  int rebin = nBins / h->GetNbinsX();
+  if (h != nullptr) {
+    for (int iBin = 0; iBin < h->GetNbinsX(); iBin++) {
+      float binC = 0.;
+      for (uint16_t i = iBin * rebin; i < (iBin + 1) * rebin; i++) {
+        binC += mBinContent[i];
+      }
+      h->SetBinContent(iBin + 1, binC);
+    }
   }
 }
+int AmplitudeSpectrum::nEventsInRange(float lR, float rR)
+{
+  if (lR < lRange) {
+    lR = lRange;
+  }
+  if (rR > rRange) {
+    rR = rRange;
+  }
+  int first = ((lR - lRange) / (rRange - lRange)) * nBins;
+  int last = ((rR - lRange) / (rRange - lRange)) * nBins;
+  int nEvents = 0;
+  for (int i = first; i < last; i++) {
+    nEvents += mBinContent[i];
+  }
+  return nEvents;
+}
 //_____________________________________________________________________________
 // GainCalibData
 //_____________________________________________________________________________
@@ -127,6 +149,8 @@ void GainCalibrator::configParameters()
   mMaxAllowedCoeff = cpvParams.gainMaxAllowedCoeff;
   mFitRangeL = cpvParams.gainFitRangeL;
   mFitRangeR = cpvParams.gainFitRangeR;
+  mUpdateTFInterval = cpvParams.gainCheckForCalibrationInterval;
+
   // adjust fit ranges to descrete binned values
   if (mFitRangeL < AmplitudeSpectrum::lRange) {
     mFitRangeL = AmplitudeSpectrum::lRange;
@@ -134,7 +158,7 @@ void GainCalibrator::configParameters()
   if (mFitRangeR > AmplitudeSpectrum::rRange) {
     mFitRangeR = AmplitudeSpectrum::rRange;
   }
-  double binWidth = (AmplitudeSpectrum::rRange - AmplitudeSpectrum::lRange) / AmplitudeSpectrum::nBins;
+  float binWidth = (AmplitudeSpectrum::rRange - AmplitudeSpectrum::lRange) / AmplitudeSpectrum::nBins;
   mFitRangeL = AmplitudeSpectrum::lRange + std::floor((mFitRangeL - AmplitudeSpectrum::lRange) / binWidth) * binWidth;
   mFitRangeR = AmplitudeSpectrum::lRange + std::floor((mFitRangeR - AmplitudeSpectrum::lRange) / binWidth) * binWidth;
 
@@ -146,6 +170,7 @@ void GainCalibrator::configParameters()
   LOG(info) << "mMaxAllowedCoeff = " << mMaxAllowedCoeff;
   LOG(info) << "mFitRangeL = " << mFitRangeL;
   LOG(info) << "mFitRangeR = " << mFitRangeR;
+  LOG(info) << "mUpdateTFInterval =  " << mUpdateTFInterval;
 }
 //_____________________________________________________________________________
 void GainCalibrator::initOutput()
@@ -170,21 +195,27 @@ void GainCalibrator::finalizeSlot(GainTimeSlot& slot)
   TF1* fLandau = new TF1("fLandau", "landau", mFitRangeL, mFitRangeR);
   fLandau->SetParLimits(0, 0., 1.E6);
   fLandau->SetParLimits(1, mDesiredLandauMPV / mMaxAllowedCoeff, mDesiredLandauMPV / mMinAllowedCoeff);
-  fLandau->SetParLimits(1, 0., 1.E3);
+  fLandau->SetParLimits(2, 0., 1.E3);
   TH1F h("histoCPVMaxAmplSpectrum", "", AmplitudeSpectrum::nBins, AmplitudeSpectrum::lRange, AmplitudeSpectrum::rRange);
+  h.Rebin(std::ceil(mMaxAllowedCoeff)); // rebin histogram in order to avoid descrete structures in it
   // double binWidth = (AmplitudeSpectrum::rRange - AmplitudeSpectrum::lRange) / AmplitudeSpectrum::nBins;
   // size_t nBinsToFit = (mFitRangeR - mFitRangeL) / binWidth;
   // uint32_t xMin = (mFitRangeL - AmplitudeSpectrum::lRange) / binWidth;
   // uint32_t xMax = (mFitRangeR - AmplitudeSpectrum::lRange) / binWidth;
   int nCalibratedChannels = 0;
-  for (int i = 0; i < Geometry::kNCHANNELS; i++) {
+  int badChi2Channels = 0;
+  for (unsigned short i = 0; i < Geometry::kNCHANNELS; i++) {
+    newGains->setGain(i, mPreviousGains.get()->getGain(i)); // copy previous gains first
     // print some info
     if ((i % 500) == 0) {
       LOG(info) << "GainCalibrator::finalizeSlot() : checking channel " << i;
     }
-    if (slot.getContainer()->mAmplitudeSpectra[i].getNEntries() > mMinEvents) { // we are ready to fit
+    if (slot.getContainer()->mAmplitudeSpectra[i].getNEntries() > mMinEvents) {                            // we are ready to fit
+      if (slot.getContainer()->mAmplitudeSpectra[i].nEventsInRange(mFitRangeL, mFitRangeR) < mMinEvents) { // actually not enough events in fit range
+        continue;
+      }
       h.Reset();
-      slot.getContainer()->mAmplitudeSpectra[i].fillBinData(&h);
+      slot.getContainer()->mAmplitudeSpectra[i].dumpToHisto(&h);
       // set some starting values
       double mean = slot.getContainer()->mAmplitudeSpectra[i].getMean();
       double rms = slot.getContainer()->mAmplitudeSpectra[i].getRMS();
@@ -193,10 +224,17 @@ void GainCalibrator::finalizeSlot(GainTimeSlot& slot)
       auto fitResult = h.Fit(fLandau, "SQL0N", "", mFitRangeL, mFitRangeR);
       // auto fitResult = o2::math_utils::fit(nBinsToFit, &(slot.getContainer()->mAmplitudeSpectra[i].getBinContent()[xMin]), xMin, xMax, *fLandau);
       if (fitResult->Chi2() / fitResult->Ndf() > mToleratedChi2PerNDF) {
-        //  in case of bad fit -> do something sofisticated. but what?
+        badChi2Channels++;
+        // in case of bad fit -> do something sofisticated. but what?
         // continue;
-        LOG(info) << "GainCalibrator::finalizeSlot() : bad chi2/ndf in fit of spectrum in channel " << i;
-        fitResult->Print("V");
+        if (badChi2Channels < 20) {
+          LOG(info) << "GainCalibrator::finalizeSlot() : bad chi2/ndf in fit of spectrum in channel " << i;
+          fitResult->Print("V");
+        } else if (badChi2Channels == 20) {
+          LOG(info) << "GainCalibrator::finalizeSlot() : bad chi2/ndf in fit of spectrum in channel " << i;
+          fitResult->Print("V");
+          LOG(info) << "GainCalibrator::finalizeSlot() : bad chi2/ndf is reported 20 times. Muting ";
+        }
       }
       // calib coeffs are defined as mDesiredLandauMPV/actualMPV*previousCoeff
       float coeff = mDesiredLandauMPV / fLandau->GetParameter(1) * mPreviousGains.get()->getGain(i);
@@ -213,12 +251,12 @@ void GainCalibrator::finalizeSlot(GainTimeSlot& slot)
   }
   delete fLandau;
   LOG(info) << "GainCalibrator::finalizeSlot() : succesfully calibrated " << nChannelsCalibrated << "channels";
-
+  LOG(info) << "GainCalibrator::finalizeSlot() : bad chi2/ndf is occured " << badChi2Channels << "times";
   // prepare new ccdb entries for sending
   mGainsVec.push_back(*newGains);
   // metadata for o2::cpv::CalibParams
   std::map metaData;
-  auto className = o2::utils::MemFileHelper::getClassName(newGains);
+  auto className = o2::utils::MemFileHelper::getClassName(*newGains);
   auto fileName = o2::ccdb::CcdbApi::generateFileName(className);
   auto timeStamp = o2::ccdb::getCurrentTimestamp();
   mCcdbInfoGainsVec.emplace_back("CPV/Calib/Gains", className, fileName, metaData, timeStamp, timeStamp + 31536000000); // one year validity time (in milliseconds!)
diff --git a/Detectors/CPV/calib/src/NoiseCalibrator.cxx b/Detectors/CPV/calib/src/NoiseCalibrator.cxx
index 15f707f7d54eb..b783cf8401366 100644
--- a/Detectors/CPV/calib/src/NoiseCalibrator.cxx
+++ b/Detectors/CPV/calib/src/NoiseCalibrator.cxx
@@ -94,54 +94,72 @@ void NoiseCalibrator::finalizeSlot(NoiseTimeSlot& slot)
   LOG(info) << "NoiseCalibrator::finalizeSlot() : finalizing slot "
             << slot.getTFStart() << " <= TF <= " << slot.getTFEnd() << " with " << calibData->mNEvents << " events.";
   o2::cpv::BadChannelMap* badMap = new o2::cpv::BadChannelMap();
-  bool badMapBool[Geometry::kNCHANNELS] = {};
+  bool badMapBool[Geometry::kNCHANNELS] = {false};
+
+  // persistent bad channels from ccdb
+  if (mPersistentBadChannels) {
+    LOG(info) << "NoiseCalibrator::finalizeSlot() : adding " << mPersistentBadChannels->size() << " permanent bad channels";
+    for (int i = 0; i < mPersistentBadChannels->size(); i++) {
+      badMapBool[(*mPersistentBadChannels)[i]] = true;
+    }
+  }
 
   // handle data from pedestal run first
   // check pedestal efficiencies
+  int badEfficiencyChannels = 0;
   if (mPedEfficiencies) {
-    LOG(info) << "NoiseCalibrator::finalizeSlot() : checking ped efficiencies";
+    LOG(info) << "NoiseCalibrator::finalizeSlot() : checking ped efficiencies from pedestal run";
     for (int i = 0; i < Geometry::kNCHANNELS; i++) {
-      badMapBool[i] = false;
-      if ((*mPedEfficiencies.get())[i] > mToleratedChannelEfficiencyHigh ||
-          (*mPedEfficiencies.get())[i] < mToleratedChannelEfficiencyLow) {
+      if ((*mPedEfficiencies)[i] > mToleratedChannelEfficiencyHigh ||
+          (*mPedEfficiencies)[i] < mToleratedChannelEfficiencyLow) {
         badMapBool[i] = true;
+        badEfficiencyChannels++;
       }
     }
+    LOG(info) << "NoiseCalibrator::finalizeSlot() : found " << badEfficiencyChannels << " bad ped efficiency channels";
   }
 
   // check dead channels
   if (mDeadChannels) {
-    LOG(info) << "NoiseCalibrator::finalizeSlot() : checking dead channels";
-    for (unsigned int i = 0; i < mDeadChannels.get()->size(); i++) {
-      badMapBool[(*mDeadChannels.get())[i]] = true;
+    LOG(info) << "NoiseCalibrator::finalizeSlot() : adding " << mDeadChannels->size() << " dead channels from pedestal run";
+    for (int i = 0; i < mDeadChannels->size(); i++) {
+      badMapBool[(*mDeadChannels)[i]] = true;
     }
   }
 
   // check channels with very high pedestal value (> 511)
   if (mHighPedChannels) {
-    LOG(info) << "NoiseCalibrator::finalizeSlot() : checking high ped channels";
-    for (unsigned int i = 0; i < mHighPedChannels.get()->size(); i++) {
-      badMapBool[(*mHighPedChannels.get())[i]] = true;
+    LOG(info) << "NoiseCalibrator::finalizeSlot() : adding " << mHighPedChannels->size() << " high ped channels from pedestal run";
+    for (int i = 0; i < mHighPedChannels->size(); i++) {
+      badMapBool[(*mHighPedChannels)[i]] = true;
     }
   }
 
   // find noisy channels
+  int noisyChannels = 0;
+  LOG(info) << "NoiseCalibrator::finalizeSlot() : checking noisy channels";
   for (int i = 0; i < Geometry::kNCHANNELS; i++) {
     if (float(calibData->mOccupancyMap[i]) / calibData->mNEvents > mNoiseFrequencyCriteria) {
       badMapBool[i] = true;
+      noisyChannels++;
     }
   }
+  LOG(info) << "NoiseCalibrator::finalizeSlot() : found " << noisyChannels << " noisy channels";
 
   // fill BadChannelMap and send it to output
+  int totalBadChannels = 0;
   for (unsigned short i = 0; i < Geometry::kNCHANNELS; i++) {
     if (badMapBool[i]) {
       badMap->addBadChannel(i);
+      totalBadChannels++;
     }
   }
+  LOG(info) << "NoiseCalibrator::finalizeSlot() : created bad channel map with " << totalBadChannels << " bad channels in total";
+
   mBadChannelMapVec.push_back(*badMap);
   // metadata for o2::cpv::BadChannelMap
   std::map metaData;
-  auto className = o2::utils::MemFileHelper::getClassName(badMap);
+  auto className = o2::utils::MemFileHelper::getClassName(*badMap);
   auto fileName = o2::ccdb::CcdbApi::generateFileName(className);
   auto timeStamp = o2::ccdb::getCurrentTimestamp();
   mCcdbInfoBadChannelMapVec.emplace_back("CPV/Calib/BadChannelMap", className, fileName, metaData, timeStamp, timeStamp + 31536000000); // one year validity time (in milliseconds!)
diff --git a/Detectors/CPV/calib/src/PedestalCalibrator.cxx b/Detectors/CPV/calib/src/PedestalCalibrator.cxx
index 0edac9a5a17a5..6bc9efc43e49d 100644
--- a/Detectors/CPV/calib/src/PedestalCalibrator.cxx
+++ b/Detectors/CPV/calib/src/PedestalCalibrator.cxx
@@ -317,7 +317,7 @@ void PedestalCalibrator::finalizeSlot(PedestalTimeSlot& slot)
 
   // metadata for o2::cpv::Pedestals
   std::map metaData;
-  auto className = o2::utils::MemFileHelper::getClassName(peds);
+  auto className = o2::utils::MemFileHelper::getClassName(*peds);
   auto fileName = o2::ccdb::CcdbApi::generateFileName(className);
   auto timeStamp = o2::ccdb::getCurrentTimestamp();
   mCcdbInfoPedestalsVec.emplace_back("CPV/Calib/Pedestals", className, fileName, metaData, timeStamp, timeStamp + 31536000000); // one year validity time (in milliseconds!)
diff --git a/Detectors/CPV/calib/testWorkflow/GainCalibratorSpec.h b/Detectors/CPV/calib/testWorkflow/GainCalibratorSpec.h
index cdec5ab043a8f..71833d6ac9843 100644
--- a/Detectors/CPV/calib/testWorkflow/GainCalibratorSpec.h
+++ b/Detectors/CPV/calib/testWorkflow/GainCalibratorSpec.h
@@ -38,21 +38,18 @@ class CPVGainCalibratorSpec : public o2::framework::Task
   void init(o2::framework::InitContext& ic) final
   {
     o2::base::GRPGeomHelper::instance().setRequest(mCCDBRequest);
-    // auto slotL = ic.options().get("tf-per-slot");
-    // auto delay = ic.options().get("max-delay");
     auto updateInterval = ic.options().get("updateInterval"); // in TF
     bool updateAtTheEndOfRunOnly = ic.options().get("updateAtTheEndOfRunOnly");
     mCalibrator = std::make_unique();
-    mCalibrator->setSlotLength(0);
-    mCalibrator->setMaxSlotsDelay(1000);
+    mCalibrator->setSlotLength(0); // infinite TF slot
+    mCalibrator->setMaxSlotsDelay(10000);
     if (updateAtTheEndOfRunOnly) {
       mCalibrator->setUpdateAtTheEndOfRunOnly();
     }
     mCalibrator->setCheckIntervalInfiniteSlot(updateInterval);
-    mCalibrator->setUpdateTFInterval(updateInterval);
     LOG(info) << "CPVGainCalibratorSpec initialized";
-    LOG(info) << "tf-per-slot = 0 (this calibrator works only in single infinite slot mode)";
-    LOG(info) << "max-delay = 1000 (inconfigurable for this calibrator)";
+    LOG(info) << "tf-per-slot = 0 (inconfigurable, this calibrator works only in single infinite slot mode)";
+    LOG(info) << "max-delay = 10000 (inconfigurable for this calibrator)";
     LOG(info) << "updateInterval = " << updateInterval;
     LOG(info) << "updateAtTheEndOfRunOnly = " << updateAtTheEndOfRunOnly;
   }
@@ -104,7 +101,7 @@ class CPVGainCalibratorSpec : public o2::framework::Task
     auto&& digits = pc.inputs().get>("calibdigs");
 
     // fill statistics
-    LOG(info) << "Processing TF " << tfcounter << " with " << digits.size() << " digits";
+    LOG(detail) << "Processing TF " << tfcounter << " with " << digits.size() << " digits";
     mCalibrator->process(digits); // fill TimeSlot with digits from 1 event and check slots for finalization
 
     // inform about results and send output to ccdb
diff --git a/Detectors/CPV/calib/testWorkflow/NoiseCalibratorSpec.h b/Detectors/CPV/calib/testWorkflow/NoiseCalibratorSpec.h
index c03b14b1004bf..f272bf9deb3d7 100644
--- a/Detectors/CPV/calib/testWorkflow/NoiseCalibratorSpec.h
+++ b/Detectors/CPV/calib/testWorkflow/NoiseCalibratorSpec.h
@@ -59,6 +59,34 @@ class CPVNoiseCalibratorSpec : public o2::framework::Task
   void finaliseCCDB(o2::framework::ConcreteDataMatcher& matcher, void* obj) final
   {
     o2::base::GRPGeomHelper::instance().finaliseCCDB(matcher, obj);
+    if (matcher == framework::ConcreteDataMatcher("CPV", "CPV_PedEffs", 0)) {
+      LOG(info) << "NoiseCalibratorSpec::finaliseCCDB() : accessing CPV/PedestalRun/ChannelEfficiencies";
+      auto pedEffs = static_cast*>(obj);
+      mCalibrator->setPedEfficiencies(pedEffs);
+      LOG(info) << "NoiseCalibratorSpec::finaliseCCDB() : I got pedestal efficiencies vetor of size " << pedEffs->size();
+      return;
+    }
+    if (matcher == framework::ConcreteDataMatcher("CPV", "CPV_DeadChnls", 0)) {
+      LOG(info) << "NoiseCalibratorSpec::finaliseCCDB() : accessing CPV/PedestalRun/DeadChannels";
+      auto deadChs = static_cast*>(obj);
+      mCalibrator->setDeadChannels(deadChs);
+      LOG(info) << "NoiseCalibratorSpec::finaliseCCDB() : I got dead channels vetor of size " << deadChs->size();
+      return;
+    }
+    if (matcher == framework::ConcreteDataMatcher("CPV", "CPV_HighThrs", 0)) {
+      LOG(info) << "NoiseCalibratorSpec::finaliseCCDB() : accessing CPV/PedestalRun/HighPedChannels";
+      auto highPeds = static_cast*>(obj);
+      mCalibrator->setHighPedChannels(highPeds);
+      LOG(info) << "NoiseCalibratorSpec::finaliseCCDB() : I got high pedestals vetor of size " << highPeds->size();
+      return;
+    }
+    if (matcher == framework::ConcreteDataMatcher("CPV", "CPV_PersiBads", 0)) {
+      LOG(info) << "NoiseCalibratorSpec::finaliseCCDB() : accessing CPV/Config/PersistentBadChannels";
+      auto persBadChs = static_cast*>(obj);
+      mCalibrator->setPersistentBadChannels(persBadChs);
+      LOG(info) << "NoiseCalibratorSpec::finaliseCCDB() : I got persistent bad channels vector of size " << persBadChs->size();
+      return;
+    }
   }
 
   //_________________________________________________________________
@@ -68,9 +96,14 @@ class CPVNoiseCalibratorSpec : public o2::framework::Task
     o2::base::TFIDInfoHelper::fillTFIDInfo(pc, mCalibrator->getCurrentTFInfo());
     TFType tfcounter = mCalibrator->getCurrentTFInfo().startTime;
 
-    // update config
+    // fetch ccdb objects
     static bool isConfigFetched = false;
     if (!isConfigFetched) {
+      pc.inputs().get*>("pedeffs");
+      pc.inputs().get*>("persbadchs");
+      pc.inputs().get*>("deadchs");
+      pc.inputs().get*>("highpeds");
+
       LOG(info) << "NoiseCalibratorSpec::run() : fetching o2::cpv::CPVCalibParams from CCDB";
       pc.inputs().get("calibparams");
       LOG(info) << "NoiseCalibratorSpec::run() : o2::cpv::CPVCalibParams::Instance() now is following:";
@@ -79,37 +112,11 @@ class CPVNoiseCalibratorSpec : public o2::framework::Task
       isConfigFetched = true;
     }
 
-    // read pedestal efficiencies, dead and high ped channels from pedestal run
-    // do it only once as they don't change during noise scan
-    if (!mCalibrator->isSettedPedEfficiencies()) {
-      const auto pedEffs = pc.inputs().get*>("pedeffs");
-      if (pedEffs) {
-        mCalibrator->setPedEfficiencies(new std::vector(pedEffs->begin(), pedEffs->end()));
-        LOG(info) << "NoiseCalibratorSpec()::run() : I got pedestal efficiencies vetor of size " << pedEffs->size();
-      }
-    }
-    if (!mCalibrator->isSettedDeadChannels()) {
-      // const auto deadChs = o2::framework::DataRefUtils::as>>(pc.inputs().get("deadchs"));
-      const auto deadChs = pc.inputs().get*>("deadchs");
-      if (deadChs) {
-        mCalibrator->setDeadChannels(new std::vector(deadChs->begin(), deadChs->end()));
-        LOG(info) << "NoiseCalibratorSpec()::run() : I got dead channels vetor of size " << deadChs->size();
-      }
-    }
-    if (!mCalibrator->isSettedHighPedChannels()) {
-      // const auto highPeds = o2::framework::DataRefUtils::as>>(pc.inputs().get("highpeds"));
-      const auto highPeds = pc.inputs().get*>("highpeds");
-      if (highPeds) {
-        mCalibrator->setHighPedChannels(new std::vector(highPeds->begin(), highPeds->end()));
-        LOG(info) << "NoiseCalibratorSpec()::run() : I got high pedestal channels vetor of size " << highPeds->size();
-      }
-    }
-
     // process data
     auto&& digits = pc.inputs().get>("digits");
     auto&& trigrecs = pc.inputs().get>("trigrecs");
 
-    LOG(info) << "Processing TF " << tfcounter << " with " << digits.size() << " digits in " << trigrecs.size() << " trigger records.";
+    LOG(detail) << "Processing TF " << tfcounter << " with " << digits.size() << " digits in " << trigrecs.size() << " trigger records.";
     auto& slotTF = mCalibrator->getSlotForTF(tfcounter);
 
     for (auto trigrec = trigrecs.begin(); trigrec != trigrecs.end(); trigrec++) { // event loop
@@ -127,7 +134,7 @@ class CPVNoiseCalibratorSpec : public o2::framework::Task
     auto infoVecSize = mCalibrator->getCcdbInfoBadChannelMapVector().size();
     auto badMapVecSize = mCalibrator->getBadChannelMapVector().size();
     if (infoVecSize > 0) {
-      LOG(info) << "Created " << infoVecSize << " ccdb infos and " << badMapVecSize << " BadChannelMap objects for TF " << tfcounter;
+      LOG(detail) << "Created " << infoVecSize << " ccdb infos and " << badMapVecSize << " BadChannelMap objects for TF " << tfcounter;
     }
     sendOutput(pc.outputs());
   }
@@ -147,6 +154,9 @@ class CPVNoiseCalibratorSpec : public o2::framework::Task
     auto&& infoVec = mCalibrator->getCcdbInfoBadChannelMapVector(); // use non-const version as we update it
 
     assert(payloadVec.size() == infoVec.size());
+    if (payloadVec.size() == 0) { // don't need to do anything if there is nothing to send
+      return;
+    }
 
     for (uint32_t i = 0; i < payloadVec.size(); i++) {
       auto& w = infoVec[i];
@@ -174,6 +184,8 @@ DataProcessorSpec getCPVNoiseCalibratorSpec()
   inputs.emplace_back("deadchs", "CPV", "CPV_DeadChnls", 0, Lifetime::Condition, ccdbParamSpec("CPV/PedestalRun/DeadChannels"));
   inputs.emplace_back("highpeds", "CPV", "CPV_HighThrs", 0, Lifetime::Condition, ccdbParamSpec("CPV/PedestalRun/HighPedChannels"));
   inputs.emplace_back("calibparams", "CPV", "CPV_CalibPars", 0, Lifetime::Condition, ccdbParamSpec("CPV/Config/CPVCalibParams"));
+  inputs.emplace_back("persbadchs", "CPV", "CPV_PersiBads", 0, Lifetime::Condition, ccdbParamSpec("CPV/Config/PersistentBadChannels"));
+
   auto ccdbRequest = std::make_shared(true,                           // orbitResetTime
                                                                 true,                           // GRPECS=true
                                                                 false,                          // GRPLHCIF
diff --git a/Detectors/CPV/calib/testWorkflow/PedestalCalibratorSpec.h b/Detectors/CPV/calib/testWorkflow/PedestalCalibratorSpec.h
index 16d565e8e1d41..9e452e2fd8886 100644
--- a/Detectors/CPV/calib/testWorkflow/PedestalCalibratorSpec.h
+++ b/Detectors/CPV/calib/testWorkflow/PedestalCalibratorSpec.h
@@ -81,7 +81,7 @@ class CPVPedestalCalibratorSpec : public o2::framework::Task
 
     auto&& digits = pc.inputs().get>("digits");
     auto&& trigrecs = pc.inputs().get>("trigrecs");
-    LOG(info) << "Processing TF " << tfcounter << " with " << digits.size() << " digits in " << trigrecs.size() << " trigger records.";
+    LOG(detail) << "Processing TF " << tfcounter << " with " << digits.size() << " digits in " << trigrecs.size() << " trigger records.";
     auto& slotTF = mCalibrator->getSlotForTF(tfcounter);
 
     for (auto trigrec = trigrecs.begin(); trigrec != trigrecs.end(); trigrec++) { // event loop
@@ -99,7 +99,7 @@ class CPVPedestalCalibratorSpec : public o2::framework::Task
     auto infoVecSize = mCalibrator->getCcdbInfoPedestalsVector().size();
     auto pedsVecSize = mCalibrator->getPedestalsVector().size();
     if (infoVecSize > 0) {
-      LOG(info) << "Created " << infoVecSize << " ccdb infos and " << pedsVecSize << " pedestal objects for TF " << tfcounter;
+      LOG(detail) << "Created " << infoVecSize << " ccdb infos and " << pedsVecSize << " pedestal objects for TF " << tfcounter;
     }
     sendOutput(pc.outputs());
   }
diff --git a/Detectors/CPV/reconstruction/include/CPVReconstruction/CTFCoder.h b/Detectors/CPV/reconstruction/include/CPVReconstruction/CTFCoder.h
index 4bdc4234cefd7..4e259c24f44a6 100644
--- a/Detectors/CPV/reconstruction/include/CPVReconstruction/CTFCoder.h
+++ b/Detectors/CPV/reconstruction/include/CPVReconstruction/CTFCoder.h
@@ -23,7 +23,6 @@
 #include "DataFormatsCPV/CTF.h"
 #include "DetectorsCommonDataFormats/DetID.h"
 #include "DetectorsBase/CTFCoderBase.h"
-#include "rANS/rans.h"
 #include "CPVReconstruction/CTFHelper.h"
 
 class TTree;
@@ -33,10 +32,10 @@ namespace o2
 namespace cpv
 {
 
-class CTFCoder : public o2::ctf::CTFCoderBase
+class CTFCoder final : public o2::ctf::CTFCoderBase
 {
  public:
-  CTFCoder(o2::ctf::CTFCoderBase::OpType op) : o2::ctf::CTFCoderBase(op, CTF::getNBlocks(), o2::detectors::DetID::CPV) {}
+  CTFCoder(o2::ctf::CTFCoderBase::OpType op, const std::string& ctfdictOpt = "none") : o2::ctf::CTFCoderBase(op, CTF::getNBlocks(), o2::detectors::DetID::CPV, 1.f, ctfdictOpt) {}
   ~CTFCoder() final = default;
 
   /// entropy-encode data to buffer with CTF
@@ -47,27 +46,52 @@ class CTFCoder : public o2::ctf::CTFCoderBase
   template 
   o2::ctf::CTFIOSize decode(const CTF::base& ec, VTRG& trigVec, VCLUSTER& cluVec);
 
-  void createCoders(const std::vector& bufVec, o2::ctf::CTFCoderBase::OpType op) final;
+  void createCoders(const std::vector& bufVec, ctf::CTFCoderBase::OpType op) final;
 
  private:
+  template 
+  o2::ctf::CTFIOSize encode_impl(VEC& buff, const gsl::span& trigData, const gsl::span& cluData);
+
   void appendToTree(TTree& tree, CTF& ec);
   void readFromTree(TTree& tree, int entry, std::vector& trigVec, std::vector& cluVec);
+  std::vector mTrgDataFilt;
+  std::vector mClusDataFilt;
 };
 
 /// entropy-encode clusters to buffer with CTF
 template 
 o2::ctf::CTFIOSize CTFCoder::encode(VEC& buff, const gsl::span& trigData, const gsl::span& cluData)
+{
+  if (mIRFrameSelector.isSet()) { // preselect data
+    mTrgDataFilt.clear();
+    mClusDataFilt.clear();
+    for (const auto& trig : trigData) {
+      if (mIRFrameSelector.check(trig.getBCData()) >= 0) {
+        mTrgDataFilt.push_back(trig);
+        auto clusIt = cluData.begin() + trig.getFirstEntry();
+        auto& trigC = mTrgDataFilt.back();
+        trigC.setDataRange((int)mClusDataFilt.size(), trig.getNumberOfObjects());
+        std::copy(clusIt, clusIt + trig.getNumberOfObjects(), std::back_inserter(mClusDataFilt));
+      }
+    }
+    return encode_impl(buff, mTrgDataFilt, mClusDataFilt);
+  }
+  return encode_impl(buff, trigData, cluData);
+}
+
+template 
+o2::ctf::CTFIOSize CTFCoder::encode_impl(VEC& buff, const gsl::span& trigData, const gsl::span& cluData)
 {
   using MD = o2::ctf::Metadata::OptStore;
   // what to do which each field: see o2::ctd::Metadata explanation
   constexpr MD optField[CTF::getNBlocks()] = {
-    MD::EENCODE, // BLC_bcIncTrig
-    MD::EENCODE, // BLC_orbitIncTrig
-    MD::EENCODE, // BLC_entriesTrig
-    MD::EENCODE, // BLC_posX
-    MD::EENCODE, // BLC_posZ
-    MD::EENCODE, // BLC_energy
-    MD::EENCODE  // BLC_status
+    MD::EENCODE_OR_PACK, // BLC_bcIncTrig
+    MD::EENCODE_OR_PACK, // BLC_orbitIncTrig
+    MD::EENCODE_OR_PACK, // BLC_entriesTrig
+    MD::EENCODE_OR_PACK, // BLC_posX
+    MD::EENCODE_OR_PACK, // BLC_posZ
+    MD::EENCODE_OR_PACK, // BLC_energy
+    MD::EENCODE_OR_PACK  // BLC_status
   };
 
   CTFHelper helper(trigData, cluData);
@@ -81,11 +105,10 @@ o2::ctf::CTFIOSize CTFCoder::encode(VEC& buff, const gsl::spansetHeader(helper.createHeader());
   assignDictVersion(static_cast(ec->getHeader()));
-  ec->getANSHeader().majorVersion = 0;
-  ec->getANSHeader().minorVersion = 1;
+  ec->setANSHeader(mANSVersion);
   // at every encoding the buffer might be autoexpanded, so we don't work with fixed pointer ec
   o2::ctf::CTFIOSize iosize;
-#define ENCODECPV(beg, end, slot, bits) CTF::get(buff.data())->encode(beg, end, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)].get(), getMemMarginFactor());
+#define ENCODECPV(beg, end, slot, bits) CTF::get(buff.data())->encode(beg, end, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)], getMemMarginFactor());
   // clang-format off
   iosize += ENCODECPV(helper.begin_bcIncTrig(),    helper.end_bcIncTrig(),     CTF::BLC_bcIncTrig,    0);
   iosize += ENCODECPV(helper.begin_orbitIncTrig(), helper.end_orbitIncTrig(),  CTF::BLC_orbitIncTrig, 0);
@@ -109,12 +132,13 @@ o2::ctf::CTFIOSize CTFCoder::decode(const CTF::base& ec, VTRG& trigVec, VCLUSTER
   auto header = ec.getHeader();
   checkDictVersion(static_cast(header));
   ec.print(getPrefix(), mVerbosity);
-  std::vector bcInc, entries, posX, posZ;
-  std::vector orbitInc;
+  std::vector bcInc;
+  std::vector orbitInc;
+  std::vector entries, posX, posZ;
   std::vector energy, status;
 
   o2::ctf::CTFIOSize iosize;
-#define DECODECPV(part, slot) ec.decode(part, int(slot), mCoders[int(slot)].get())
+#define DECODECPV(part, slot) ec.decode(part, int(slot), mCoders[int(slot)])
   // clang-format off
   iosize += DECODECPV(bcInc,       CTF::BLC_bcIncTrig);
   iosize += DECODECPV(orbitInc,    CTF::BLC_orbitIncTrig);
@@ -132,7 +156,7 @@ o2::ctf::CTFIOSize CTFCoder::decode(const CTF::base& ec, VTRG& trigVec, VCLUSTER
 
   uint32_t firstEntry = 0, cluCount = 0;
   o2::InteractionRecord ir(header.firstBC, header.firstOrbit);
-
+  bool checkIROK = (mBCShift == 0); // need to check if CTP offset correction does not make the local time negative ?
   Cluster clu;
   for (uint32_t itrig = 0; itrig < header.nTriggers; itrig++) {
     // restore TrigRecord
@@ -142,14 +166,19 @@ o2::ctf::CTFIOSize CTFCoder::decode(const CTF::base& ec, VTRG& trigVec, VCLUSTER
     } else {
       ir.bc += bcInc[itrig];
     }
-
+    if (checkIROK || canApplyBCShift(ir)) { // correction will be ok
+      checkIROK = true;
+    } else { // correction would make IR prior to mFirstTFOrbit, skip
+      cluCount += entries[itrig];
+      continue;
+    }
     firstEntry = cluVec.size();
     for (uint16_t ic = 0; ic < entries[itrig]; ic++) {
       clu.setPacked(posX[cluCount], posZ[cluCount], energy[cluCount], status[cluCount]);
       cluVec.emplace_back(clu);
       cluCount++;
     }
-    trigVec.emplace_back(ir, firstEntry, entries[itrig]);
+    trigVec.emplace_back(ir - mBCShift, firstEntry, entries[itrig]);
   }
   assert(cluCount == header.nClusters);
   iosize.rawIn = trigVec.size() * sizeof(TriggerRecord) + cluVec.size() * sizeof(Cluster);
diff --git a/Detectors/CPV/reconstruction/include/CPVReconstruction/CTFHelper.h b/Detectors/CPV/reconstruction/include/CPVReconstruction/CTFHelper.h
index 3f000fc3db05f..0c86c7ccc7f17 100644
--- a/Detectors/CPV/reconstruction/include/CPVReconstruction/CTFHelper.h
+++ b/Detectors/CPV/reconstruction/include/CPVReconstruction/CTFHelper.h
@@ -50,7 +50,7 @@ class CTFHelper
   class _Iter
   {
    public:
-    using difference_type = int64_t;
+    using difference_type = std::ptrdiff_t;
     using value_type = T;
     using pointer = const T*;
     using reference = const T&;
@@ -59,52 +59,98 @@ class CTFHelper
     _Iter(const gsl::span& data, bool end = false) : mData(data), mIndex(end ? data.size() : 0){};
     _Iter() = default;
 
-    const I& operator++()
+    inline I& operator++() noexcept
     {
       ++mIndex;
-      return (I&)(*this);
+      return static_cast(*this);
     }
 
-    const I& operator--()
+    inline I operator++(int)
+    {
+      I res = *(static_cast(this));
+      ++mIndex;
+      return res;
+    }
+
+    inline I& operator--() noexcept
     {
       mIndex--;
-      return (I&)(*this);
+      return static_cast(*this);
+    }
+
+    inline I operator--(int)
+    {
+      I res = *(static_cast(this));
+      --mIndex;
+      return res;
+    }
+
+    I& operator+=(difference_type i) noexcept
+    {
+      mIndex += i;
+      return static_cast(*this);
     }
 
-    difference_type operator-(const I& other) const { return mIndex - other.mIndex; }
+    I operator+(difference_type i) const
+    {
+      I res = *(const_cast(static_cast(this)));
+      return res += i;
+    }
 
-    difference_type operator-(size_t idx) const { return mIndex - idx; }
+    I& operator-=(difference_type i) noexcept
+    {
+      mIndex -= i;
+      return static_cast(*this);
+    }
 
-    const I& operator-(size_t idx)
+    I operator-(difference_type i) const
     {
-      mIndex -= idx;
-      return (I&)(*this);
+      I res = *(const_cast(static_cast(this)));
+      return res -= i;
     }
 
-    bool operator!=(const I& other) const { return mIndex != other.mIndex; }
-    bool operator==(const I& other) const { return mIndex == other.mIndex; }
-    bool operator>(const I& other) const { return mIndex > other.mIndex; }
-    bool operator<(const I& other) const { return mIndex < other.mIndex; }
+    difference_type operator-(const I& other) const noexcept { return mIndex - other.mIndex; }
+
+    inline friend I operator+(difference_type i, const I& iter) { return iter + i; };
+
+    bool operator!=(const I& other) const noexcept { return mIndex != other.mIndex; }
+    bool operator==(const I& other) const noexcept { return mIndex == other.mIndex; }
+    bool operator>(const I& other) const noexcept { return mIndex > other.mIndex; }
+    bool operator<(const I& other) const noexcept { return mIndex < other.mIndex; }
+    bool operator>=(const I& other) const noexcept { return mIndex >= other.mIndex; }
+    bool operator<=(const I& other) const noexcept { return mIndex <= other.mIndex; }
 
    protected:
     gsl::span mData{};
-    size_t mIndex = 0;
+    difference_type mIndex = 0;
   };
 
   //_______________________________________________
   // BC difference wrt previous if in the same orbit, otherwise the abs.value.
   // For the very 1st entry return 0 (diff wrt 1st BC in the CTF header)
-  class Iter_bcIncTrig : public _Iter
+  class Iter_bcIncTrig : public _Iter
   {
    public:
-    using _Iter::_Iter;
+    using _Iter::_Iter;
     value_type operator*() const
     {
       if (mIndex) {
         if (mData[mIndex].getBCData().orbit == mData[mIndex - 1].getBCData().orbit) {
-          return mData[mIndex].getBCData().bc - mData[mIndex - 1].getBCData().bc;
+          return value_type(mData[mIndex].getBCData().bc - mData[mIndex - 1].getBCData().bc);
+        } else {
+          return value_type(mData[mIndex].getBCData().bc);
+        }
+      }
+      return 0;
+    }
+    value_type operator[](difference_type i) const
+    {
+      size_t id = mIndex + i;
+      if (id) {
+        if (mData[id].getBCData().orbit == mData[id - 1].getBCData().orbit) {
+          return value_type(mData[id].getBCData().bc - mData[id - 1].getBCData().bc);
         } else {
-          return mData[mIndex].getBCData().bc;
+          return value_type(mData[id].getBCData().bc);
         }
       }
       return 0;
@@ -113,11 +159,16 @@ class CTFHelper
 
   //_______________________________________________
   // Orbit difference wrt previous. For the very 1st entry return 0 (diff wrt 1st BC in the CTF header)
-  class Iter_orbitIncTrig : public _Iter
+  class Iter_orbitIncTrig : public _Iter
   {
    public:
-    using _Iter::_Iter;
-    value_type operator*() const { return mIndex ? mData[mIndex].getBCData().orbit - mData[mIndex - 1].getBCData().orbit : 0; }
+    using _Iter::_Iter;
+    value_type operator*() const { return value_type(mIndex ? mData[mIndex].getBCData().orbit - mData[mIndex - 1].getBCData().orbit : 0); }
+    value_type operator[](difference_type i) const
+    {
+      size_t id = mIndex + i;
+      return value_type(id ? mData[id].getBCData().orbit - mData[id - 1].getBCData().orbit : 0);
+    }
   };
 
   //_______________________________________________
@@ -127,6 +178,7 @@ class CTFHelper
    public:
     using _Iter::_Iter;
     value_type operator*() const { return mData[mIndex].getNumberOfObjects(); }
+    value_type operator[](difference_type i) const { return mData[mIndex + i].getNumberOfObjects(); }
   };
 
   //_______________________________________________
@@ -135,6 +187,7 @@ class CTFHelper
    public:
     using _Iter::_Iter;
     value_type operator*() const { return mData[mIndex].getPackedPosX(); }
+    value_type operator[](difference_type i) const { return mData[mIndex + i].getPackedPosX(); }
   };
 
   //_______________________________________________
@@ -143,14 +196,16 @@ class CTFHelper
    public:
     using _Iter::_Iter;
     value_type operator*() const { return mData[mIndex].getPackedPosZ(); }
+    value_type operator[](difference_type i) const { return mData[mIndex + i].getPackedPosZ(); }
   };
 
   //_______________________________________________
-  class Iter_energy : public _Iter
+  class Iter_energy : public _Iter
   {
    public:
-    using _Iter::_Iter;
+    using _Iter::_Iter;
     value_type operator*() const { return mData[mIndex].getPackedEnergy(); }
+    value_type operator[](difference_type i) const { return mData[mIndex + i].getPackedEnergy(); }
   };
 
   //_______________________________________________
@@ -159,6 +214,7 @@ class CTFHelper
    public:
     using _Iter::_Iter;
     value_type operator*() const { return mData[mIndex].getPackedClusterStatus(); }
+    value_type operator[](difference_type i) const { return mData[mIndex + i].getPackedClusterStatus(); }
   };
 
   //<<< =========================== ITERATORS ========================================
diff --git a/Detectors/CPV/reconstruction/include/CPVReconstruction/RawReaderMemory.h b/Detectors/CPV/reconstruction/include/CPVReconstruction/RawReaderMemory.h
index 1ad21619c738c..33698a788c812 100644
--- a/Detectors/CPV/reconstruction/include/CPVReconstruction/RawReaderMemory.h
+++ b/Detectors/CPV/reconstruction/include/CPVReconstruction/RawReaderMemory.h
@@ -42,7 +42,8 @@ enum RawErrorType_t {
   kEOE_HEADER_ERROR,
   kPADERROR,
   kUNKNOWN_WORD,
-  kPadAddress
+  kPadAddress,
+  kWRONG_DATAFORMAT
 };
 
 /// \class RawReaderMemory
@@ -103,6 +104,10 @@ class RawReaderMemory
   /// \return HeartBeatFrame orbit number
   uint32_t getCurrentHBFOrbit() const { return mCurrentHBFOrbit; }
 
+  /// \brief get data format from RDH
+  /// \return data format read from RDH (0x0 for RDH version < 7)
+  uint8_t getDataFormat() const { return mDataFormat; }
+
  protected:
   /// \brief Initialize the raw stream
   ///
@@ -121,6 +126,7 @@ class RawReaderMemory
   uint32_t mCurrentHBFOrbit = 0;          ///< Current orbit of HBF
   bool mStopBitWasNotFound;               ///< True if StopBit was not found but HBF orbit changed
   bool mIsJustInited = false;             ///< True if init() was just called
+  uint8_t mDataFormat = 0x0;              ///< Data format (read from RDH version >= 7)
 
   ClassDefNV(RawReaderMemory, 2);
 };
diff --git a/Detectors/CPV/reconstruction/src/CTFCoder.cxx b/Detectors/CPV/reconstruction/src/CTFCoder.cxx
index 7c94e125dc449..b86706dee6729 100644
--- a/Detectors/CPV/reconstruction/src/CTFCoder.cxx
+++ b/Detectors/CPV/reconstruction/src/CTFCoder.cxx
@@ -41,10 +41,11 @@ void CTFCoder::createCoders(const std::vector& bufVec, o2::ctf::CTFCoderBa
 {
   const auto ctf = CTF::getImage(bufVec.data());
   // just to get types
-  uint16_t bcInc = 0, entries = 0, cluPosX = 0, cluPosZ = 0;
-  uint32_t orbitInc = 0;
+  int16_t bcInc = 0;
+  int32_t orbitInc = 0;
+  uint16_t entries = 0, cluPosX = 0, cluPosZ = 0;
   uint8_t energy = 0, status = 0;
-#define MAKECODER(part, slot) createCoder(op, ctf.getFrequencyTable(slot), int(slot))
+#define MAKECODER(part, slot) createCoder(op, std::get>(ctf.getDictionary(slot, mANSVersion)), int(slot))
   // clang-format off
   MAKECODER(bcInc,      CTF::BLC_bcIncTrig);
   MAKECODER(orbitInc,   CTF::BLC_orbitIncTrig);
diff --git a/Detectors/CPV/reconstruction/src/Clusterer.cxx b/Detectors/CPV/reconstruction/src/Clusterer.cxx
index a823c11219ae5..b1d8cf5f66df2 100644
--- a/Detectors/CPV/reconstruction/src/Clusterer.cxx
+++ b/Detectors/CPV/reconstruction/src/Clusterer.cxx
@@ -22,7 +22,7 @@
 #include "CPVBase/CPVSimParams.h"
 #include "CPVBase/CPVCalibParams.h"
 #include 
-#include "FairLogger.h" // for LOG
+#include  // for LOG
 
 using namespace o2::cpv;
 
diff --git a/Detectors/CPV/reconstruction/src/FullCluster.cxx b/Detectors/CPV/reconstruction/src/FullCluster.cxx
index dbafb3b9cc864..c23830dacd64b 100644
--- a/Detectors/CPV/reconstruction/src/FullCluster.cxx
+++ b/Detectors/CPV/reconstruction/src/FullCluster.cxx
@@ -14,7 +14,7 @@
 #include "CPVBase/Geometry.h"
 #include "CPVBase/CPVSimParams.h"
 
-#include "FairLogger.h" // for LOG
+#include  // for LOG
 
 using namespace o2::cpv;
 
diff --git a/Detectors/CPV/reconstruction/src/RawDecoder.cxx b/Detectors/CPV/reconstruction/src/RawDecoder.cxx
index 468e622ae6f49..c2ee11fbe9171 100644
--- a/Detectors/CPV/reconstruction/src/RawDecoder.cxx
+++ b/Detectors/CPV/reconstruction/src/RawDecoder.cxx
@@ -8,7 +8,7 @@
 // In applying this license CERN does not waive the privileges and immunities
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
-#include 
+#include 
 #include "CPVReconstruction/RawReaderMemory.h"
 #include "CPVReconstruction/RawDecoder.h"
 #include "DataFormatsCPV/RawFormats.h"
@@ -49,6 +49,15 @@ RawErrorType_t RawDecoder::readChannels()
   // }
   // mErrors.emplace_back(-1, 0, 0, 0, kOK); //5 is non-existing link with general errors
 
+  uint8_t dataFormat = mRawReader.getDataFormat();
+  int wordLength;
+  if (dataFormat == 0x0) {
+    wordLength = 16; // 128 bits word with padding
+  } else if (dataFormat == 0x2) {
+    wordLength = 10; // 80 bits word without padding
+  } else {
+    return RawErrorType_t::kWRONG_DATAFORMAT;
+  }
   auto& payloadWords = mRawReader.getPayload();
   uint32_t wordCountFromLastHeader = 1; // header word is included
   int nDigitsAddedFromLastHeader = 0;
@@ -87,7 +96,7 @@ RawErrorType_t RawDecoder::readChannels()
       }
     } else {
       if (skipUntilNextHeader) {
-        b += 16;
+        b += wordLength;
         continue; // continue while'ing until it's not header
       }
       CpvWord word(b, e);
@@ -136,18 +145,27 @@ RawErrorType_t RawDecoder::readChannels()
           }
           isHeaderExpected = true;
         } else {
-          wordCountFromLastHeader++;
-          // error
-          if (!mIsMuteErrors) {
-            LOG(error) << "RawDecoder::readChannels() : "
-                       << "Read unknown word";
+          uint8_t unknownWord[10];
+          bool isPadding = isHeaderExpected && dataFormat == 0x2; // may this be padding?
+          for (int i = 0; i < 10 && (b + i) != e; i++) {          // read up to 10 mBytes
+            unknownWord[i] = *(b + i);
+            if (unknownWord[i] != 0xff) { // padding
+              isPadding = false;
+            }
+          }
+          if (!isPadding) { // this is unknown word error
+            if (!mIsMuteErrors) {
+              LOGF(info, "RawDecoder::readChannels() : Read unknown word  0x: %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x",
+                   unknownWord[9], unknownWord[8], unknownWord[7], unknownWord[6], unknownWord[5], unknownWord[4], unknownWord[3],
+                   unknownWord[2], unknownWord[1], unknownWord[0]);
+            }
+            mErrors.emplace_back(-1, 0, 0, 0, kUNKNOWN_WORD); // add error for non-existing row
+            wordCountFromLastHeader++;
           }
-          mErrors.emplace_back(-1, 0, 0, 0, kUNKNOWN_WORD); // add error for non-existing row
-          // what to do?
         }
       }
     }
-    b += 16;
+    b += wordLength;
   }
   mChannelsInitialized = true;
   return kOK;
diff --git a/Detectors/CPV/reconstruction/src/RawReaderMemory.cxx b/Detectors/CPV/reconstruction/src/RawReaderMemory.cxx
index 7330a125bacb3..10118dd1e68e0 100644
--- a/Detectors/CPV/reconstruction/src/RawReaderMemory.cxx
+++ b/Detectors/CPV/reconstruction/src/RawReaderMemory.cxx
@@ -11,13 +11,13 @@
 
 #include 
 #include 
-#include "FairLogger.h"
+#include 
 #include "CPVReconstruction/RawReaderMemory.h"
 #include "DetectorsRaw/RDHUtils.h"
 
 using namespace o2::cpv;
 
-using RDHDecoder = o2::raw::RDHUtils;
+using RDHUtils = o2::raw::RDHUtils;
 
 RawReaderMemory::RawReaderMemory(gsl::span rawmemory) : mRawMemoryBuffer(rawmemory)
 {
@@ -32,16 +32,12 @@ void RawReaderMemory::setRawMemory(const gsl::span rawmemory)
 
 o2::header::RDHAny RawReaderMemory::decodeRawHeader(const void* payloadwords)
 {
-  auto headerversion = RDHDecoder::getVersion(payloadwords);
-  if (headerversion == 4) {
-    return o2::header::RDHAny(*reinterpret_cast(payloadwords));
-  } else if (headerversion == 5) {
-    return o2::header::RDHAny(*reinterpret_cast(payloadwords));
-  } else if (headerversion == 6) {
-    return o2::header::RDHAny(*reinterpret_cast(payloadwords));
+  auto headerversion = RDHUtils::getVersion(payloadwords);
+  if (headerversion < RDHUtils::getVersion() || headerversion > RDHUtils::getVersion()) {
+    LOG(error) << "Wrong header version " << headerversion;
+    throw RawErrorType_t::kRDH_DECODING;
   }
-  LOG(error) << "RawReaderMemory::decodeRawHeader() : Unknown RDH version";
-  throw RawErrorType_t::kRDH_DECODING;
+  return {*reinterpret_cast(payloadwords)};
 }
 
 void RawReaderMemory::init()
@@ -70,7 +66,7 @@ RawErrorType_t RawReaderMemory::next()
         e == RawErrorType_t::kOFFSET_TO_NEXT_IS_0) { // offset to next package is 0 -> do not know how to read next
       throw e;                                       // some principal error occured -> stop reading.
     }
-    isStopBitFound = RDHDecoder::getStop(mRawHeader);
+    isStopBitFound = RDHUtils::getStop(mRawHeader);
   } while (!isStopBitFound);
 
   return RawErrorType_t::kOK;
@@ -90,25 +86,32 @@ RawErrorType_t RawReaderMemory::nextPage()
   o2::header::RDHAny rawHeader;
   try {
     rawHeader = decodeRawHeader(mRawMemoryBuffer.data() + mCurrentPosition);
-    if (RDHDecoder::getOffsetToNext(rawHeader) == 0) { // dont' know how to read next -> skip to next HBF
+    if (RDHUtils::getOffsetToNext(rawHeader) == 0) { // dont' know how to read next -> skip to next HBF
       return RawErrorType_t::kOFFSET_TO_NEXT_IS_0;
     }
-    if (RDHDecoder::getSourceID(rawHeader) != 0x8) {
+    if (RDHUtils::getSourceID(rawHeader) != 0x8) {
       // Not a CPV RDH
-      mCurrentPosition += RDHDecoder::getOffsetToNext(rawHeader); // not cpv rdh -> skip to next HBF
+      mCurrentPosition += RDHUtils::getOffsetToNext(rawHeader); // not cpv rdh -> skip to next HBF
       return RawErrorType_t::kNOT_CPV_RDH;
     }
+    // Check validity of data format
+    auto dataFormat = RDHUtils::getDataFormat(rawHeader);
+    if (dataFormat != 0x0 && dataFormat != 0x2) { // invalid data format
+      return RawErrorType_t::kRDH_INVALID;
+    }
+    // save first RDH of the HBF
     if (mIsJustInited || mStopBitWasNotFound) { // reading first time after init() or stopbit was not found
-      mCurrentHBFOrbit = RDHDecoder::getHeartBeatOrbit(rawHeader);
-      mRawHeader = rawHeader; // save RDH of first page as mRawHeader
+      mCurrentHBFOrbit = RDHUtils::getHeartBeatOrbit(rawHeader);
+      mDataFormat = dataFormat; // save data format
+      mRawHeader = rawHeader;   // save RDH of first page as mRawHeader
       mRawHeaderInitialized = true;
       mStopBitWasNotFound = false; // reset this flag as we start to read again
       mIsJustInited = false;
-    } else if (mCurrentHBFOrbit != RDHDecoder::getHeartBeatOrbit(rawHeader)) {
+    } else if (mCurrentHBFOrbit != RDHUtils::getHeartBeatOrbit(rawHeader)) {
       // next HBF started but we didn't find stop bit.
       mStopBitWasNotFound = true;
-      mCurrentPosition += RDHDecoder::getOffsetToNext(rawHeader); // moving on
-      return RawErrorType_t::kSTOPBIT_NOTFOUND;                   // Stop bit was not found -> skip to next HBF
+      mCurrentPosition += RDHUtils::getOffsetToNext(rawHeader); // moving on
+      return RawErrorType_t::kSTOPBIT_NOTFOUND;                 // Stop bit was not found -> skip to next HBF
     }
   } catch (...) {
     return RawErrorType_t::kRDH_DECODING; // this is fatal error -> skip whole TF
@@ -117,10 +120,14 @@ RawErrorType_t RawReaderMemory::nextPage()
   mRawHeaderInitialized = true;
 
   auto tmp = mRawMemoryBuffer.data();
-  int start = (mCurrentPosition + RDHDecoder::getHeaderSize(mRawHeader));
-  int end = (mCurrentPosition + RDHDecoder::getMemorySize(mRawHeader));
+  int start = (mCurrentPosition + RDHUtils::getHeaderSize(mRawHeader));
+  int end = (mCurrentPosition + RDHUtils::getMemorySize(mRawHeader));
+  if (mDataFormat == 0x2) { // remove padding
+    int padding = (end - start) % 10;
+    end -= padding;
+  }
   bool isPayloadIncomplete = false;
-  if (mCurrentPosition + RDHDecoder::getMemorySize(mRawHeader) > mRawMemoryBuffer.size()) {
+  if (mCurrentPosition + RDHUtils::getMemorySize(mRawHeader) > mRawMemoryBuffer.size()) {
     // Payload incomplete
     isPayloadIncomplete = true;
     end = mRawMemoryBuffer.size(); // OK, lets read it anyway. Maybe there still are some completed events...
@@ -130,7 +137,7 @@ RawErrorType_t RawReaderMemory::nextPage()
   }
   mPayloadInitialized = true;
 
-  mCurrentPosition += RDHDecoder::getOffsetToNext(mRawHeader); /// Assume fixed 8 kB page size
+  mCurrentPosition += RDHUtils::getOffsetToNext(mRawHeader); // Assume fixed 8 kB page size
   if (isPayloadIncomplete) {
     return RawErrorType_t::kPAYLOAD_INCOMPLETE; // skip to next HBF
   }
diff --git a/Detectors/CPV/simulation/CMakeLists.txt b/Detectors/CPV/simulation/CMakeLists.txt
index 250656cefb14e..a5c4399bcc6c2 100644
--- a/Detectors/CPV/simulation/CMakeLists.txt
+++ b/Detectors/CPV/simulation/CMakeLists.txt
@@ -22,6 +22,7 @@ o2_add_library(CPVSimulation
                                      O2::SimConfig
                                      O2::SimulationDataFormat
                                      O2::Headers
+                                     ROOT::TreePlayer
                                      O2::DetectorsRaw)
 
 o2_target_root_dictionary(CPVSimulation
diff --git a/Detectors/CPV/simulation/include/CPVSimulation/Digitizer.h b/Detectors/CPV/simulation/include/CPVSimulation/Digitizer.h
index 79fdc957fe481..56c85534c5e79 100644
--- a/Detectors/CPV/simulation/include/CPVSimulation/Digitizer.h
+++ b/Detectors/CPV/simulation/include/CPVSimulation/Digitizer.h
@@ -35,6 +35,9 @@ class Digitizer : public TObject
 
   void init();
   void finish();
+  void setPedestals(const Pedestals* peds) { mPedestals = peds; }
+  void setBadChannelMap(const BadChannelMap* bcm) { mBadMap = bcm; }
+  void setGains(const CalibParams* gains) { mGains = gains; }
 
   /// Steer conversion of hits to digits
   void processHits(const std::vector* mHits, const std::vector& digitsBg,
@@ -45,10 +48,10 @@ class Digitizer : public TObject
   float simulatePedestalNoise(int absId);
 
  private:
-  static constexpr short NCHANNELS = 23040;      //128*60*3:  toatl number of CPV channels
-  CalibParams* mCalibParams;                     /// Calibration coefficients
-  Pedestals* mPedestals;                         /// Pedestals
-  BadChannelMap* mBadMap;                        /// Bad channel map
+  static constexpr short NCHANNELS = 23040;      // 128*60*3:  toatl number of CPV channels
+  const CalibParams* mGains = nullptr;           /// Calibration coefficients
+  const Pedestals* mPedestals = nullptr;         /// Pedestals
+  const BadChannelMap* mBadMap = nullptr;        /// Bad channel map
   std::array mArrayD;          /// array of digits (for inner use)
   std::array mDigitThresholds; /// array of readout thresholds (for inner use)
   ClassDefOverride(Digitizer, 3);
diff --git a/Detectors/CPV/simulation/include/CPVSimulation/RawWriter.h b/Detectors/CPV/simulation/include/CPVSimulation/RawWriter.h
index 2d1506042b451..ff8cfd2b84de4 100644
--- a/Detectors/CPV/simulation/include/CPVSimulation/RawWriter.h
+++ b/Detectors/CPV/simulation/include/CPVSimulation/RawWriter.h
@@ -83,6 +83,8 @@ class RawWriter
   void setOutputLocation(const char* outputdir) { mOutputLocation = outputdir; }
   void setCcdbUrl(const char* ccdbUrl) { mCcdbUrl = ccdbUrl; }
   void setFileFor(FileFor_t filefor) { mFileFor = filefor; }
+  void setRDHVersion(int v) {}
+  void setDataFormat(unsigned char v) { mDataFormat = v; }
 
   void init();
   void digitsToRaw(gsl::span digits, gsl::span triggers);
@@ -101,11 +103,13 @@ class RawWriter
   Pedestals* mPedestals = nullptr;                                ///< CPV pedestals
   BadChannelMap* mBadMap = nullptr;                               ///< CPV bad channel map
   int64_t mLM_L0_delay = 15;                                      ///< LM-L0 delay
+  unsigned char mDataFormat = 0;                                  ///< data format (0 or 2)
+  int mRDHVersion = 7;                                            ///< RDH version
   std::vector mPayload[kNGBTLinks];                         ///< Preformatted payload for every link to be written
   gsl::span mDigits;                              ///< Digits input vector - must be in digitized format
   std::unique_ptr mRawWriter;             ///< Raw writer
 
-  ClassDefNV(RawWriter, 2);
+  ClassDefNV(RawWriter, 3);
 };
 
 } // namespace cpv
diff --git a/Detectors/CPV/simulation/src/Detector.cxx b/Detectors/CPV/simulation/src/Detector.cxx
index 790db690b4d9e..fda5751464f01 100644
--- a/Detectors/CPV/simulation/src/Detector.cxx
+++ b/Detectors/CPV/simulation/src/Detector.cxx
@@ -29,7 +29,7 @@
 #include "CPVSimulation/GeometryParams.h"
 
 #include "DetectorsBase/GeometryManager.h"
-#include "SimulationDataFormat/Stack.h"
+#include "DetectorsBase/Stack.h"
 
 #include 
 #include 
@@ -229,7 +229,8 @@ Bool_t Detector::ProcessHits(FairVolume* v)
   int nz3 = (cpvparam.mNgamz + 1) / 2;
   int nx3 = (cpvparam.mNgamx + 1) / 2;
 
-  TVirtualMCStack* stack = fMC->GetStack();
+  o2::data::Stack* stack = static_cast(fMC->GetStack());
+  stack->addHit(GetDetId());
   const int partID = stack->GetCurrentTrackNumber();
 
   for (int iter = 0; iter < nIter; iter++) {
diff --git a/Detectors/CPV/simulation/src/Digitizer.cxx b/Detectors/CPV/simulation/src/Digitizer.cxx
index a0822617215ef..73a265667b3a9 100644
--- a/Detectors/CPV/simulation/src/Digitizer.cxx
+++ b/Detectors/CPV/simulation/src/Digitizer.cxx
@@ -16,7 +16,7 @@
 #include "CCDB/BasicCCDBManager.h"
 
 #include 
-#include "FairLogger.h" // for LOG
+#include  // for LOG
 
 ClassImp(o2::cpv::Digitizer);
 
@@ -28,50 +28,15 @@ using namespace o2::cpv;
 //_______________________________________________________________________
 void Digitizer::init()
 {
-  LOG(info) << "CPVDigitizer::init() : CCDB Url = " << o2::base::NameConf::getCCDBServer();
-  if (o2::base::NameConf::getCCDBServer().compare("localtest") == 0) {
-    mCalibParams = new CalibParams(1); // test default calibration
-    mPedestals = new Pedestals(1);     // test default pedestals
-    mBadMap = new BadChannelMap(1);    // test default bad channels
-    LOG(info) << "[CPVDigitizer] No reading calibration from ccdb requested, set default";
-  } else {
-    auto& ccdbMgr = o2::ccdb::BasicCCDBManager::instance();
-    ccdbMgr.setCaching(true);                     //make local cache of remote objects
-    ccdbMgr.setLocalObjectValidityChecking(true); //query objects from remote site only when local one is not valid
-    // read calibration from ccdb (for now do it only at the beginning of dataprocessing)
-    // TODO: setup timestam according to anchors
-    // Do not set timestamp here: This should be set from the framework and is done via the digitizer workflow
-    // ccdbMgr.setTimestamp(o2::ccdb::getCurrentTimestamp());
-
-    LOG(info) << "CCDB: Reading o2::cpv::CalibParams from CPV/Calib/Gains";
-    mCalibParams = ccdbMgr.get("CPV/Calib/Gains");
-    if (!mCalibParams) {
-      LOG(error) << "Cannot get o2::cpv::CalibParams from CCDB. using dummy calibration!";
-      mCalibParams = new CalibParams(1);
-    }
-
-    LOG(info) << "CCDB: Reading o2::cpv::Pedestals from CPV/Calib/Pedestals";
-    mPedestals = ccdbMgr.get("CPV/Calib/Pedestals");
-    if (!mPedestals) {
-      LOG(error) << "Cannot get o2::cpv::Pedestals from CCDB. using dummy calibration!";
-      mPedestals = new Pedestals(1);
-    }
-
-    LOG(info) << "CCDB: Reading o2::cpv::BadChannelMap from CPV/Calib/BadChannelMap";
-    mBadMap = ccdbMgr.get("CPV/Calib/BadChannelMap");
-    if (!mBadMap) {
-      LOG(error) << "Cannot get o2::cpv::BadChannelMap from CCDB. using dummy calibration!";
-      mBadMap = new BadChannelMap(1);
-    }
-
-    LOG(info) << "Task configuration is done.";
+  if (!mPedestals || !mBadMap || !mGains) {
+    LOG(fatal) << "Digitizer::init() : ccdb objects were not setted yet!";
+    return;
   }
-
-  //signal thresolds for digits
-  //note that digits are calibrated objects
+  // signal thresolds for digits
+  // note that digits are calibrated objects
   for (int i = 0; i < NCHANNELS; i++) {
     mDigitThresholds[i] = o2::cpv::CPVSimParams::Instance().mZSnSigmas *
-                          mPedestals->getPedSigma(i) * mCalibParams->getGain(i);
+                          mPedestals->getPedSigma(i) * mGains->getGain(i);
   }
 }
 
@@ -96,26 +61,26 @@ void Digitizer::processHits(const std::vector* hits, const std::vector mDigitThresholds[i]) { //add noise digit if its signal > threshold
+      if (amplitude > mDigitThresholds[i]) { // add noise digit if its signal > threshold
         mArrayD[i].setAmplitude(simulatePedestalNoise(i));
         mArrayD[i].setAbsId(i);
         // mArrayD[i].setLabel(-1); // noise marking (not needed to set as all mArrayD[i] elements are just resetted)
       }
     }
-  } else {                       //if digits exist, noise is already added
-    for (auto& dBg : digitsBg) { //digits are sorted and unique
+  } else {                       // if digits exist, noise is already added
+    for (auto& dBg : digitsBg) { // digits are sorted and unique
       mArrayD[dBg.getAbsId()] = dBg;
     }
   }
 
-  //Second, add Hits
+  // Second, add Hits
   for (auto& h : *hits) {
     int i = h.GetDetectorID();
     if (mArrayD[i].getAmplitude() > 0) {
-      mArrayD[i].setAmplitude(mArrayD[i].getAmplitude() + h.GetEnergyLoss()); //if amplitude > 0 then pedestal noise is already added
+      mArrayD[i].setAmplitude(mArrayD[i].getAmplitude() + h.GetEnergyLoss()); // if amplitude > 0 then pedestal noise is already added
     } else {
       mArrayD[i].setAbsId(i);
-      mArrayD[i].setAmplitude(h.GetEnergyLoss() + simulatePedestalNoise(i)); //if not then add pedestal noise to signal
+      mArrayD[i].setAmplitude(h.GetEnergyLoss() + simulatePedestalNoise(i)); // if not then add pedestal noise to signal
     }
     if (mArrayD[i].getAmplitude() > mDigitThresholds[i]) {
       int labelIndex = mArrayD[i].getLabel();
@@ -124,7 +89,7 @@ void Digitizer::processHits(const std::vector* hits, const std::vector sp = labels.getLabels(labelIndex);
         bool found = false;
         for (MCCompLabel& te : sp) {
@@ -135,17 +100,17 @@ void Digitizer::processHits(const std::vector* hits, const std::vectorisChannelGood(i)) {
-      continue; //bad channel -> skip this digit
+      continue; // bad channel -> skip this digit
     }
     if (mArrayD[i].getAmplitude() > mDigitThresholds[i]) {
       digitsOut.push_back(mArrayD[i]);
@@ -155,9 +120,9 @@ void Digitizer::processHits(const std::vector* hits, const std::vector= NCHANNELS) {
     return 0.;
   }
-  return gRandom->Gaus(0, mPedestals->getPedSigma(absId) * mCalibParams->getGain(absId));
+  return gRandom->Gaus(0, mPedestals->getPedSigma(absId) * mGains->getGain(absId));
 }
diff --git a/Detectors/CPV/simulation/src/RawCreator.cxx b/Detectors/CPV/simulation/src/RawCreator.cxx
index 77307428a2721..84e93cb9dc8d9 100644
--- a/Detectors/CPV/simulation/src/RawCreator.cxx
+++ b/Detectors/CPV/simulation/src/RawCreator.cxx
@@ -32,6 +32,8 @@
 
 namespace bpo = boost::program_options;
 
+constexpr int DefRDHVersion = o2::raw::RDHUtils::getVersion();
+
 int main(int argc, const char** argv)
 {
   bpo::variables_map vm;
@@ -48,8 +50,10 @@ int main(int argc, const char** argv)
     add_option("help,h", "Print this help message");
     add_option("verbose,v", bpo::value()->default_value(0), "Select verbosity level [0 = no output]");
     add_option("input-file,i", bpo::value()->default_value("cpvdigits.root"), "Specifies digit input file.");
-    add_option("file-for,f", bpo::value()->default_value("all"), "single file per: all,cru,link");
+    add_option("file-for,f", bpo::value()->default_value("all"), "single file per: all,cruendpoint,link");
     add_option("output-dir,o", bpo::value()->default_value("./"), "output directory for raw data");
+    add_option("rdh-version,r", bpo::value()->default_value(DefRDHVersion), "RDH version to use");
+    add_option("enable-padding", bpo::value()->default_value(false)->implicit_value(true), "enable GBT word padding to 128 bits even for RDH V7");
     add_option("debug,d", bpo::value()->default_value(0), "Select debug output level [0 = no debug output]");
     add_option("hbfutils-config,u", bpo::value()->default_value(std::string(o2::base::NameConf::DIGITIZATIONCONFIGFILE)), "config file for HBFUtils (or none)");
     add_option("configKeyValues", bpo::value()->default_value(""), "comma-separated configKeyValues");
@@ -82,6 +86,14 @@ int main(int argc, const char** argv)
        outputdir = vm["output-dir"].as(),
        filefor = vm["file-for"].as();
 
+  auto rdhV = vm["rdh-version"].as();
+  auto enablePadding = vm["enable-padding"].as();
+
+  if (rdhV < 7 && !enablePadding) {
+    enablePadding = true;
+    LOG(info) << "padding is always ON for RDH version " << rdhV;
+  }
+
   // if needed, create output directory
   if (!std::filesystem::exists(outputdir)) {
     if (!std::filesystem::create_directories(outputdir)) {
@@ -97,7 +109,7 @@ int main(int argc, const char** argv)
   TTreeReaderValue> triggerbranch(*treereader, "CPVDigitTrigRecords");
 
   o2::cpv::RawWriter::FileFor_t granularity = o2::cpv::RawWriter::FileFor_t::kFullDet;
-  if ((filefor == "all") || (filefor == "cru")) { // CPV has only 1 cru so "all" is identical to "cru"
+  if ((filefor == "all") || (filefor == "cruendpoint")) { // CPV has only 1 cru so "all" is identical to "cruendpoint"
     granularity = o2::cpv::RawWriter::FileFor_t::kFullDet;
   } else if (filefor == "link") {
     granularity = o2::cpv::RawWriter::FileFor_t::kLink;
@@ -110,6 +122,8 @@ int main(int argc, const char** argv)
   rawwriter.setOutputLocation(outputdir.data());
   rawwriter.setFileFor(granularity);
   rawwriter.setCcdbUrl(o2::base::NameConf::getCCDBServer().c_str());
+  rawwriter.setRDHVersion(rdhV);
+  rawwriter.setDataFormat(enablePadding ? 0 : 2);
   rawwriter.init();
   rawwriter.getWriter().setContinuousReadout(grp->isDetContinuousReadOut(o2::detectors::DetID::CPV)); // must be set explicitly
 
diff --git a/Detectors/CPV/simulation/src/RawWriter.cxx b/Detectors/CPV/simulation/src/RawWriter.cxx
index 8a5e05a950b8f..bed3a4b51f4a6 100644
--- a/Detectors/CPV/simulation/src/RawWriter.cxx
+++ b/Detectors/CPV/simulation/src/RawWriter.cxx
@@ -9,7 +9,7 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 
-#include "FairLogger.h"
+#include 
 
 #include 
 #include 
@@ -31,6 +31,8 @@ void RawWriter::init()
   mRawWriter = std::make_unique(o2::header::gDataOriginCPV, true); // true = cru detector
   mRawWriter->setCarryOverCallBack(this);
   mRawWriter->setApplyCarryOverToLastPage(true);
+  mRawWriter->useRDHVersion(mRDHVersion);
+  mRawWriter->useRDHDataFormat(mDataFormat);
 
   // register all cpv links
   for (auto&& link : links) {
@@ -125,8 +127,9 @@ void RawWriter::digitsToRaw(gsl::span digitsbranch, gsl::span digitsbranch, const gsl::span trgs)
 {
-  static int nMaxGbtWordsPerPage = o2::raw::RDHUtils::MAXCRUPage / o2::raw::RDHUtils::GBTWord - 4; // 512*16/16 - 4 = 508;
-                                                                                                   // 4 gbt words are reserved for RDH
+  static int nMaxGbtWordsPerPage = o2::raw::RDHUtils::MAXCRUPage / o2::raw::RDHUtils::GBTWord128 - 4; // 512*16/16 - 4 = 508;
+                                                                                                      // 4 gbt words are reserved for RDH
+  static int nMaxCpvWordsPerPage = (mDataFormat == 2 ? (nMaxGbtWordsPerPage * 16 / 10) : nMaxGbtWordsPerPage);
 
   // clear payloads of all links
   for (auto& payload : mPayload) {
@@ -136,8 +139,8 @@ bool RawWriter::processOrbit(const gsl::span digitsbranch, const
   // we're going to prepare preformatted pages
   bool preformatted = true;
 
-  int gbtWordCounter[kNGBTLinks] = {0, 0, 0};
-  int gbtWordCounterBeforeCPVTrailer[kNGBTLinks] = {0, 0, 0};
+  int cpvWordCounter[kNGBTLinks] = {0, 0, 0};
+  int cpvWordCounterBeforeCPVTrailer[kNGBTLinks] = {0, 0, 0};
   bool isHeaderClosedWithTrailer[kNGBTLinks] = {false, false, false};
   for (auto& trg : trgs) {
     o2::InteractionRecord currentIR = trg.getBCData();
@@ -176,28 +179,40 @@ bool RawWriter::processOrbit(const gsl::span digitsbranch, const
 
     // we need to write header + at least 1 payload word + trailer
     for (int iLink = 0; iLink < kNGBTLinks; iLink++) { // looping links
-      gbtWordCounterBeforeCPVTrailer[iLink] = 0;
-      if (nMaxGbtWordsPerPage - gbtWordCounter[iLink] < 3) { // otherwise flush already prepared data to file
+      cpvWordCounterBeforeCPVTrailer[iLink] = 0;
+      if (nMaxCpvWordsPerPage - cpvWordCounter[iLink] < 3) { // write dma page to file because there are no space left for new trigger
         LOG(debug) << "RawWriter::processOrbit() : before header: adding preformatted dma page of size " << mPayload[iLink].size();
+        // add 0xff padding in case when payload is not comlete 128-bits words
+        if (mDataFormat == 2 && mPayload[iLink].size() % 16) {
+          for (int i = 0; i < 16 - (mPayload[iLink].size() % 16); i++) {
+            mPayload[iLink].push_back(char(0xff));
+          }
+        }
         mRawWriter->addData(links[iLink].feeId, links[iLink].cruId, links[iLink].linkId, links[iLink].endPointId, currentIR,
                             gsl::span(mPayload[iLink].data(), mPayload[iLink].size()), preformatted);
         mPayload[iLink].clear();
-        gbtWordCounter[iLink] = 0;
-        gbtWordCounterBeforeCPVTrailer[iLink] = 0;
+        cpvWordCounter[iLink] = 0;
+        cpvWordCounterBeforeCPVTrailer[iLink] = 0;
       }
 
       // first, header goes
       CpvHeader header(currentIR, false, false);
-      for (int i = 0; i < 16; i++) {
+      for (int i = 0; i < 10; i++) {
         mPayload[iLink].push_back(header.mBytes[i]);
       }
+      // add padding
+      if (mDataFormat == 0) {
+        for (int i = 0; i < 6; i++) {
+          mPayload[iLink].push_back(char(0));
+        }
+      }
       isHeaderClosedWithTrailer[iLink] = false;
       LOG(debug) << "RawWriter::processOrbit() : "
                  << "I wrote cpv header for orbit = " << currentIR.orbit
                  << " and BC = " << currentIR.bc;
 
-      gbtWordCounter[iLink]++;
-      gbtWordCounterBeforeCPVTrailer[iLink]++;
+      cpvWordCounter[iLink]++;
+      cpvWordCounterBeforeCPVTrailer[iLink]++;
 
       int nDigsToWriteLeft = nDigsInTrg[iLink];
 
@@ -219,32 +234,53 @@ bool RawWriter::processOrbit(const gsl::span digitsbranch, const
               nDigsToWriteLeft--;
               if (ccWordCounter % 3 == 0) { // complete 3 channels (72 bit) + CC index (8 bits) + 6 empty bits = Generate 128 bits of data
                 mPayload[iLink].push_back(ccId);
-                for (int i = 6; i--;) {
-                  mPayload[iLink].push_back(char(0));
+                if (mDataFormat == 0) { // add padding
+                  for (int i = 6; i--;) {
+                    mPayload[iLink].push_back(char(0));
+                  }
                 }
-                gbtWordCounter[iLink]++;
-                gbtWordCounterBeforeCPVTrailer[iLink]++;
-                if (nMaxGbtWordsPerPage - gbtWordCounter[iLink] == 1) {                                      // the only space for trailer left on current page
-                  CpvTrailer tr(gbtWordCounterBeforeCPVTrailer[iLink], currentIR.bc, nDigsToWriteLeft == 0); // add trailer and flush page to file
-                  for (int i = 0; i < 16; i++) {
+                cpvWordCounter[iLink]++;
+                cpvWordCounterBeforeCPVTrailer[iLink]++;
+                if (nMaxCpvWordsPerPage - cpvWordCounter[iLink] == 1) {                                      // the only space for trailer left on current page
+                  CpvTrailer tr(cpvWordCounterBeforeCPVTrailer[iLink], currentIR.bc, nDigsToWriteLeft == 0); // add trailer and flush page to file
+                  for (int i = 0; i < 10; i++) {
                     mPayload[iLink].push_back(tr.mBytes[i]);
                   }
+                  // add padding
+                  if (mDataFormat == 0) {
+                    for (int i = 0; i < 6; i++) {
+                      mPayload[iLink].push_back(char(0));
+                    }
+                  }
+
                   isHeaderClosedWithTrailer[iLink] = true;
                   LOG(debug) << "RawWriter::processOrbit() : middle of payload: adding preformatted dma page of size " << mPayload[iLink].size();
+                  // add 0xff padding in case when payload is not complete 128-bits words
+                  if (mDataFormat == 2 && mPayload[iLink].size() % 16) {
+                    for (int i = 0; i < 16 - (mPayload[iLink].size() % 16); i++) {
+                      mPayload[iLink].push_back(char(0xff));
+                    }
+                  }
                   mRawWriter->addData(links[iLink].feeId, links[iLink].cruId, links[iLink].linkId, links[iLink].endPointId, currentIR,
                                       gsl::span(mPayload[iLink].data(), mPayload[iLink].size()), preformatted);
 
                   mPayload[iLink].clear();
-                  gbtWordCounter[iLink] = 0;
-                  gbtWordCounterBeforeCPVTrailer[iLink] = 0;
+                  cpvWordCounter[iLink] = 0;
+                  cpvWordCounterBeforeCPVTrailer[iLink] = 0;
                   if (nDigsToWriteLeft) { // some digits left for writing
                     CpvHeader newHeader(currentIR, false, true);
-                    for (int i = 0; i < 16; i++) { // so put a new header and continue
+                    for (int i = 0; i < 10; i++) { // so put a new header and continue
                       mPayload[iLink].push_back(newHeader.mBytes[i]);
                     }
+                    // add padding
+                    if (mDataFormat == 0) {
+                      for (int i = 0; i < 6; i++) {
+                        mPayload[iLink].push_back(char(0));
+                      }
+                    }
                     isHeaderClosedWithTrailer[iLink] = false;
-                    gbtWordCounter[iLink]++;
-                    gbtWordCounterBeforeCPVTrailer[iLink]++;
+                    cpvWordCounter[iLink]++;
+                    cpvWordCounterBeforeCPVTrailer[iLink]++;
                   }
                 }
               }
@@ -259,42 +295,69 @@ bool RawWriter::processOrbit(const gsl::span digitsbranch, const
             ccWordCounter++;
           }
           mPayload[iLink].push_back(ccId);
-          for (int i = 6; i--;) {
-            mPayload[iLink].push_back(char(0));
+          if (mDataFormat == 0) { // add padding
+            for (int i = 6; i--;) {
+              mPayload[iLink].push_back(char(0));
+            }
           }
-          gbtWordCounter[iLink]++;
-          gbtWordCounterBeforeCPVTrailer[iLink]++;
-          if (nMaxGbtWordsPerPage - gbtWordCounter[iLink] == 1) {                                      // the only space for trailer left on current page
-            CpvTrailer tr(gbtWordCounterBeforeCPVTrailer[iLink], currentIR.bc, nDigsToWriteLeft == 0); // add trailer and flush page to file
-            for (int i = 0; i < 16; i++) {
+          cpvWordCounter[iLink]++;
+          cpvWordCounterBeforeCPVTrailer[iLink]++;
+          if (nMaxCpvWordsPerPage - cpvWordCounter[iLink] == 1) {                                      // the only space for trailer left on current page
+            CpvTrailer tr(cpvWordCounterBeforeCPVTrailer[iLink], currentIR.bc, nDigsToWriteLeft == 0); // add trailer and flush page to file
+            for (int i = 0; i < 10; i++) {
               mPayload[iLink].push_back(tr.mBytes[i]);
             }
+            if (mDataFormat == 0) { // add padding
+              for (int i = 6; i--;) {
+                mPayload[iLink].push_back(char(0));
+              }
+            }
+
             isHeaderClosedWithTrailer[iLink] = true;
             LOG(debug) << "RawWriter::processOrbit() : middle of payload (after filling empty words): adding preformatted dma page of size " << mPayload[iLink].size();
+            // add 0xff padding in case when payload is not complete 128-bits words
+            if (mDataFormat == 2 && mPayload[iLink].size() % 16) {
+              for (int i = 0; i < 16 - (mPayload[iLink].size() % 16); i++) {
+                mPayload[iLink].push_back(char(0xff));
+              }
+            }
             mRawWriter->addData(links[iLink].feeId, links[iLink].cruId, links[iLink].linkId, links[iLink].endPointId, currentIR,
                                 gsl::span(mPayload[iLink].data(), mPayload[iLink].size()), preformatted);
+
             mPayload[iLink].clear();
-            gbtWordCounter[iLink] = 0;
-            gbtWordCounterBeforeCPVTrailer[iLink] = 0;
+            cpvWordCounter[iLink] = 0;
+            cpvWordCounterBeforeCPVTrailer[iLink] = 0;
             if (nDigsToWriteLeft) {          // some digits left for writing
-              for (int i = 0; i < 16; i++) { // so put a new header and continue
+              for (int i = 0; i < 10; i++) { // so put a new header and continue
                 mPayload[iLink].push_back(header.mBytes[i]);
               }
+              if (mDataFormat == 0) { // add padding
+                for (int i = 6; i--;) {
+                  mPayload[iLink].push_back(char(0));
+                }
+              }
+
               isHeaderClosedWithTrailer[iLink] = false;
-              gbtWordCounter[iLink]++;
-              gbtWordCounterBeforeCPVTrailer[iLink]++;
+              cpvWordCounter[iLink]++;
+              cpvWordCounterBeforeCPVTrailer[iLink]++;
             }
           }
         }
       } // end of ccId cycle
       if (!isHeaderClosedWithTrailer[iLink]) {
-        CpvTrailer tr(gbtWordCounterBeforeCPVTrailer[iLink], currentIR.bc, true);
-        for (int i = 0; i < 16; i++) {
+        CpvTrailer tr(cpvWordCounterBeforeCPVTrailer[iLink], currentIR.bc, true);
+        for (int i = 0; i < 10; i++) {
           mPayload[iLink].push_back(tr.mBytes[i]);
         }
+        if (mDataFormat == 0) { // add padding
+          for (int i = 6; i--;) {
+            mPayload[iLink].push_back(char(0));
+          }
+        }
+
         isHeaderClosedWithTrailer[iLink] = true;
-        gbtWordCounterBeforeCPVTrailer[iLink] = 0;
-        gbtWordCounter[iLink]++;
+        cpvWordCounterBeforeCPVTrailer[iLink] = 0;
+        cpvWordCounter[iLink]++;
       }
     } // end of iLink cycle
   }   // end of "for (auto& trg : trgs)""
@@ -303,6 +366,12 @@ bool RawWriter::processOrbit(const gsl::span digitsbranch, const
   for (int iLink = 0; iLink < kNGBTLinks; iLink++) {
     if (mPayload[iLink].size()) {
       LOG(debug) << "RawWriter::processOrbit() : final payload: adding preformatted dma page of size " << mPayload[iLink].size();
+      // add 0xff padding in case when payload is not complete 128-bits words
+      if (mDataFormat == 2 && mPayload[iLink].size() % 16) {
+        for (int i = 0; i < 16 - (mPayload[iLink].size() % 16); i++) {
+          mPayload[iLink].push_back(char(0xff));
+        }
+      }
       mRawWriter->addData(links[iLink].feeId, links[iLink].cruId, links[iLink].linkId, links[iLink].endPointId,
                           trgs.back().getBCData(), gsl::span(mPayload[iLink].data(), mPayload[iLink].size()), preformatted);
       mPayload[iLink].clear();
diff --git a/Detectors/CPV/testsimulation/plot_clu_cpv.C b/Detectors/CPV/testsimulation/plot_clu_cpv.C
index a3d899640c1bc..e17f81156eb39 100644
--- a/Detectors/CPV/testsimulation/plot_clu_cpv.C
+++ b/Detectors/CPV/testsimulation/plot_clu_cpv.C
@@ -8,7 +8,7 @@
 #include "TH2.h"
 //#include "DataFormatsParameters/GRPObject.h"
 #include "FairFileSource.h"
-#include "FairLogger.h"
+#include 
 #include "FairRunAna.h"
 //#include "FairRuntimeDb.h"
 #include "FairParRootFileIo.h"
diff --git a/Detectors/CPV/testsimulation/plot_dig_cpv.C b/Detectors/CPV/testsimulation/plot_dig_cpv.C
index e66b4ef438221..c6116f09e6bb0 100644
--- a/Detectors/CPV/testsimulation/plot_dig_cpv.C
+++ b/Detectors/CPV/testsimulation/plot_dig_cpv.C
@@ -9,7 +9,7 @@
 #include "TH1.h"
 //#include "DataFormatsParameters/GRPObject.h"
 #include "FairFileSource.h"
-#include "FairLogger.h"
+#include 
 #include "FairRunAna.h"
 //#include "FairRuntimeDb.h"
 #include "FairParRootFileIo.h"
diff --git a/Detectors/CPV/testsimulation/plot_hit_cpv.C b/Detectors/CPV/testsimulation/plot_hit_cpv.C
index d2da2978539b2..7605b7be8c1fd 100644
--- a/Detectors/CPV/testsimulation/plot_hit_cpv.C
+++ b/Detectors/CPV/testsimulation/plot_hit_cpv.C
@@ -7,7 +7,7 @@
 #include "TH2.h"
 //#include "DataFormatsParameters/GRPObject.h"
 #include "FairFileSource.h"
-#include "FairLogger.h"
+#include 
 #include "FairRunAna.h"
 #include "FairParRootFileIo.h"
 #include "FairSystemInfo.h"
diff --git a/Detectors/CPV/workflow/CMakeLists.txt b/Detectors/CPV/workflow/CMakeLists.txt
index 04e4b075a7ec4..ba21452393b99 100644
--- a/Detectors/CPV/workflow/CMakeLists.txt
+++ b/Detectors/CPV/workflow/CMakeLists.txt
@@ -12,6 +12,8 @@
 o2_add_library(CPVWorkflow
                SOURCES src/RecoWorkflow.cxx
                        src/ReaderSpec.cxx
+                       src/DigitReaderSpec.cxx
+                       src/ClusterReaderSpec.cxx
                        src/WriterSpec.cxx
                        src/ClusterizerSpec.cxx
                        src/DigitsPrinterSpec.cxx
@@ -41,3 +43,13 @@ o2_add_executable(cluster-writer-workflow
                   COMPONENT_NAME cpv
                   SOURCES src/cluster-writer-workflow.cxx
                   PUBLIC_LINK_LIBRARIES O2::CPVWorkflow)
+
+o2_add_executable(digit-reader-workflow
+                  COMPONENT_NAME cpv
+                  SOURCES src/digit-reader-workflow.cxx
+                  PUBLIC_LINK_LIBRARIES O2::CPVWorkflow)
+
+o2_add_executable(cluster-reader-workflow
+                  COMPONENT_NAME cpv
+                  SOURCES src/cluster-reader-workflow.cxx
+                  PUBLIC_LINK_LIBRARIES O2::CPVWorkflow)
diff --git a/Detectors/CPV/workflow/include/CPVWorkflow/ClusterReaderSpec.h b/Detectors/CPV/workflow/include/CPVWorkflow/ClusterReaderSpec.h
new file mode 100644
index 0000000000000..ee1c195b0aec5
--- /dev/null
+++ b/Detectors/CPV/workflow/include/CPVWorkflow/ClusterReaderSpec.h
@@ -0,0 +1,68 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// @file   ClusterReaderSpec.h
+
+#ifndef O2_CPV_CLUSTERREADER
+#define O2_CPV_CLUSTERREADER
+
+#include "TFile.h"
+#include "TTree.h"
+
+#include "Framework/DataProcessorSpec.h"
+#include "Framework/Task.h"
+#include "Headers/DataHeader.h"
+#include "DataFormatsCPV/Cluster.h"
+#include "DataFormatsCPV/TriggerRecord.h"
+#include "SimulationDataFormat/MCCompLabel.h"
+#include "SimulationDataFormat/MCTruthContainer.h"
+
+namespace o2
+{
+namespace cpv
+{
+
+class ClusterReader : public o2::framework::Task
+{
+ public:
+  ClusterReader(bool useMC = true);
+  ~ClusterReader() override = default;
+  void init(o2::framework::InitContext& ic) final;
+  void run(o2::framework::ProcessingContext& pc) final;
+
+ protected:
+  void connectTree(const std::string& filename);
+
+  std::vector mClusters, *mClustersInp = &mClusters;
+  std::vector mTRs, *mTRsInp = &mTRs;
+  o2::dataformats::MCTruthContainer mMCTruth, *mMCTruthInp = &mMCTruth;
+
+  o2::header::DataOrigin mOrigin = o2::header::gDataOriginCPV;
+
+  bool mUseMC = true; // use MC truth
+
+  std::unique_ptr mFile;
+  std::unique_ptr mTree;
+  std::string mInputFileName = "";
+  std::string mClusterTreeName = "o2sim";
+  std::string mClusterBranchName = "CPVCluster";
+  std::string mTRBranchName = "CPVClusterTrigRec";
+  std::string mClusterMCTruthBranchName = "CPVClusterTrueMC";
+};
+
+/// create a processor spec
+/// read CPV Cluster data from a root file
+framework::DataProcessorSpec getCPVClusterReaderSpec(bool useMC = true);
+
+} // namespace cpv
+} // namespace o2
+
+#endif /* O2_CPV_CLUSTERREADER */
diff --git a/Detectors/CPV/workflow/include/CPVWorkflow/DigitReaderSpec.h b/Detectors/CPV/workflow/include/CPVWorkflow/DigitReaderSpec.h
new file mode 100644
index 0000000000000..00fce64f89125
--- /dev/null
+++ b/Detectors/CPV/workflow/include/CPVWorkflow/DigitReaderSpec.h
@@ -0,0 +1,69 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// @file   DigitReaderSpec.h
+
+#ifndef O2_CPV_DIGITREADER
+#define O2_CPV_DIGITREADER
+
+#include "TFile.h"
+#include "TTree.h"
+
+#include "Framework/DataProcessorSpec.h"
+#include "Framework/Task.h"
+#include "Headers/DataHeader.h"
+#include "DataFormatsCPV/Digit.h"
+#include "DataFormatsCPV/TriggerRecord.h"
+#include "SimulationDataFormat/MCCompLabel.h"
+#include "SimulationDataFormat/MCTruthContainer.h"
+
+namespace o2
+{
+namespace cpv
+{
+
+class DigitReader : public o2::framework::Task
+{
+ public:
+  DigitReader(bool useMC = true);
+  ~DigitReader() override = default;
+  void init(o2::framework::InitContext& ic) final;
+  void run(o2::framework::ProcessingContext& pc) final;
+
+ protected:
+  void connectTree(const std::string& filename);
+
+  std::vector mDigits, *mDigitsInp = &mDigits;
+  std::vector mTRs, *mTRsInp = &mTRs;
+  // std::vector mMCTruth, *mMCTruthInp = &mMCTruth;
+  o2::dataformats::MCTruthContainer mMCTruth, *mMCTruthInp = &mMCTruth;
+
+  o2::header::DataOrigin mOrigin = o2::header::gDataOriginCPV;
+
+  bool mUseMC = true; // use MC truth
+
+  std::unique_ptr mFile;
+  std::unique_ptr mTree;
+  std::string mInputFileName = "";
+  std::string mDigitTreeName = "o2sim";
+  std::string mDigitBranchName = "CPVDigit";
+  std::string mTRBranchName = "CPVDigitTrigRecords";
+  std::string mDigitMCTruthBranchName = "CPVDigitMCTruth";
+};
+
+/// create a processor spec
+/// read CPV Digit data from a root file
+framework::DataProcessorSpec getCPVDigitReaderSpec(bool useMC = true);
+
+} // namespace cpv
+} // namespace o2
+
+#endif /* O2_CPV_DIGITREADER */
diff --git a/Detectors/CPV/workflow/include/CPVWorkflow/EntropyDecoderSpec.h b/Detectors/CPV/workflow/include/CPVWorkflow/EntropyDecoderSpec.h
index 09de778360d74..7192b1b2f6353 100644
--- a/Detectors/CPV/workflow/include/CPVWorkflow/EntropyDecoderSpec.h
+++ b/Detectors/CPV/workflow/include/CPVWorkflow/EntropyDecoderSpec.h
@@ -28,7 +28,7 @@ namespace cpv
 class EntropyDecoderSpec : public o2::framework::Task
 {
  public:
-  EntropyDecoderSpec(int verbosity);
+  EntropyDecoderSpec(int verbosity, const std::string& ctfdictOpt = "none");
   ~EntropyDecoderSpec() override = default;
   void run(o2::framework::ProcessingContext& pc) final;
   void init(o2::framework::InitContext& ic) final;
@@ -41,7 +41,7 @@ class EntropyDecoderSpec : public o2::framework::Task
 };
 
 /// create a processor spec
-framework::DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec);
+framework::DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec, const std::string& ctfdictOpt);
 
 } // namespace cpv
 } // namespace o2
diff --git a/Detectors/CPV/workflow/include/CPVWorkflow/EntropyEncoderSpec.h b/Detectors/CPV/workflow/include/CPVWorkflow/EntropyEncoderSpec.h
index 4528ee1528752..a1851ebb97377 100644
--- a/Detectors/CPV/workflow/include/CPVWorkflow/EntropyEncoderSpec.h
+++ b/Detectors/CPV/workflow/include/CPVWorkflow/EntropyEncoderSpec.h
@@ -28,7 +28,7 @@ namespace cpv
 class EntropyEncoderSpec : public o2::framework::Task
 {
  public:
-  EntropyEncoderSpec();
+  EntropyEncoderSpec(bool selIR = false, const std::string& ctfdictOpt = "none");
   ~EntropyEncoderSpec() override = default;
   void run(o2::framework::ProcessingContext& pc) final;
   void init(o2::framework::InitContext& ic) final;
@@ -37,11 +37,12 @@ class EntropyEncoderSpec : public o2::framework::Task
 
  private:
   o2::cpv::CTFCoder mCTFCoder;
+  bool mSelIR = false;
   TStopwatch mTimer;
 };
 
 /// create a processor spec
-framework::DataProcessorSpec getEntropyEncoderSpec();
+framework::DataProcessorSpec getEntropyEncoderSpec(bool selIR = false, const std::string& ctfdictOpt = "none");
 
 } // namespace cpv
 } // namespace o2
diff --git a/Detectors/CPV/workflow/src/ClusterReaderSpec.cxx b/Detectors/CPV/workflow/src/ClusterReaderSpec.cxx
new file mode 100644
index 0000000000000..f9d0817325c36
--- /dev/null
+++ b/Detectors/CPV/workflow/src/ClusterReaderSpec.cxx
@@ -0,0 +1,100 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// @file   ClusterReaderSpec.cxx
+
+#include 
+#include 
+#include "Framework/ControlService.h"
+#include "Framework/ConfigParamRegistry.h"
+#include "CPVWorkflow/ClusterReaderSpec.h"
+#include "CommonUtils/NameConf.h"
+
+using namespace o2::framework;
+using namespace o2::cpv;
+
+namespace o2
+{
+namespace cpv
+{
+
+ClusterReader::ClusterReader(bool useMC)
+{
+  mUseMC = useMC;
+}
+
+void ClusterReader::init(InitContext& ic)
+{
+  mInputFileName = o2::utils::Str::concat_string(o2::utils::Str::rectifyDirectory(ic.options().get("input-dir")),
+                                                 ic.options().get("cpv-clusters-infile"));
+  connectTree(mInputFileName);
+}
+
+void ClusterReader::run(ProcessingContext& pc)
+{
+  auto ent = mTree->GetReadEntry() + 1;
+  assert(ent < mTree->GetEntries()); // this should not happen
+  mTree->GetEntry(ent);
+  LOG(info) << "Pushing " << mClusters.size() << " Clusters in " << mTRs.size() << " TriggerRecords at entry " << ent;
+  pc.outputs().snapshot(Output{mOrigin, "CLUSTERS", 0}, mClusters);
+  pc.outputs().snapshot(Output{mOrigin, "CLUSTERTRIGRECS", 0}, mTRs);
+  if (mUseMC) {
+    pc.outputs().snapshot(Output{mOrigin, "CLUSTERTRUEMC", 0}, mMCTruth);
+  }
+
+  if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
+    pc.services().get().endOfStream();
+    pc.services().get().readyToQuit(QuitRequest::Me);
+  }
+}
+
+void ClusterReader::connectTree(const std::string& filename)
+{
+  mTree.reset(nullptr); // in case it was already loaded
+  mFile.reset(TFile::Open(filename.c_str()));
+  assert(mFile && !mFile->IsZombie());
+  mTree.reset((TTree*)mFile->Get(mClusterTreeName.c_str()));
+  assert(mTree);
+  assert(mTree->GetBranch(mTRBranchName.c_str()));
+
+  mTree->SetBranchAddress(mTRBranchName.c_str(), &mTRsInp);
+  mTree->SetBranchAddress(mClusterBranchName.c_str(), &mClustersInp);
+  if (mUseMC) {
+    if (mTree->GetBranch(mClusterMCTruthBranchName.c_str())) {
+      mTree->SetBranchAddress(mClusterMCTruthBranchName.c_str(), &mMCTruthInp);
+    } else {
+      LOG(warning) << "MC-truth is missing, message will be empty";
+    }
+  }
+  LOG(info) << "Loaded tree from " << filename << " with " << mTree->GetEntries() << " entries";
+}
+
+DataProcessorSpec getCPVClusterReaderSpec(bool useMC)
+{
+  std::vector outputSpec;
+  outputSpec.emplace_back("CPV", "CLUSTERS", 0, Lifetime::Timeframe);
+  outputSpec.emplace_back("CPV", "CLUSTERTRIGRECS", 0, Lifetime::Timeframe);
+  if (useMC) {
+    outputSpec.emplace_back("CPV", "CLUSTERTRUEMC", 0, Lifetime::Timeframe);
+  }
+
+  return DataProcessorSpec{
+    "cpv-cluster-reader",
+    Inputs{},
+    outputSpec,
+    AlgorithmSpec{adaptFromTask(useMC)},
+    Options{
+      {"cpv-clusters-infile", VariantType::String, "cpvclusters.root", {"Name of the input Cluster file"}},
+      {"input-dir", VariantType::String, "none", {"Input directory"}}}};
+}
+
+} // namespace cpv
+} // namespace o2
diff --git a/Detectors/CPV/workflow/src/ClusterizerSpec.cxx b/Detectors/CPV/workflow/src/ClusterizerSpec.cxx
index 60c9d176ec634..e004c3cec8949 100644
--- a/Detectors/CPV/workflow/src/ClusterizerSpec.cxx
+++ b/Detectors/CPV/workflow/src/ClusterizerSpec.cxx
@@ -8,7 +8,7 @@
 // In applying this license CERN does not waive the privileges and immunities
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
-#include "FairLogger.h"
+#include 
 
 #include "DataFormatsCPV/Digit.h"
 #include "DataFormatsCPV/Cluster.h"
@@ -49,14 +49,14 @@ void ClusterizerSpec::run(framework::ProcessingContext& ctx)
   if (!digits.size()) { // nothing to process
     LOG(info) << "ClusterizerSpec::run() : no digits; moving on";
     mOutputClusters.clear();
-    ctx.outputs().snapshot(o2::framework::Output{"CPV", "CLUSTERS", 0, o2::framework::Lifetime::Timeframe}, mOutputClusters);
+    ctx.outputs().snapshot(o2::framework::Output{"CPV", "CLUSTERS", 0}, mOutputClusters);
     mOutputClusterTrigRecs.clear();
-    ctx.outputs().snapshot(o2::framework::Output{"CPV", "CLUSTERTRIGRECS", 0, o2::framework::Lifetime::Timeframe}, mOutputClusterTrigRecs);
+    ctx.outputs().snapshot(o2::framework::Output{"CPV", "CLUSTERTRIGRECS", 0}, mOutputClusterTrigRecs);
     mCalibDigits.clear();
-    ctx.outputs().snapshot(o2::framework::Output{"CPV", "CALIBDIGITS", 0, o2::framework::Lifetime::Timeframe}, mCalibDigits);
+    ctx.outputs().snapshot(o2::framework::Output{"CPV", "CALIBDIGITS", 0}, mCalibDigits);
     if (mPropagateMC) {
       mOutputTruthCont.clear();
-      ctx.outputs().snapshot(o2::framework::Output{"CPV", "CLUSTERTRUEMC", 0, o2::framework::Lifetime::Timeframe}, mOutputTruthCont);
+      ctx.outputs().snapshot(o2::framework::Output{"CPV", "CLUSTERTRUEMC", 0}, mOutputTruthCont);
     }
     return;
   }
@@ -73,12 +73,12 @@ void ClusterizerSpec::run(framework::ProcessingContext& ctx)
 
   LOG(debug) << "CPVClusterizer::run() : Received " << digitsTR.size() << " TR, calling clusterizer ...";
 
-  ctx.outputs().snapshot(o2::framework::Output{"CPV", "CLUSTERS", 0, o2::framework::Lifetime::Timeframe}, mOutputClusters);
-  ctx.outputs().snapshot(o2::framework::Output{"CPV", "CLUSTERTRIGRECS", 0, o2::framework::Lifetime::Timeframe}, mOutputClusterTrigRecs);
+  ctx.outputs().snapshot(o2::framework::Output{"CPV", "CLUSTERS", 0}, mOutputClusters);
+  ctx.outputs().snapshot(o2::framework::Output{"CPV", "CLUSTERTRIGRECS", 0}, mOutputClusterTrigRecs);
   if (mPropagateMC) {
-    ctx.outputs().snapshot(o2::framework::Output{"CPV", "CLUSTERTRUEMC", 0, o2::framework::Lifetime::Timeframe}, mOutputTruthCont);
+    ctx.outputs().snapshot(o2::framework::Output{"CPV", "CLUSTERTRUEMC", 0}, mOutputTruthCont);
   }
-  ctx.outputs().snapshot(o2::framework::Output{"CPV", "CALIBDIGITS", 0, o2::framework::Lifetime::Timeframe}, mCalibDigits);
+  ctx.outputs().snapshot(o2::framework::Output{"CPV", "CALIBDIGITS", 0}, mCalibDigits);
   LOG(info) << "Finished, wrote  " << mOutputClusters.size() << " clusters, " << mOutputClusterTrigRecs.size() << "TR and " << mOutputTruthCont.getIndexedSize() << " Labels";
 }
 o2::framework::DataProcessorSpec o2::cpv::reco_workflow::getClusterizerSpec(bool propagateMC)
diff --git a/Detectors/CPV/workflow/src/DigitReaderSpec.cxx b/Detectors/CPV/workflow/src/DigitReaderSpec.cxx
new file mode 100644
index 0000000000000..20fe497eb5d0c
--- /dev/null
+++ b/Detectors/CPV/workflow/src/DigitReaderSpec.cxx
@@ -0,0 +1,100 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// @file   DigitReaderSpec.cxx
+
+#include 
+#include 
+#include "Framework/ControlService.h"
+#include "Framework/ConfigParamRegistry.h"
+#include "CPVWorkflow/DigitReaderSpec.h"
+#include "CommonUtils/NameConf.h"
+
+using namespace o2::framework;
+using namespace o2::cpv;
+
+namespace o2
+{
+namespace cpv
+{
+
+DigitReader::DigitReader(bool useMC)
+{
+  mUseMC = useMC;
+}
+
+void DigitReader::init(InitContext& ic)
+{
+  mInputFileName = o2::utils::Str::concat_string(o2::utils::Str::rectifyDirectory(ic.options().get("input-dir")),
+                                                 ic.options().get("cpv-digits-infile"));
+  connectTree(mInputFileName);
+}
+
+void DigitReader::run(ProcessingContext& pc)
+{
+  auto ent = mTree->GetReadEntry() + 1;
+  assert(ent < mTree->GetEntries()); // this should not happen
+  mTree->GetEntry(ent);
+  LOG(info) << "Pushing " << mDigits.size() << " Digits in " << mTRs.size() << " TriggerRecords at entry " << ent;
+  pc.outputs().snapshot(Output{mOrigin, "DIGITS", 0}, mDigits);
+  pc.outputs().snapshot(Output{mOrigin, "DIGITTRIGREC", 0}, mTRs);
+  if (mUseMC) {
+    pc.outputs().snapshot(Output{mOrigin, "DIGITSMCTR", 0}, mMCTruth);
+  }
+
+  if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
+    pc.services().get().endOfStream();
+    pc.services().get().readyToQuit(QuitRequest::Me);
+  }
+}
+
+void DigitReader::connectTree(const std::string& filename)
+{
+  mTree.reset(nullptr); // in case it was already loaded
+  mFile.reset(TFile::Open(filename.c_str()));
+  assert(mFile && !mFile->IsZombie());
+  mTree.reset((TTree*)mFile->Get(mDigitTreeName.c_str()));
+  assert(mTree);
+  assert(mTree->GetBranch(mTRBranchName.c_str()));
+
+  mTree->SetBranchAddress(mTRBranchName.c_str(), &mTRsInp);
+  mTree->SetBranchAddress(mDigitBranchName.c_str(), &mDigitsInp);
+  if (mUseMC) {
+    if (mTree->GetBranch(mDigitMCTruthBranchName.c_str())) {
+      mTree->SetBranchAddress(mDigitMCTruthBranchName.c_str(), &mMCTruthInp);
+    } else {
+      LOG(warning) << "MC-truth is missing, message will be empty";
+    }
+  }
+  LOG(info) << "Loaded tree from " << filename << " with " << mTree->GetEntries() << " entries";
+}
+
+DataProcessorSpec getCPVDigitReaderSpec(bool useMC)
+{
+  std::vector outputSpec;
+  outputSpec.emplace_back("CPV", "DIGITS", 0, Lifetime::Timeframe);
+  outputSpec.emplace_back("CPV", "DIGITTRIGREC", 0, Lifetime::Timeframe);
+  if (useMC) {
+    outputSpec.emplace_back("CPV", "DIGITSMCTR", 0, Lifetime::Timeframe);
+  }
+
+  return DataProcessorSpec{
+    "cpv-digit-reader",
+    Inputs{},
+    outputSpec,
+    AlgorithmSpec{adaptFromTask(useMC)},
+    Options{
+      {"cpv-digits-infile", VariantType::String, "cpvdigits.root", {"Name of the input Digit file"}},
+      {"input-dir", VariantType::String, "none", {"Input directory"}}}};
+}
+
+} // namespace cpv
+} // namespace o2
diff --git a/Detectors/CPV/workflow/src/DigitsPrinterSpec.cxx b/Detectors/CPV/workflow/src/DigitsPrinterSpec.cxx
index 81dd37f520b8c..35a9d5061d37b 100644
--- a/Detectors/CPV/workflow/src/DigitsPrinterSpec.cxx
+++ b/Detectors/CPV/workflow/src/DigitsPrinterSpec.cxx
@@ -12,7 +12,7 @@
 #include 
 #include 
 
-#include "FairLogger.h"
+#include 
 
 #include "Framework/RootSerializationSupport.h"
 #include "Framework/ControlService.h"
diff --git a/Detectors/CPV/workflow/src/EntropyDecoderSpec.cxx b/Detectors/CPV/workflow/src/EntropyDecoderSpec.cxx
index 72300c474fcfa..518a646e23cb9 100644
--- a/Detectors/CPV/workflow/src/EntropyDecoderSpec.cxx
+++ b/Detectors/CPV/workflow/src/EntropyDecoderSpec.cxx
@@ -25,11 +25,13 @@ namespace o2
 namespace cpv
 {
 
-EntropyDecoderSpec::EntropyDecoderSpec(int verbosity) : mCTFCoder(o2::ctf::CTFCoderBase::OpType::Decoder)
+EntropyDecoderSpec::EntropyDecoderSpec(int verbosity, const std::string& ctfdictOpt) : mCTFCoder(o2::ctf::CTFCoderBase::OpType::Decoder, ctfdictOpt)
 {
   mTimer.Stop();
   mTimer.Reset();
   mCTFCoder.setVerbosity(verbosity);
+  mCTFCoder.setSupportBCShifts(true);
+  mCTFCoder.setDictBinding("ctfdict_CPV");
 }
 
 void EntropyDecoderSpec::finaliseCCDB(o2::framework::ConcreteDataMatcher& matcher, void* obj)
@@ -50,8 +52,8 @@ void EntropyDecoderSpec::run(ProcessingContext& pc)
   mTimer.Start(false);
   o2::ctf::CTFIOSize iosize;
 
-  mCTFCoder.updateTimeDependentParams(pc);
-  auto buff = pc.inputs().get>("ctf");
+  mCTFCoder.updateTimeDependentParams(pc, true);
+  auto buff = pc.inputs().get>("ctf_CPV");
 
   auto& triggers = pc.outputs().make>(OutputRef{"triggers"});
   auto& clusters = pc.outputs().make>(OutputRef{"clusters"});
@@ -72,7 +74,7 @@ void EntropyDecoderSpec::endOfStream(EndOfStreamContext& ec)
        mTimer.CpuTime(), mTimer.RealTime(), mTimer.Counter() - 1);
 }
 
-DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec)
+DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec, const std::string& ctfdictOpt)
 {
   std::vector outputs{
     OutputSpec{{"triggers"}, "CPV", "CLUSTERTRIGRECS", 0, Lifetime::Timeframe},
@@ -80,15 +82,18 @@ DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec)
     OutputSpec{{"ctfrep"}, "CPV", "CTFDECREP", 0, Lifetime::Timeframe}};
 
   std::vector inputs;
-  inputs.emplace_back("ctf", "CPV", "CTFDATA", sspec, Lifetime::Timeframe);
-  inputs.emplace_back("ctfdict", "CPV", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("CPV/Calib/CTFDictionary"));
+  inputs.emplace_back("ctf_CPV", "CPV", "CTFDATA", sspec, Lifetime::Timeframe);
+  if (ctfdictOpt.empty() || ctfdictOpt == "ccdb") {
+    inputs.emplace_back("ctfdict_CPV", "CPV", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("CPV/Calib/CTFDictionaryTree"));
+  }
+  inputs.emplace_back("trigoffset", "CTP", "Trig_Offset", 0, Lifetime::Condition, ccdbParamSpec("CTP/Config/TriggerOffsets"));
 
   return DataProcessorSpec{
     "cpv-entropy-decoder",
     inputs,
     outputs,
-    AlgorithmSpec{adaptFromTask(verbosity)},
-    Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}}};
+    AlgorithmSpec{adaptFromTask(verbosity, ctfdictOpt)},
+    Options{{"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}};
 }
 
 } // namespace cpv
diff --git a/Detectors/CPV/workflow/src/EntropyEncoderSpec.cxx b/Detectors/CPV/workflow/src/EntropyEncoderSpec.cxx
index a1c1073d8c78a..54fb1354ad60c 100644
--- a/Detectors/CPV/workflow/src/EntropyEncoderSpec.cxx
+++ b/Detectors/CPV/workflow/src/EntropyEncoderSpec.cxx
@@ -26,7 +26,7 @@ namespace o2
 namespace cpv
 {
 
-EntropyEncoderSpec::EntropyEncoderSpec() : mCTFCoder(o2::ctf::CTFCoderBase::OpType::Encoder)
+EntropyEncoderSpec::EntropyEncoderSpec(bool selIR, const std::string& ctfdictOpt) : mCTFCoder(o2::ctf::CTFCoderBase::OpType::Encoder, ctfdictOpt), mSelIR(selIR)
 {
   mTimer.Stop();
   mTimer.Reset();
@@ -48,13 +48,18 @@ void EntropyEncoderSpec::run(ProcessingContext& pc)
 {
   auto cput = mTimer.CpuTime();
   mTimer.Start(false);
-  mCTFCoder.updateTimeDependentParams(pc);
+  mCTFCoder.updateTimeDependentParams(pc, true);
   auto triggers = pc.inputs().get>("triggers");
   auto clusters = pc.inputs().get>("clusters");
-
-  auto& buffer = pc.outputs().make>(Output{"CPV", "CTFDATA", 0, Lifetime::Timeframe});
+  if (mSelIR) {
+    mCTFCoder.setSelectedIRFrames(pc.inputs().get>("selIRFrames"));
+  }
+  auto& buffer = pc.outputs().make>(Output{"CPV", "CTFDATA", 0});
   auto iosize = mCTFCoder.encode(buffer, triggers, clusters);
   pc.outputs().snapshot({"ctfrep", 0}, iosize);
+  if (mSelIR) {
+    mCTFCoder.getIRFramesSelector().clear();
+  }
   mTimer.Stop();
   LOG(info) << iosize.asString() << " in " << mTimer.CpuTime() - cput << " s";
 }
@@ -65,21 +70,27 @@ void EntropyEncoderSpec::endOfStream(EndOfStreamContext& ec)
        mTimer.CpuTime(), mTimer.RealTime(), mTimer.Counter() - 1);
 }
 
-DataProcessorSpec getEntropyEncoderSpec()
+DataProcessorSpec getEntropyEncoderSpec(bool selIR, const std::string& ctfdictOpt)
 {
   std::vector inputs;
   inputs.emplace_back("triggers", "CPV", "CLUSTERTRIGRECS", 0, Lifetime::Timeframe);
   inputs.emplace_back("clusters", "CPV", "CLUSTERS", 0, Lifetime::Timeframe);
-  inputs.emplace_back("ctfdict", "CPV", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("CPV/Calib/CTFDictionary"));
-
+  if (ctfdictOpt.empty() || ctfdictOpt == "ccdb") {
+    inputs.emplace_back("ctfdict", "CPV", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("CPV/Calib/CTFDictionaryTree"));
+  }
+  if (selIR) {
+    inputs.emplace_back("selIRFrames", "CTF", "SELIRFRAMES", 0, Lifetime::Timeframe);
+  }
   return DataProcessorSpec{
     "cpv-entropy-encoder",
     inputs,
     Outputs{{"CPV", "CTFDATA", 0, Lifetime::Timeframe},
             {{"ctfrep"}, "CPV", "CTFENCREP", 0, Lifetime::Timeframe}},
-    AlgorithmSpec{adaptFromTask()},
-    Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}},
-            {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}}}};
+    AlgorithmSpec{adaptFromTask(selIR, ctfdictOpt)},
+    Options{{"irframe-margin-bwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame lower boundary when selection is requested"}},
+            {"irframe-margin-fwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame upper boundary when selection is requested"}},
+            {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}},
+            {"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}};
 }
 
 } // namespace cpv
diff --git a/Detectors/CPV/workflow/src/RawToDigitConverterSpec.cxx b/Detectors/CPV/workflow/src/RawToDigitConverterSpec.cxx
index cca764cf0ef91..766902a2fdc95 100644
--- a/Detectors/CPV/workflow/src/RawToDigitConverterSpec.cxx
+++ b/Detectors/CPV/workflow/src/RawToDigitConverterSpec.cxx
@@ -9,7 +9,7 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 #include 
-#include "FairLogger.h"
+#include 
 #include "CommonDataFormat/InteractionRecord.h"
 #include "Framework/InputRecordWalker.h"
 #include "Framework/DataRefUtils.h"
@@ -136,11 +136,11 @@ void RawToDigitConverterSpec::run(framework::ProcessingContext& ctx)
              contDeadBeef == maxWarn ? fmt::format(". {} such inputs in row received, stopping reporting", contDeadBeef) : "");
       }
       mOutputDigits.clear();
-      ctx.outputs().snapshot(o2::framework::Output{"CPV", "DIGITS", 0, o2::framework::Lifetime::Timeframe}, mOutputDigits);
+      ctx.outputs().snapshot(o2::framework::Output{"CPV", "DIGITS", 0}, mOutputDigits);
       mOutputTriggerRecords.clear();
-      ctx.outputs().snapshot(o2::framework::Output{"CPV", "DIGITTRIGREC", 0, o2::framework::Lifetime::Timeframe}, mOutputTriggerRecords);
+      ctx.outputs().snapshot(o2::framework::Output{"CPV", "DIGITTRIGREC", 0}, mOutputTriggerRecords);
       mOutputHWErrors.clear();
-      ctx.outputs().snapshot(o2::framework::Output{"CPV", "RAWHWERRORS", 0, o2::framework::Lifetime::Timeframe}, mOutputHWErrors);
+      ctx.outputs().snapshot(o2::framework::Output{"CPV", "RAWHWERRORS", 0}, mOutputHWErrors);
       return; // empty TF, nothing to process
     }
   }
@@ -272,11 +272,11 @@ void RawToDigitConverterSpec::run(framework::ProcessingContext& ctx)
     if (skipTF) {
       // Send no digits
       mOutputDigits.clear();
-      ctx.outputs().snapshot(o2::framework::Output{"CPV", "DIGITS", 0, o2::framework::Lifetime::Timeframe}, mOutputDigits);
+      ctx.outputs().snapshot(o2::framework::Output{"CPV", "DIGITS", 0}, mOutputDigits);
       mOutputTriggerRecords.clear();
-      ctx.outputs().snapshot(o2::framework::Output{"CPV", "DIGITTRIGREC", 0, o2::framework::Lifetime::Timeframe}, mOutputTriggerRecords);
+      ctx.outputs().snapshot(o2::framework::Output{"CPV", "DIGITTRIGREC", 0}, mOutputTriggerRecords);
       // Send errors
-      ctx.outputs().snapshot(o2::framework::Output{"CPV", "RAWHWERRORS", 0, o2::framework::Lifetime::Timeframe}, mOutputHWErrors);
+      ctx.outputs().snapshot(o2::framework::Output{"CPV", "RAWHWERRORS", 0}, mOutputHWErrors);
       return;
     }
   }
@@ -305,15 +305,15 @@ void RawToDigitConverterSpec::run(framework::ProcessingContext& ctx)
   digitBuffer.clear();
 
   LOG(info) << "[CPVRawToDigitConverter - run] Sending " << mOutputDigits.size() << " digits in " << mOutputTriggerRecords.size() << "trigger records.";
-  ctx.outputs().snapshot(o2::framework::Output{"CPV", "DIGITS", 0, o2::framework::Lifetime::Timeframe}, mOutputDigits);
-  ctx.outputs().snapshot(o2::framework::Output{"CPV", "DIGITTRIGREC", 0, o2::framework::Lifetime::Timeframe}, mOutputTriggerRecords);
-  ctx.outputs().snapshot(o2::framework::Output{"CPV", "RAWHWERRORS", 0, o2::framework::Lifetime::Timeframe}, mOutputHWErrors);
+  ctx.outputs().snapshot(o2::framework::Output{"CPV", "DIGITS", 0}, mOutputDigits);
+  ctx.outputs().snapshot(o2::framework::Output{"CPV", "DIGITTRIGREC", 0}, mOutputTriggerRecords);
+  ctx.outputs().snapshot(o2::framework::Output{"CPV", "RAWHWERRORS", 0}, mOutputHWErrors);
 }
 //_____________________________________________________________________________
 o2::framework::DataProcessorSpec o2::cpv::reco_workflow::getRawToDigitConverterSpec(bool askDISTSTF, bool isPedestal, bool useBadChannelMap, bool useGainCalibration)
 {
   std::vector inputs;
-  inputs.emplace_back("RAWDATA", o2::framework::ConcreteDataTypeMatcher{"CPV", "RAWDATA"}, o2::framework::Lifetime::Optional);
+  inputs.emplace_back("RAWDATA", o2::framework::ConcreteDataTypeMatcher{"CPV", "RAWDATA"}, o2::framework::Lifetime::Timeframe);
   // receive at least 1 guaranteed input (which will allow to acknowledge the TF)
   if (askDISTSTF) {
     inputs.emplace_back("STFDist", "FLP", "DISTSUBTIMEFRAME", 0, o2::framework::Lifetime::Timeframe);
diff --git a/Detectors/CPV/workflow/src/RecoWorkflow.cxx b/Detectors/CPV/workflow/src/RecoWorkflow.cxx
index d15a909880d85..66b5f3dbe588a 100644
--- a/Detectors/CPV/workflow/src/RecoWorkflow.cxx
+++ b/Detectors/CPV/workflow/src/RecoWorkflow.cxx
@@ -13,7 +13,7 @@
 #include 
 #include 
 
-#include "FairLogger.h"
+#include 
 
 #include "Framework/RootSerializationSupport.h"
 #include "Algorithm/RangeTokenizer.h"
@@ -24,6 +24,7 @@
 #include "SimulationDataFormat/MCCompLabel.h"
 #include "DataFormatsCPV/TriggerRecord.h"
 #include "CPVWorkflow/RecoWorkflow.h"
+#include "CPVWorkflow/DigitReaderSpec.h"
 #include "CPVWorkflow/ClusterizerSpec.h"
 #include "CPVWorkflow/ReaderSpec.h"
 #include "CPVWorkflow/WriterSpec.h"
@@ -107,7 +108,8 @@ o2::framework::WorkflowSpec getWorkflow(bool disableRootInp,
   // Digits to ....
   if (inputType == InputType::Digits) {
     if (!disableRootInp) {
-      specs.emplace_back(o2::cpv::getDigitsReaderSpec(propagateMC));
+      specs.emplace_back(o2::cpv::getCPVDigitReaderSpec(propagateMC));
+      // specs.emplace_back(o2::cpv::getDigitsReaderSpec(propagateMC));
     }
 
     if (isEnabled(OutputType::Clusters)) {
diff --git a/Detectors/CPV/workflow/src/cluster-reader-workflow.cxx b/Detectors/CPV/workflow/src/cluster-reader-workflow.cxx
new file mode 100644
index 0000000000000..33f389f15f9da
--- /dev/null
+++ b/Detectors/CPV/workflow/src/cluster-reader-workflow.cxx
@@ -0,0 +1,56 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "Framework/ConfigParamSpec.h"
+#include "CommonUtils/ConfigurableParam.h"
+#include "DetectorsRaw/HBFUtilsInitializer.h"
+#include "Framework/CallbacksPolicy.h"
+
+using namespace o2::framework;
+
+void customize(std::vector& policies)
+{
+  o2::raw::HBFUtilsInitializer::addNewTimeSliceCallback(policies);
+}
+
+void customize(std::vector& workflowOptions)
+{
+  workflowOptions.push_back(
+    ConfigParamSpec{
+      "with-mc",
+      o2::framework::VariantType::Bool,
+      false,
+      {"propagate MC labels"}});
+  workflowOptions.push_back(
+    ConfigParamSpec{
+      "configKeyValues",
+      VariantType::String,
+      "",
+      {"Semicolon separated key=value strings"}});
+  o2::raw::HBFUtilsInitializer::addConfigOption(workflowOptions);
+}
+
+#include "Framework/runDataProcessing.h"
+#include "CPVWorkflow/ClusterReaderSpec.h"
+
+WorkflowSpec defineDataProcessing(ConfigContext const& cc)
+{
+  WorkflowSpec specs;
+  o2::conf::ConfigurableParam::updateFromString(cc.options().get("configKeyValues"));
+  auto withMC = cc.options().get("with-mc");
+
+  specs.emplace_back(o2::cpv::getCPVClusterReaderSpec(withMC));
+
+  // configure dpl timer to inject correct firstTForbit: start from the 1st orbit of TF containing 1st sampled orbit
+  o2::raw::HBFUtilsInitializer hbfIni(cc, specs);
+
+  return specs;
+}
diff --git a/Detectors/CPV/workflow/src/digit-reader-workflow.cxx b/Detectors/CPV/workflow/src/digit-reader-workflow.cxx
new file mode 100644
index 0000000000000..dc068573b1218
--- /dev/null
+++ b/Detectors/CPV/workflow/src/digit-reader-workflow.cxx
@@ -0,0 +1,56 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "Framework/ConfigParamSpec.h"
+#include "CommonUtils/ConfigurableParam.h"
+#include "DetectorsRaw/HBFUtilsInitializer.h"
+#include "Framework/CallbacksPolicy.h"
+
+using namespace o2::framework;
+
+void customize(std::vector& policies)
+{
+  o2::raw::HBFUtilsInitializer::addNewTimeSliceCallback(policies);
+}
+
+void customize(std::vector& workflowOptions)
+{
+  workflowOptions.push_back(
+    ConfigParamSpec{
+      "with-mc",
+      o2::framework::VariantType::Bool,
+      false,
+      {"propagate MC labels"}});
+  workflowOptions.push_back(
+    ConfigParamSpec{
+      "configKeyValues",
+      VariantType::String,
+      "",
+      {"Semicolon separated key=value strings"}});
+  o2::raw::HBFUtilsInitializer::addConfigOption(workflowOptions);
+}
+
+#include "Framework/runDataProcessing.h"
+#include "CPVWorkflow/DigitReaderSpec.h"
+
+WorkflowSpec defineDataProcessing(ConfigContext const& cc)
+{
+  WorkflowSpec specs;
+  o2::conf::ConfigurableParam::updateFromString(cc.options().get("configKeyValues"));
+  auto withMC = cc.options().get("with-mc");
+
+  specs.emplace_back(o2::cpv::getCPVDigitReaderSpec(withMC));
+
+  // configure dpl timer to inject correct firstTForbit: start from the 1st orbit of TF containing 1st sampled orbit
+  o2::raw::HBFUtilsInitializer hbfIni(cc, specs);
+
+  return specs;
+}
diff --git a/Detectors/CPV/workflow/src/entropy-encoder-workflow.cxx b/Detectors/CPV/workflow/src/entropy-encoder-workflow.cxx
index 9c79fe431e27f..90fedb1d5eb83 100644
--- a/Detectors/CPV/workflow/src/entropy-encoder-workflow.cxx
+++ b/Detectors/CPV/workflow/src/entropy-encoder-workflow.cxx
@@ -21,7 +21,10 @@ using namespace o2::framework;
 void customize(std::vector& workflowOptions)
 {
   // option allowing to set parameters
-  std::vector options{ConfigParamSpec{"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}}};
+  std::vector options{
+    ConfigParamSpec{"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}},
+    ConfigParamSpec{"ctf-dict", VariantType::String, "none", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}},
+    ConfigParamSpec{"select-ir-frames", VariantType::Bool, false, {"Subscribe and filter according to external IR Frames"}}};
 
   std::swap(workflowOptions, options);
 }
@@ -35,6 +38,6 @@ WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
   WorkflowSpec wf;
   // Update the (declared) parameters if changed from the command line
   o2::conf::ConfigurableParam::updateFromString(cfgc.options().get("configKeyValues"));
-  wf.emplace_back(o2::cpv::getEntropyEncoderSpec());
+  wf.emplace_back(o2::cpv::getEntropyEncoderSpec(cfgc.options().get("select-ir-frames"), cfgc.options().get("ctf-dict")));
   return wf;
 }
diff --git a/Detectors/CTF/README.md b/Detectors/CTF/README.md
index 9d5c800be6d18..47ce765de289a 100644
--- a/Detectors/CTF/README.md
+++ b/Detectors/CTF/README.md
@@ -51,6 +51,22 @@ o2-ctf-reader-workflow --ctf-input input.lst --onlyDet ITS,TPC,TOF --its-entropy
 
 See below for the details of `--ctf-dict` option.
 
+One can pause the writing if available disk space is low using a combination of following options:
+```bash
+--require-free-disk : pause writing operation if available disk space is below this margin in bytes (if > 0) or this fraction of total (if < 0)
+```
+
+```bash
+--wait-for-free-disk : if paused due to the low disk space, recheck after this time (in s)
+```
+
+```bash
+--max-wait-for-free-disk : produce fatal if paused due to the low disk space for more than this amount( in s).
+```
+
+
+
+
 ## CTF reader workflow
 
 `o2-ctf-reader-workflow` should be the 1st workflow in the piped chain of CTF processing.
@@ -79,11 +95,24 @@ comma-separated list of detectors to read, Overrides skipDet
 ```
 comma-separated list of detectors to skip
 
+
+By default an exception will be thrown if detector is requested but missing in the CTF. To enable injection of the empty output in such case one should use option `--allow-missing-detectors`.
+
+```
+--ctf-data-subspec arg (=0)
+```
+allows to alter the `subSpecification` used to send the CTFDATA from the reader to decoders. Non-0 value must be used in case the data extracted by the CTF-reader should be processed and stored in new CTFs (in order to avoid clash of CTFDATA messages of the reader and writer).
+
 ```
 --max-tf arg (=-1)
 ```
 max CTFs to process (<= 0 : infinite)
 
+```
+--max-tf-per-file arg (=-1)
+```
+max TFs to process from every CTF file (<= 0 : infinite)
+
 ```
 --loop arg (=0)
 ```
@@ -120,6 +149,8 @@ There is a possibility to read remote root files directly, w/o caching them loca
 2) provide proper regex to define remote files, e.g. for the example above: `--remote-regex "^root://.+/eos/aliceo2/.+"`.
 3) pass an option `--copy-cmd no-copy`.
 
+## Selective TF reading
+
 ```
 --select-ctf-ids 
 ```
@@ -127,24 +158,25 @@ This is a `ctf-reader` device local option allowing selective reading of particu
 Note that the index corresponds not to the entry of the TF in the CTF tree but to the reader own counter incremented throught all input files (e.g. if the 10 CTF files with 20 TFs each are provided for the input and the selection of TFs
 `0,2,22,66` is provided, the reader will inject to the DPL the TFs at entries 0 and 2 from the 1st CTF file, entry 5 of the second file, entry 6 of the 3d and will finish the job.
 
-For the ITS and MFT entropy decoding one can request either to decompose clusters to digits and send them instead of clusters (via `o2-ctf-reader-workflow` global options `--its-digits` and `--mft-digits` respectively)
-or to apply the noise mask to decoded clusters (or decoded digits). If the masking (e.g. via option `--its-entropy-decoder " --mask-noise "`) is requested, user should provide to the entropy decoder the noise mask file (eventually will be loaded from CCDB) and cluster patterns decoding dictionary (if the clusters were encoded with patterns IDs).
-For example,
 ```
-o2-ctf-reader-workflow --ctf-input  --onlyDet ITS,MFT --its-entropy-decoder ' --mask-noise' | ...
+--ir-frames-files  --skip-skimmed-out-tf
 ```
-will decode ITS and MFT data, decompose on the fly ITS clusters to digits, mask the noisy pixels with the provided masks, recluster remaining ITS digits and send the new clusters out, together with unchanged MFT clusters.
+This option (used for skimming) allow to push to DPL only those TFs which overlap with selected BC-ranges provided via input root file (for various formats see `o2::utils::IRFrameSelector::loadIRFrames` method).
+
 ```
-o2-ctf-reader-workflow --ctf-input  --onlyDet ITS,MFT --mft-digits --mft-entropy-decoder ' --mask-noise' | ...
+--ir-frames-files 
 ```
-will send decompose clusters to digits and send ben out after masking the noise for the MFT, while ITS clusters will be sent as decoded.
-
-By default an exception will be thrown if detector is requested but missing in the CTF. To enable injection of the empty output in such case one should use option `--allow-missing-detectors`.
+This option allows to push to DPL only those TFs which overlap with the `  ` (separators can be any whitespace, comma or semicolon) records provided via text file (assuming that there are some entries for a given run, otherwise the option is ignored).
+Multiple ranges per run and multiple runs can be mentioned in a single input file. The range limits can be indicated either as a UNIX timestamp in `ms` or as an orbit number (in the fill the run belongs to).
 
+In case an option
 ```
---ctf-data-subspec arg (=0)
+--invert-irframe-selection
 ```
-allows to alter the `subSpecification` used to send the CTFDATA from the reader to decoders. Non-0 value must be used in case the data extracted by the CTF-reader should be processed and stored in new CTFs (in order to avoid clash of CTFDATA messages of the reader and writer).
+is provided, the selections above are inverted: TFs matching some of the provided ranges will be discarded, while the rest will be pushed to the DPL
+
+At the end of the processing the `ctf-writer` will create a local file `ctf_read_ntf.txt` containing only the number of TFs pushed to the DPL.
+In case no TF passed the selections above, this file will contain 0.
 
 ## Support for externally provided encoding dictionaries
 
@@ -177,3 +209,21 @@ To apply TF rate limiting (make sure that no more than N TFs are in processing)
 too all workflows (e.g. via ARGS_ALL).
 The IPCID is the NUMA domain ID (usually 0 on non-EPN workflow).
 Additionally, one may throttle on the free SHM by providing an option to the reader `--timeframes-shm-limit `.
+
+Note that by default the reader reads into the memory the CTF data and prepares all output messages but injects them only once the rate-limiter allows that.
+With the option `--limit-tf-before-reading` set also the preparation of the data to inject will be conditioned by the green light from the rate-limiter.
+
+
+## Modifying ITS/MFT CTF output
+
+For the ITS and MFT entropy decoding one can request either to decompose clusters to digits and send them instead of clusters (via `o2-ctf-reader-workflow` global options `--its-digits` and `--mft-digits` respectively)
+or to apply the noise mask to decoded clusters (or decoded digits). If the masking (e.g. via option `--its-entropy-decoder " --mask-noise "`) is requested, user should provide to the entropy decoder the noise mask file (eventually will be loaded from CCDB) and cluster patterns decoding dictionary (if the clusters were encoded with patterns IDs).
+For example,
+```
+o2-ctf-reader-workflow --ctf-input  --onlyDet ITS,MFT --its-entropy-decoder ' --mask-noise' | ...
+```
+will decode ITS and MFT data, decompose on the fly ITS clusters to digits, mask the noisy pixels with the provided masks, recluster remaining ITS digits and send the new clusters out, together with unchanged MFT clusters.
+```
+o2-ctf-reader-workflow --ctf-input  --onlyDet ITS,MFT --mft-digits --mft-entropy-decoder ' --mask-noise' | ...
+```
+will send decompose clusters to digits and send ben out after masking the noise for the MFT, while ITS clusters will be sent as decoded.
diff --git a/Detectors/CTF/test/test_ctf_io_cpv.cxx b/Detectors/CTF/test/test_ctf_io_cpv.cxx
index 2648cd40374cb..34a383a6875a0 100644
--- a/Detectors/CTF/test/test_ctf_io_cpv.cxx
+++ b/Detectors/CTF/test/test_ctf_io_cpv.cxx
@@ -12,7 +12,13 @@
 #define BOOST_TEST_MODULE Test CPVCTFIO
 #define BOOST_TEST_MAIN
 #define BOOST_TEST_DYN_LINK
+
+#undef NDEBUG
+#include 
+
 #include 
+#include 
+#include 
 #include "CommonUtils/NameConf.h"
 #include "CPVReconstruction/CTFCoder.h"
 #include "DataFormatsCPV/CTF.h"
@@ -22,11 +28,15 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 
 using namespace o2::cpv;
+namespace boost_data = boost::unit_test::data;
+
+inline std::vector ANSVersions{o2::ctf::ANSVersionCompat, o2::ctf::ANSVersion1};
 
-BOOST_AUTO_TEST_CASE(CTFTest)
+BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions), ansVersion)
 {
   std::vector triggers;
   std::vector clusters;
@@ -55,6 +65,7 @@ BOOST_AUTO_TEST_CASE(CTFTest)
   std::vector vec;
   {
     CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Encoder);
+    coder.setANSVersion(ansVersion);
     coder.encode(vec, triggers, clusters); // compress
   }
   sw.Stop();
diff --git a/Detectors/CTF/test/test_ctf_io_ctp.cxx b/Detectors/CTF/test/test_ctf_io_ctp.cxx
index 4f48f83685de0..be9cdd1667ba8 100644
--- a/Detectors/CTF/test/test_ctf_io_ctp.cxx
+++ b/Detectors/CTF/test/test_ctf_io_ctp.cxx
@@ -12,7 +12,13 @@
 #define BOOST_TEST_MODULE Test CTPCTFIO
 #define BOOST_TEST_MAIN
 #define BOOST_TEST_DYN_LINK
+
+#undef NDEBUG
+#include 
+
 #include 
+#include 
+#include 
 #include "CommonUtils/NameConf.h"
 #include "CTPReconstruction/CTFCoder.h"
 #include "DataFormatsCTP/CTF.h"
@@ -25,8 +31,11 @@
 #include 
 
 using namespace o2::ctp;
+namespace boost_data = boost::unit_test::data;
+
+inline std::vector ANSVersions{o2::ctf::ANSVersionCompat, o2::ctf::ANSVersion1};
 
-BOOST_AUTO_TEST_CASE(CTFTest, *boost::unit_test::enabled())
+BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions), ansVersion)
 {
   std::vector digits;
   TStopwatch sw;
@@ -36,7 +45,6 @@ BOOST_AUTO_TEST_CASE(CTFTest, *boost::unit_test::enabled())
   std::random_device rd;
   std::mt19937_64 eng(rd());
   std::uniform_int_distribution distr;
-
   for (int itrg = 0; itrg < 1000; itrg++) {
     ir += 1 + distr(eng) % 200;
     auto& dig = digits.emplace_back();
@@ -44,12 +52,14 @@ BOOST_AUTO_TEST_CASE(CTFTest, *boost::unit_test::enabled())
     dig.CTPInputMask |= distr(eng);
     dig.CTPClassMask |= distr(eng);
   }
+  LumiInfo lumi{digits.front().intRecord.orbit, 30, 12345};
 
   sw.Start();
   std::vector vec;
   {
     CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Encoder);
-    coder.encode(vec, digits); // compress
+    coder.setANSVersion(ansVersion);
+    coder.encode(vec, digits, lumi); // compress
   }
   sw.Stop();
   LOG(info) << "Compressed in " << sw.CpuTime() << " s";
@@ -80,12 +90,12 @@ BOOST_AUTO_TEST_CASE(CTFTest, *boost::unit_test::enabled())
   }
 
   std::vector digitsD;
-
+  LumiInfo lumiD;
   sw.Start();
   const auto ctfImage = o2::ctp::CTF::getImage(vec.data());
   {
     CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Decoder);
-    coder.decode(ctfImage, digitsD); // decompress
+    coder.decode(ctfImage, digitsD, lumiD); // decompress
   }
   sw.Stop();
   LOG(info) << "Decompressed in " << sw.CpuTime() << " s";
@@ -93,4 +103,5 @@ BOOST_AUTO_TEST_CASE(CTFTest, *boost::unit_test::enabled())
   LOG(info) << " BOOST_CHECK(digitsD.size() " << digitsD.size() << " digigits.size()) " << digits.size();
 
   BOOST_TEST(digits == digitsD, boost::test_tools::per_element());
+  BOOST_CHECK(lumiD.orbit == lumi.orbit && lumiD.nHBFCounted == lumi.nHBFCounted && lumiD.orbit == lumi.orbit);
 }
diff --git a/Detectors/CTF/test/test_ctf_io_emcal.cxx b/Detectors/CTF/test/test_ctf_io_emcal.cxx
index fb91254f6847f..4c6f6f8193672 100644
--- a/Detectors/CTF/test/test_ctf_io_emcal.cxx
+++ b/Detectors/CTF/test/test_ctf_io_emcal.cxx
@@ -12,8 +12,15 @@
 #define BOOST_TEST_MODULE Test EMCCTFIO
 #define BOOST_TEST_MAIN
 #define BOOST_TEST_DYN_LINK
+
+#undef NDEBUG
+#include 
+
 #include 
+#include 
+#include 
 #include "CommonUtils/NameConf.h"
+#include "DataFormatsEMCAL/Constants.h"
 #include "EMCALReconstruction/CTFCoder.h"
 #include "DataFormatsEMCAL/CTF.h"
 #include "Framework/Logger.h"
@@ -24,8 +31,11 @@
 #include 
 
 using namespace o2::emcal;
+namespace boost_data = boost::unit_test::data;
 
-BOOST_AUTO_TEST_CASE(CTFTest)
+inline std::vector ANSVersions{o2::ctf::ANSVersionCompat, o2::ctf::ANSVersion1};
+
+BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions), ansVersion)
 {
   std::vector triggers;
   std::vector cells;
@@ -41,8 +51,33 @@ BOOST_AUTO_TEST_CASE(CTFTest)
     while (tower < 17665) {
       float timeCell = gRandom->Rndm() * 1500 - 600.;
       float en = gRandom->Rndm() * 250.;
-      int stat = gRandom->Integer(5);
-      cells.emplace_back(tower, en, timeCell, (ChannelType_t)stat);
+      // In case of cell type 3 cases must be distinguished (FEE, LEDMON, TRU)
+      // In case the cell is a FEE cell the cell type is correlated with the energy
+      int readoutsource = gRandom->Integer(3); //
+      ChannelType_t chantype = ChannelType_t::HIGH_GAIN;
+      switch (readoutsource) {
+        case 0: {
+          // Cell is a FEE cell, determine cell type according to HGLG transition
+          const auto ENHGLG = o2::emcal::constants::EMCAL_HGLGTRANSITION * o2::emcal::constants::EMCAL_ADCENERGY;
+          if (en >= ENHGLG) {
+            chantype = ChannelType_t::LOW_GAIN;
+          } else {
+            chantype = ChannelType_t::HIGH_GAIN;
+          }
+          break;
+        }
+        case 1:
+          chantype = ChannelType_t::LEDMON;
+          break;
+        case 2:
+          chantype = ChannelType_t::TRU;
+          break;
+
+        default:
+          std::cerr << "Unknown type" << std::endl;
+          break;
+      }
+      cells.emplace_back(tower, en, timeCell, chantype);
       tower += 1 + gRandom->Integer(100);
     }
     uint32_t trigBits = gRandom->Integer(0xFFFFFFFF); // will be converted internally to uint16_t by the coder
@@ -53,6 +88,7 @@ BOOST_AUTO_TEST_CASE(CTFTest)
   std::vector vec;
   {
     CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Encoder);
+    coder.setANSVersion(ansVersion);
     coder.encode(vec, triggers, cells); // compress
   }
   sw.Stop();
@@ -126,9 +162,9 @@ BOOST_AUTO_TEST_CASE(CTFTest)
   for (size_t i = 0; i < cells.size(); i++) {
     const auto& cor = cells[i];
     const auto& cdc = cellsD[i];
-    BOOST_CHECK_EQUAL(cor.getPackedTowerID(), cdc.getPackedTowerID());
-    BOOST_CHECK_EQUAL(cor.getPackedTime(), cdc.getPackedTime());
-    BOOST_CHECK_EQUAL(cor.getPackedEnergy(), cdc.getPackedEnergy());
-    BOOST_CHECK_EQUAL(cor.getPackedCellStatus(), cdc.getPackedCellStatus());
+    BOOST_CHECK_EQUAL(cor.getTowerIDEncoded(), cdc.getTowerIDEncoded());
+    BOOST_CHECK_EQUAL(cor.getTimeStampEncoded(), cdc.getTimeStampEncoded());
+    BOOST_CHECK_EQUAL(cor.getEnergyEncoded(), cdc.getEnergyEncoded());
+    BOOST_CHECK_EQUAL(cor.getCellTypeEncoded(), cdc.getCellTypeEncoded());
   }
 }
diff --git a/Detectors/CTF/test/test_ctf_io_fdd.cxx b/Detectors/CTF/test/test_ctf_io_fdd.cxx
index a1da64af04968..3dd87c1618598 100644
--- a/Detectors/CTF/test/test_ctf_io_fdd.cxx
+++ b/Detectors/CTF/test/test_ctf_io_fdd.cxx
@@ -12,7 +12,13 @@
 #define BOOST_TEST_MODULE Test FDDCTFIO
 #define BOOST_TEST_MAIN
 #define BOOST_TEST_DYN_LINK
+
+#undef NDEBUG
+#include 
+
 #include 
+#include 
+#include 
 #include "CommonUtils/NameConf.h"
 #include "FDDReconstruction/CTFCoder.h"
 #include "FDDBase/Constants.h"
@@ -23,8 +29,11 @@
 #include 
 
 using namespace o2::fdd;
+namespace boost_data = boost::unit_test::data;
+
+inline std::vector ANSVersions{o2::ctf::ANSVersionCompat, o2::ctf::ANSVersion1};
 
-BOOST_AUTO_TEST_CASE(CTFTest)
+BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions), ansVersion)
 {
   std::vector digits;
   std::vector channels;
@@ -81,6 +90,7 @@ BOOST_AUTO_TEST_CASE(CTFTest)
   std::vector vec;
   {
     CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Encoder);
+    coder.setANSVersion(ansVersion);
     coder.encode(vec, digits, channels); // compress
   }
   sw.Stop();
diff --git a/Detectors/CTF/test/test_ctf_io_ft0.cxx b/Detectors/CTF/test/test_ctf_io_ft0.cxx
index 4b7bee2460909..6e2746444e662 100644
--- a/Detectors/CTF/test/test_ctf_io_ft0.cxx
+++ b/Detectors/CTF/test/test_ctf_io_ft0.cxx
@@ -12,7 +12,13 @@
 #define BOOST_TEST_MODULE Test FT0CTFIO
 #define BOOST_TEST_MAIN
 #define BOOST_TEST_DYN_LINK
+
+#undef NDEBUG
+#include 
+
 #include 
+#include 
+#include 
 #include "CommonUtils/NameConf.h"
 #include "FT0Reconstruction/CTFCoder.h"
 #include "FT0Base/FT0DigParam.h"
@@ -23,8 +29,11 @@
 #include 
 
 using namespace o2::ft0;
+namespace boost_data = boost::unit_test::data;
+
+inline std::vector ANSVersions{o2::ctf::ANSVersionCompat, o2::ctf::ANSVersion1};
 
-BOOST_AUTO_TEST_CASE(CTFTest)
+BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions), ansVersion)
 {
   std::vector digits;
   std::vector channels;
@@ -87,6 +96,7 @@ BOOST_AUTO_TEST_CASE(CTFTest)
   std::vector vec;
   {
     CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Encoder);
+    coder.setANSVersion(ansVersion);
     coder.encode(vec, digits, channels); // compress
   }
   sw.Stop();
diff --git a/Detectors/CTF/test/test_ctf_io_fv0.cxx b/Detectors/CTF/test/test_ctf_io_fv0.cxx
index f9bf4a8a878e5..2e558b91a8e1c 100644
--- a/Detectors/CTF/test/test_ctf_io_fv0.cxx
+++ b/Detectors/CTF/test/test_ctf_io_fv0.cxx
@@ -12,7 +12,13 @@
 #define BOOST_TEST_MODULE Test FV0CTFIO
 #define BOOST_TEST_MAIN
 #define BOOST_TEST_DYN_LINK
+
+#undef NDEBUG
+#include 
+
 #include 
+#include 
+#include 
 #include "CommonUtils/NameConf.h"
 #include "FV0Reconstruction/CTFCoder.h"
 #include "FV0Base/Constants.h"
@@ -23,8 +29,11 @@
 #include 
 
 using namespace o2::fv0;
+namespace boost_data = boost::unit_test::data;
+
+inline std::vector ANSVersions{o2::ctf::ANSVersionCompat, o2::ctf::ANSVersion1};
 
-BOOST_AUTO_TEST_CASE(CTFTest)
+BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions), ansVersion)
 {
   std::vector digits;
   std::vector channels;
@@ -59,6 +68,7 @@ BOOST_AUTO_TEST_CASE(CTFTest)
   std::vector vec;
   {
     CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Encoder);
+    coder.setANSVersion(ansVersion);
     coder.encode(vec, digits, channels); // compress
   }
   sw.Stop();
diff --git a/Detectors/CTF/test/test_ctf_io_hmpid.cxx b/Detectors/CTF/test/test_ctf_io_hmpid.cxx
index 88df30c8419c0..fd12ee7eb1e36 100644
--- a/Detectors/CTF/test/test_ctf_io_hmpid.cxx
+++ b/Detectors/CTF/test/test_ctf_io_hmpid.cxx
@@ -12,7 +12,13 @@
 #define BOOST_TEST_MODULE Test HMPIDCTFIO
 #define BOOST_TEST_MAIN
 #define BOOST_TEST_DYN_LINK
+
+#undef NDEBUG
+#include 
+
 #include 
+#include 
+#include 
 #include "CommonUtils/NameConf.h"
 #include "HMPIDReconstruction/CTFCoder.h"
 #include "DataFormatsHMP/CTF.h"
@@ -24,8 +30,11 @@
 #include 
 
 using namespace o2::hmpid;
+namespace boost_data = boost::unit_test::data;
+
+inline std::vector ANSVersions{o2::ctf::ANSVersionCompat, o2::ctf::ANSVersion1};
 
-BOOST_AUTO_TEST_CASE(CTFTest)
+BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions), ansVersion)
 {
   std::vector triggers;
   std::vector digits;
@@ -53,6 +62,7 @@ BOOST_AUTO_TEST_CASE(CTFTest)
   std::vector vec;
   {
     CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Encoder);
+    coder.setANSVersion(ansVersion);
     coder.encode(vec, triggers, digits); // compress
   }
   sw.Stop();
diff --git a/Detectors/CTF/test/test_ctf_io_itsmft.cxx b/Detectors/CTF/test/test_ctf_io_itsmft.cxx
index 38d288d3703ad..13cbdf7745961 100644
--- a/Detectors/CTF/test/test_ctf_io_itsmft.cxx
+++ b/Detectors/CTF/test/test_ctf_io_itsmft.cxx
@@ -12,12 +12,19 @@
 #define BOOST_TEST_MODULE Test ITSMFTCTFIO
 #define BOOST_TEST_MAIN
 #define BOOST_TEST_DYN_LINK
+
+#undef NDEBUG
+#include 
+
 #include 
+#include 
+#include 
 #include "DataFormatsITSMFT/CompCluster.h"
 #include "DataFormatsITSMFT/CTF.h"
 #include "DataFormatsITSMFT/ROFRecord.h"
 #include "CommonUtils/NameConf.h"
 #include "ITSMFTReconstruction/CTFCoder.h"
+#include "ITSMFTReconstruction/LookUp.h"
 #include "Framework/Logger.h"
 #include 
 #include 
@@ -25,14 +32,17 @@
 #include 
 
 using namespace o2::itsmft;
+namespace boost_data = boost::unit_test::data;
 
-BOOST_AUTO_TEST_CASE(CompressedClustersTest)
+inline std::vector ANSVersions{o2::ctf::ANSVersionCompat, o2::ctf::ANSVersion1};
+
+BOOST_DATA_TEST_CASE(CompressedClustersTest, boost_data::make(ANSVersions), ansVersion)
 {
 
   std::vector rofRecVec;
   std::vector cclusVec;
   std::vector pattVec;
-
+  LookUp pattIdConverter;
   TStopwatch sw;
   sw.Start();
   std::vector row, col;
@@ -72,7 +82,8 @@ BOOST_AUTO_TEST_CASE(CompressedClustersTest)
   std::vector vec;
   {
     CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Encoder, o2::detectors::DetID::ITS);
-    coder.encode(vec, rofRecVec, cclusVec, pattVec); // compress
+    coder.setANSVersion(ansVersion);
+    coder.encode(vec, rofRecVec, cclusVec, pattVec, pattIdConverter, 0); // compress
   }
   sw.Stop();
   LOG(info) << "Compressed in " << sw.CpuTime() << " s";
diff --git a/Detectors/CTF/test/test_ctf_io_mch.cxx b/Detectors/CTF/test/test_ctf_io_mch.cxx
index 9a6553ba70d81..a12ca58a574da 100644
--- a/Detectors/CTF/test/test_ctf_io_mch.cxx
+++ b/Detectors/CTF/test/test_ctf_io_mch.cxx
@@ -12,7 +12,13 @@
 #define BOOST_TEST_MODULE Test MCHCTFIO
 #define BOOST_TEST_MAIN
 #define BOOST_TEST_DYN_LINK
+
+#undef NDEBUG
+#include 
+
 #include 
+#include 
+#include 
 #include "CommonUtils/NameConf.h"
 #include "MCHCTF/CTFCoder.h"
 #include "DataFormatsMCH/CTF.h"
@@ -26,8 +32,11 @@
 #include 
 
 using namespace o2::mch;
+namespace boost_data = boost::unit_test::data;
+
+inline std::vector ANSVersions{o2::ctf::ANSVersionCompat, o2::ctf::ANSVersion1};
 
-BOOST_AUTO_TEST_CASE(CTFTest, *boost::unit_test::enabled())
+BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions), ansVersion)
 {
   std::vector rofs;
   std::vector digs;
@@ -59,6 +68,7 @@ BOOST_AUTO_TEST_CASE(CTFTest, *boost::unit_test::enabled())
   std::vector vec;
   {
     CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Encoder);
+    coder.setANSVersion(ansVersion);
     coder.encode(vec, rofs, digs); // compress
   }
   sw.Stop();
diff --git a/Detectors/CTF/test/test_ctf_io_mid.cxx b/Detectors/CTF/test/test_ctf_io_mid.cxx
index 0c6d47220835a..2a7122f96e1da 100644
--- a/Detectors/CTF/test/test_ctf_io_mid.cxx
+++ b/Detectors/CTF/test/test_ctf_io_mid.cxx
@@ -12,8 +12,15 @@
 #define BOOST_TEST_MODULE Test MIDCTFIO
 #define BOOST_TEST_MAIN
 #define BOOST_TEST_DYN_LINK
+
+#undef NDEBUG
+#include 
+
 #include 
+#include 
+#include 
 #include "CommonUtils/NameConf.h"
+#include "CommonUtils/IRFrameSelector.h"
 #include "MIDCTF/CTFCoder.h"
 #include "DataFormatsMID/CTF.h"
 #include "Framework/Logger.h"
@@ -24,8 +31,11 @@
 #include 
 
 using namespace o2::mid;
+namespace boost_data = boost::unit_test::data;
+
+inline std::vector ANSVersions{o2::ctf::ANSVersionCompat, o2::ctf::ANSVersion1};
 
-BOOST_AUTO_TEST_CASE(CTFTest)
+BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions), ansVersion)
 {
   std::array, NEvTypes> colData{};
   std::array, NEvTypes> rofData{};
@@ -65,12 +75,14 @@ BOOST_AUTO_TEST_CASE(CTFTest)
     tfData.colData[i] = {colData[i].data(), colData[i].size()};
     tfData.rofData[i] = {rofData[i].data(), rofData[i].size()};
   }
-  tfData.buildReferences();
+  o2::utils::IRFrameSelector irSelector;
+  tfData.buildReferences(irSelector);
 
   sw.Start();
   std::vector vec;
   {
     CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Encoder);
+    coder.setANSVersion(ansVersion);
     coder.encode(vec, tfData); // compress
   }
   sw.Stop();
diff --git a/Detectors/CTF/test/test_ctf_io_phos.cxx b/Detectors/CTF/test/test_ctf_io_phos.cxx
index f564dfc2b8fc8..6850642ba129e 100644
--- a/Detectors/CTF/test/test_ctf_io_phos.cxx
+++ b/Detectors/CTF/test/test_ctf_io_phos.cxx
@@ -12,7 +12,13 @@
 #define BOOST_TEST_MODULE Test PHSCTFIO
 #define BOOST_TEST_MAIN
 #define BOOST_TEST_DYN_LINK
+
+#undef NDEBUG
+#include 
+
 #include 
+#include 
+#include 
 #include "CommonUtils/NameConf.h"
 #include "PHOSReconstruction/CTFCoder.h"
 #include "DataFormatsPHOS/CTF.h"
@@ -24,8 +30,11 @@
 #include 
 
 using namespace o2::phos;
+namespace boost_data = boost::unit_test::data;
+
+inline std::vector ANSVersions{o2::ctf::ANSVersionCompat, o2::ctf::ANSVersion1};
 
-BOOST_AUTO_TEST_CASE(CTFTest)
+BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions), ansVersion)
 {
   std::vector triggers;
   std::vector cells;
@@ -51,6 +60,7 @@ BOOST_AUTO_TEST_CASE(CTFTest)
   std::vector vec;
   {
     CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Encoder);
+    coder.setANSVersion(ansVersion);
     coder.encode(vec, triggers, cells); // compress
   }
   sw.Stop();
diff --git a/Detectors/CTF/test/test_ctf_io_tof.cxx b/Detectors/CTF/test/test_ctf_io_tof.cxx
index b770dbb2f78d5..276e773f77c69 100644
--- a/Detectors/CTF/test/test_ctf_io_tof.cxx
+++ b/Detectors/CTF/test/test_ctf_io_tof.cxx
@@ -12,7 +12,13 @@
 #define BOOST_TEST_MODULE Test TOFCTFIO
 #define BOOST_TEST_MAIN
 #define BOOST_TEST_DYN_LINK
+
+#undef NDEBUG
+#include 
+
 #include 
+#include 
+#include 
 #include "DataFormatsTOF/CTF.h"
 #include "TOFBase/Geo.h"
 #include "TOFBase/Digit.h"
@@ -25,8 +31,11 @@
 #include 
 
 using namespace o2::tof;
+namespace boost_data = boost::unit_test::data;
+
+inline std::vector ANSVersions{o2::ctf::ANSVersionCompat, o2::ctf::ANSVersion1};
 
-BOOST_AUTO_TEST_CASE(CompressedClustersTest)
+BOOST_DATA_TEST_CASE(CompressedClustersTest, boost_data::make(ANSVersions), ansVersion)
 {
 
   std::vector digits;
@@ -86,6 +95,7 @@ BOOST_AUTO_TEST_CASE(CompressedClustersTest)
   std::vector vec;
   {
     CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Encoder);
+    coder.setANSVersion(ansVersion);
     coder.encode(vec, rows, digits, pattVec); // compress
   }
   sw.Stop();
diff --git a/Detectors/CTF/test/test_ctf_io_tpc.cxx b/Detectors/CTF/test/test_ctf_io_tpc.cxx
index 5d9b5e48662e9..cb087e6f2a38f 100644
--- a/Detectors/CTF/test/test_ctf_io_tpc.cxx
+++ b/Detectors/CTF/test/test_ctf_io_tpc.cxx
@@ -12,8 +12,15 @@
 #define BOOST_TEST_MODULE Test TPCCTFIO
 #define BOOST_TEST_MAIN
 #define BOOST_TEST_DYN_LINK
+
+#undef NDEBUG
+#include 
+
 #include 
+#include 
+#include 
 #include "DataFormatsTPC/CompressedClusters.h"
+#include "DataFormatsTPC/ZeroSuppression.h"
 #include "DataFormatsTPC/CTF.h"
 #include "CommonUtils/NameConf.h"
 #include "TPCReconstruction/CTFCoder.h"
@@ -24,15 +31,28 @@
 #include 
 
 using namespace o2::tpc;
+namespace boost_data = boost::unit_test::data;
 
-BOOST_AUTO_TEST_CASE(CTFTest)
+inline std::vector ANSVersions{o2::ctf::ANSVersionCompat, o2::ctf::ANSVersion1};
+inline std::vector CombineColumns(true, false);
+
+BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions) ^ boost_data::make(CombineColumns), ansVersion, combineColumns)
 {
+  std::vector triggers, triggersR;
   CompressedClusters c;
   c.nAttachedClusters = 99;
   c.nUnattachedClusters = 88;
   c.nAttachedClustersReduced = 77;
   c.nTracks = 66;
 
+  triggers.emplace_back();
+  triggers.back().orbit = 1234;
+  triggers.back().triggerWord.triggerEntries[0] = (10 & 0xFFF) | ((o2::tpc::TriggerWordDLBZS::TriggerType::PhT & 0x7) << 12) | 0x8000;
+  triggers.back().triggerWord.triggerEntries[1] = (30 & 0xFFF) | ((o2::tpc::TriggerWordDLBZS::TriggerType::PP & 0x7) << 12) | 0x8000;
+  triggers.emplace_back();
+  triggers.back().orbit = 1236;
+  triggers.back().triggerWord.triggerEntries[0] = (40 & 0xFFF) | ((o2::tpc::TriggerWordDLBZS::TriggerType::Cal & 0x7) << 12) | 0x8000;
+
   std::vector bVec;
   CompressedClustersFlat* ccFlat = nullptr;
   size_t sizeCFlatBody = CTFCoder::alignSize(ccFlat);
@@ -43,7 +63,7 @@ BOOST_AUTO_TEST_CASE(CTFTest)
   {
     CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Encoder);
     coder.setCompClusAddresses(c, buff);
-    coder.setCombineColumns(true);
+    coder.setCombineColumns(combineColumns);
   }
   ccFlat->set(sz, c);
 
@@ -86,9 +106,43 @@ BOOST_AUTO_TEST_CASE(CTFTest)
   sw.Start();
   std::vector vecIO;
   {
-    CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Decoder);
-    coder.setCombineColumns(true);
-    coder.encode(vecIO, c); // compress
+    CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Encoder);
+    coder.setCombineColumns(combineColumns);
+    coder.setANSVersion(ansVersion);
+    // prepare trigger info
+    o2::tpc::detail::TriggerInfo trigComp;
+    for (const auto& trig : triggers) {
+      for (int it = 0; it < o2::tpc::TriggerWordDLBZS::MaxTriggerEntries; it++) {
+        if (trig.triggerWord.isValid(it)) {
+          trigComp.deltaOrbit.push_back(trig.orbit);
+          trigComp.deltaBC.push_back(trig.triggerWord.getTriggerBC(it));
+          trigComp.triggerType.push_back(trig.triggerWord.getTriggerType(it));
+        } else {
+          break;
+        }
+      }
+    }
+    // transform trigger info to differential form
+    uint32_t prevOrbit = -1;
+    uint16_t prevBC = -1;
+    if (trigComp.triggerType.size()) {
+      prevOrbit = trigComp.firstOrbit = trigComp.deltaOrbit[0];
+      prevBC = trigComp.deltaBC[0];
+      trigComp.deltaOrbit[0] = 0;
+      for (size_t it = 1; it < trigComp.triggerType.size(); it++) {
+        if (trigComp.deltaOrbit[it] == prevOrbit) {
+          auto bc = trigComp.deltaBC[it];
+          trigComp.deltaBC[it] -= prevBC;
+          prevBC = bc;
+          trigComp.deltaOrbit[it] = 0;
+        } else {
+          auto orb = trigComp.deltaOrbit[it];
+          trigComp.deltaOrbit[it] -= prevOrbit;
+          prevOrbit = orb;
+        }
+      }
+    }
+    coder.encode(vecIO, c, c, trigComp); // compress
   }
   sw.Stop();
   LOG(info) << "Compressed in " << sw.CpuTime() << " s";
@@ -124,12 +178,24 @@ BOOST_AUTO_TEST_CASE(CTFTest)
   {
     CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Decoder);
     coder.setCombineColumns(true);
-    coder.decode(ctfImage, vecIn); // decompress
+    coder.decode(ctfImage, vecIn, triggersR); // decompress
   }
   sw.Stop();
   LOG(info) << "Decompressed in " << sw.CpuTime() << " s";
   //
   // compare with original flat clusters
   BOOST_CHECK(vecIn.size() == bVec.size());
-  BOOST_CHECK(memcmp(vecIn.data(), bVec.data(), bVec.size()) == 0);
+  const CompressedClustersCounters* countOrig = reinterpret_cast(bVec.data());
+  const CompressedClustersCounters* countDeco = reinterpret_cast(vecIn.data());
+  BOOST_CHECK(countOrig->nTracks == countDeco->nTracks);
+  BOOST_CHECK(countOrig->nAttachedClusters == countDeco->nAttachedClusters);
+  BOOST_CHECK(countOrig->nUnattachedClusters == countDeco->nUnattachedClusters);
+  BOOST_CHECK(countOrig->nAttachedClustersReduced == countDeco->nAttachedClustersReduced);
+  BOOST_CHECK(countOrig->nSliceRows == countDeco->nSliceRows);
+  BOOST_CHECK(countOrig->nComppressionModes == countDeco->nComppressionModes);
+  BOOST_CHECK(countOrig->solenoidBz == countDeco->solenoidBz);
+  BOOST_CHECK(countOrig->maxTimeBin == countDeco->maxTimeBin);
+  BOOST_CHECK(memcmp(vecIn.data() + sizeof(o2::tpc::CompressedClustersCounters), bVec.data() + sizeof(o2::tpc::CompressedClustersCounters), bVec.size() - sizeof(o2::tpc::CompressedClustersCounters)) == 0);
+  BOOST_CHECK(triggers.size() == triggersR.size());
+  BOOST_CHECK(memcmp(triggers.data(), triggersR.data(), triggers.size() * sizeof(o2::tpc::TriggerInfoDLBZS)) == 0);
 }
diff --git a/Detectors/CTF/test/test_ctf_io_trd.cxx b/Detectors/CTF/test/test_ctf_io_trd.cxx
index 158ccd187e6b8..66203074ce817 100644
--- a/Detectors/CTF/test/test_ctf_io_trd.cxx
+++ b/Detectors/CTF/test/test_ctf_io_trd.cxx
@@ -12,7 +12,13 @@
 #define BOOST_TEST_MODULE Test TRDCTFIO
 #define BOOST_TEST_MAIN
 #define BOOST_TEST_DYN_LINK
+
+#undef NDEBUG
+#include 
+
 #include 
+#include 
+#include 
 #include "CommonUtils/NameConf.h"
 #include "TRDReconstruction/CTFCoder.h"
 #include "DataFormatsTRD/CTF.h"
@@ -24,8 +30,11 @@
 #include 
 
 using namespace o2::trd;
+namespace boost_data = boost::unit_test::data;
+
+inline std::vector ANSVersions{o2::ctf::ANSVersionCompat, o2::ctf::ANSVersion1};
 
-BOOST_AUTO_TEST_CASE(CTFTest)
+BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions), ansVersion)
 {
   std::vector triggers;
   std::vector tracklets;
@@ -70,6 +79,7 @@ BOOST_AUTO_TEST_CASE(CTFTest)
   std::vector vec;
   {
     CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Encoder);
+    coder.setANSVersion(ansVersion);
     coder.encode(vec, triggers, tracklets, digits); // compress
   }
   sw.Stop();
diff --git a/Detectors/CTF/test/test_ctf_io_zdc.cxx b/Detectors/CTF/test/test_ctf_io_zdc.cxx
index 6222951c080e6..3ae6f5b991ad1 100644
--- a/Detectors/CTF/test/test_ctf_io_zdc.cxx
+++ b/Detectors/CTF/test/test_ctf_io_zdc.cxx
@@ -12,7 +12,13 @@
 #define BOOST_TEST_MODULE Test ZDCCTFIO
 #define BOOST_TEST_MAIN
 #define BOOST_TEST_DYN_LINK
+
+#undef NDEBUG
+#include 
+
 #include 
+#include 
+#include 
 #include "CommonUtils/NameConf.h"
 #include "ZDCReconstruction/CTFCoder.h"
 #include "DataFormatsZDC/CTF.h"
@@ -24,8 +30,11 @@
 #include 
 
 using namespace o2::zdc;
+namespace boost_data = boost::unit_test::data;
+
+inline std::vector ANSVersions{o2::ctf::ANSVersionCompat, o2::ctf::ANSVersion1};
 
-BOOST_AUTO_TEST_CASE(CTFTest)
+BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions), ansVersion)
 {
   std::vector bcdata;
   std::vector chandata;
@@ -72,7 +81,7 @@ BOOST_AUTO_TEST_CASE(CTFTest)
     pedsdata[i].ir = irPed;
     for (int ic = 0; ic < NChannels; ic++) {
       pedsdata[i].data[ic] = gRandom->Integer(0xffff);
-      pedsdata[i].scaler[ic] = (i > 0 ? pedsdata[i].scaler[ic - 1] : 0) + gRandom->Integer(20);
+      pedsdata[i].scaler[ic] = (ic > 0 ? pedsdata[i].scaler[ic - 1] : 0) + gRandom->Integer(20);
     }
     irPed.orbit++;
   }
@@ -81,6 +90,7 @@ BOOST_AUTO_TEST_CASE(CTFTest)
   std::vector vec;
   {
     CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Encoder);
+    coder.setANSVersion(ansVersion);
     coder.encode(vec, bcdata, chandata, pedsdata); // compress
   }
   sw.Stop();
diff --git a/Detectors/CTF/utils/CMakeLists.txt b/Detectors/CTF/utils/CMakeLists.txt
index 025f7586a0e45..f1b55e027f64a 100644
--- a/Detectors/CTF/utils/CMakeLists.txt
+++ b/Detectors/CTF/utils/CMakeLists.txt
@@ -29,3 +29,7 @@ o2_add_test_root_macro(dumpCTF.C
 o2_add_test_root_macro(CTFdict2CCDBfiles.C
                        PUBLIC_LINK_LIBRARIES O2::CTFWorkflow fmt::fmt
                        LABELS ctf COMPILE_ONLY)
+
+o2_add_test_root_macro(convCTFDict.C
+                       PUBLIC_LINK_LIBRARIES O2::CTFWorkflow fmt::fmt
+                       LABELS ctf COMPILE_ONLY)
diff --git a/Detectors/CTF/utils/convCTFDict.C b/Detectors/CTF/utils/convCTFDict.C
new file mode 100644
index 0000000000000..1b42cda3e2a1b
--- /dev/null
+++ b/Detectors/CTF/utils/convCTFDict.C
@@ -0,0 +1,121 @@
+// Copyright 2019-2023 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#if !defined(__CLING__) || defined(__ROOTCLING__)
+
+#include 
+#include "DetectorsCommonDataFormats/CTFHeader.h"
+#include "DetectorsCommonDataFormats/CTFDictHeader.h"
+#include "DetectorsCommonDataFormats/EncodedBlocks.h"
+#include "DataFormatsCTP/CTF.h"
+#include "DataFormatsTRD/CTF.h"
+#include "DataFormatsTOF/CTF.h"
+#include "DataFormatsTPC/CTF.h"
+#include "DataFormatsFT0/CTF.h"
+#include "DataFormatsFV0/CTF.h"
+#include "DataFormatsFDD/CTF.h"
+#include "DataFormatsEMCAL/CTF.h"
+#include "DataFormatsITSMFT/CTF.h"
+#include "DataFormatsPHOS/CTF.h"
+#include "DataFormatsZDC/CTF.h"
+#include "DataFormatsMID/CTF.h"
+#include "DataFormatsMCH/CTF.h"
+#include "DataFormatsHMP/CTF.h"
+#include "DataFormatsCPV/CTF.h"
+
+#include 
+#include 
+
+#endif
+
+template 
+size_t appendToTree(TTree& tree, const std::string brname, T& ptr)
+{
+  size_t s = 0;
+  auto* br = tree.GetBranch(brname.c_str());
+  auto* pptr = &ptr;
+  if (br) {
+    br->SetAddress(&pptr);
+  } else {
+    br = tree.Branch(brname.c_str(), &pptr);
+  }
+  int res = br->Fill();
+  if (res < 0) {
+    throw std::runtime_error(fmt::format("Failed to fill CTF branch {}", brname));
+  }
+  s += res;
+  br->ResetAddress();
+  return s;
+}
+
+template 
+void conv(const std::vector* buff)
+{
+  const auto& ctf = C::getImage(buff->data());
+  const auto& ctfhead = ctf.getHeader();
+  const auto& dictHead = (const o2::ctf::CTFDictHeader&)ctfhead;
+  o2::ctf::CTFHeader header{};
+  TFile fl(Form("ctf_dictionaryTree_%s_%u_0.root", dictHead.det.getName(), dictHead.dictTimeStamp), "recreate");
+  TTree* tree = new TTree("ccdb_object", "O2 CTF dictionary");
+  header.detectors.set(dictHead.det);
+  ctf.appendToTree(*tree, dictHead.det.getName());
+  appendToTree(*tree, "CTFHeader", header);
+  tree->SetEntries(1);
+  tree->Write(tree->GetName(), TObject::kSingleKey);
+  delete tree;
+  fl.Close();
+  printf("Stored dictionary to %s\n", fl.GetName());
+}
+
+void convCTFDict(const char* det, long ts = 0, const char* ccdburl = "http://alice-ccdb.cern.ch")
+{
+  auto& cm = o2::ccdb::BasicCCDBManager::instance();
+  if (ts > 0) {
+    cm.setTimestamp(ts);
+  }
+  std::string pth = std::string(Form("%s/Calib/CTFDictionary", det));
+  const auto* buff = cm.get>(pth);
+  if (!buff) {
+    printf("Failed to fetch from %s for %ld\n", pth.c_str(), cm.getTimestamp());
+  }
+  std::string dets{det};
+  if (dets == "ITS" || dets == "MFT") {
+    conv(buff);
+  } else if (dets == "TPC") {
+    conv(buff);
+  } else if (dets == "TRD") {
+    conv(buff);
+  } else if (dets == "TOF") {
+    conv(buff);
+  } else if (dets == "FT0") {
+    conv(buff);
+  } else if (dets == "FV0") {
+    conv(buff);
+  } else if (dets == "FDD") {
+    conv(buff);
+  } else if (dets == "EMC") {
+    conv(buff);
+  } else if (dets == "PHS") {
+    conv(buff);
+  } else if (dets == "ZDC") {
+    conv(buff);
+  } else if (dets == "MID") {
+    conv(buff);
+  } else if (dets == "MCH") {
+    conv(buff);
+  } else if (dets == "HMP") {
+    conv(buff);
+  } else if (dets == "CTP") {
+    conv(buff);
+  } else if (dets == "CPV") {
+    conv(buff);
+  }
+}
diff --git a/Detectors/CTF/workflow/CMakeLists.txt b/Detectors/CTF/workflow/CMakeLists.txt
index 493f22578da94..f7adeaed991c0 100644
--- a/Detectors/CTF/workflow/CMakeLists.txt
+++ b/Detectors/CTF/workflow/CMakeLists.txt
@@ -9,6 +9,8 @@
 # granted to it by virtue of its status as an Intergovernmental Organization
 # or submit itself to any jurisdiction.
 
+#add_compile_options(-O0 -g -fPIC)
+
 o2_add_library(CTFWorkflow
                SOURCES src/CTFWriterSpec.cxx
                        src/CTFReaderSpec.cxx
@@ -35,7 +37,7 @@ o2_add_library(CTFWorkflow
                                      O2::FT0Workflow
                                      O2::FV0Workflow
                                      O2::FDDWorkflow
-                                     O2::MCHWorkflow
+                                     O2::MCHCTF
                                      O2::MIDWorkflow
                                      O2::EMCALWorkflow
                                      O2::PHOSWorkflow
@@ -54,5 +56,10 @@ o2_add_executable(writer-workflow
 o2_add_executable(reader-workflow
                   SOURCES src/ctf-reader-workflow.cxx
                   COMPONENT_NAME ctf
+                  TARGETVARNAME targetName
                   PUBLIC_LINK_LIBRARIES O2::CTFWorkflow)
 
+if (OpenMP_CXX_FOUND)
+    target_compile_definitions(${targetName} PRIVATE WITH_OPENMP)
+    target_link_libraries(${targetName} PRIVATE OpenMP::OpenMP_CXX)
+endif()
diff --git a/Detectors/CTF/workflow/include/CTFWorkflow/CTFReaderSpec.h b/Detectors/CTF/workflow/include/CTFWorkflow/CTFReaderSpec.h
index 8e74cd4ae4546..081e6cf4d968a 100644
--- a/Detectors/CTF/workflow/include/CTFWorkflow/CTFReaderSpec.h
+++ b/Detectors/CTF/workflow/include/CTFWorkflow/CTFReaderSpec.h
@@ -30,16 +30,26 @@ struct CTFReaderInp {
   std::string tffileRegex{};
   std::string remoteRegex{};
   std::string metricChannel{};
+  std::string fileIRFrames{};
+  std::string fileRunTimeSpans{};
+  std::string dictOpt{};
   std::vector ctfIDs{};
+  bool reverseCTFIDs{false};
+  bool skipSkimmedOutTF = false;
+  bool invertIRFramesSelection = false;
   bool allowMissingDetectors = false;
+  bool checkTFLimitBeforeReading = false;
   bool sup0xccdb = false;
   int maxFileCache = 1;
   int64_t delay_us = 0;
   int maxLoops = 0;
   int maxTFs = -1;
+  int maxTFsPerFile = -1;
   unsigned int subspec = 0;
-  int tfRateLimit = 0;
+  unsigned int decSSpecEMC = 0;
+  int tfRateLimit = -999;
   size_t minSHM = 0;
+  bool shuffle{false};
 };
 
 /// create a processor spec
diff --git a/Detectors/CTF/workflow/include/CTFWorkflow/CTFWriterSpec.h b/Detectors/CTF/workflow/include/CTFWorkflow/CTFWriterSpec.h
index 363938b0f0676..5eb6d65e26cec 100644
--- a/Detectors/CTF/workflow/include/CTFWorkflow/CTFWriterSpec.h
+++ b/Detectors/CTF/workflow/include/CTFWorkflow/CTFWriterSpec.h
@@ -24,7 +24,7 @@ namespace ctf
 {
 
 /// create a processor spec
-framework::DataProcessorSpec getCTFWriterSpec(o2::detectors::DetID::mask_t dets, uint64_t run, const std::string& outType, int verbosity, int reportInterval);
+framework::DataProcessorSpec getCTFWriterSpec(o2::detectors::DetID::mask_t dets, const std::string& outType, int verbosity, int reportInterval);
 
 } // namespace ctf
 } // namespace o2
diff --git a/Detectors/CTF/workflow/src/CTFReaderSpec.cxx b/Detectors/CTF/workflow/src/CTFReaderSpec.cxx
index 4a79d55130d39..4100ebb37c61d 100644
--- a/Detectors/CTF/workflow/src/CTFReaderSpec.cxx
+++ b/Detectors/CTF/workflow/src/CTFReaderSpec.cxx
@@ -11,9 +11,14 @@
 
 /// @file   CTFReaderSpec.cxx
 
+#include 
 #include 
+#include 
+#include 
+
 #include 
 #include 
+#include 
 
 #include "Framework/Logger.h"
 #include "Framework/ControlService.h"
@@ -23,6 +28,8 @@
 #include "Framework/RateLimiter.h"
 #include "CommonUtils/StringUtils.h"
 #include "CommonUtils/FileFetcher.h"
+#include "CommonUtils/IRFrameSelector.h"
+#include "DetectorsRaw/HBFUtils.h"
 #include "CTFWorkflow/CTFReaderSpec.h"
 #include "DetectorsCommonDataFormats/EncodedBlocks.h"
 #include "CommonUtils/NameConf.h"
@@ -43,8 +50,10 @@
 #include "DataFormatsZDC/CTF.h"
 #include "DataFormatsHMP/CTF.h"
 #include "DataFormatsCTP/CTF.h"
+#include "DataFormatsParameters/AggregatedRunInfo.h"
+#include "CCDB/BasicCCDBManager.h"
+#include "CommonConstants/LHCConstants.h"
 #include "Algorithm/RangeTokenizer.h"
-#include 
 #include 
 
 using namespace o2::framework;
@@ -79,8 +88,10 @@ class CTFReaderSpec : public o2::framework::Task
   void run(o2::framework::ProcessingContext& pc) final;
 
  private:
+  void runTimeRangesToIRFrameSelector(const o2::framework::TimingInfo& timingInfo);
+  void loadRunTimeSpans(const std::string& flname);
   void openCTFFile(const std::string& flname);
-  void processTF(ProcessingContext& pc);
+  bool processTF(ProcessingContext& pc);
   void checkTreeEntries();
   void stopReader();
   template 
@@ -88,14 +99,23 @@ class CTFReaderSpec : public o2::framework::Task
   void setMessageHeader(ProcessingContext& pc, const CTFHeader& ctfHeader, const std::string& lbl, unsigned subspec) const; // keep just for the reference
   void tryToFixCTFHeader(CTFHeader& ctfHeader) const;
   CTFReaderInp mInput{};
+  o2::utils::IRFrameSelector mIRFrameSelector; // optional IR frames selector
+  std::map>> mRunTimeRanges;
   std::unique_ptr mFileFetcher;
   std::unique_ptr mCTFFile;
   std::unique_ptr mCTFTree;
   bool mRunning = false;
   bool mUseLocalTFCounter = false;
+  bool mIFRamesOut = false;
+  int mConvRunTimeRangesToOrbits = -1; // not defined yet
   int mCTFCounter = 0;
+  int mCTFCounterAcc = 0;
   int mNFailedFiles = 0;
   int mFilesRead = 0;
+  int mTFLength = 32;
+  int mNWaits = 0;
+  int mRunNumberPrev = -1;
+  long mTotalWaitTime = 0;
   long mLastSendTime = 0L;
   long mCurrTreeEntry = 0L;
   long mImposeRunStartMS = 0L;
@@ -123,8 +143,8 @@ void CTFReaderSpec::stopReader()
     return;
   }
   LOGP(info, "CTFReader stops processing, {} files read, {} files failed", mFilesRead - mNFailedFiles, mNFailedFiles);
-  LOGP(info, "CTF reading total timing: Cpu: {:.3f} Real: {:.3f} s for {} TFs in {} loops",
-       mTimer.CpuTime(), mTimer.RealTime(), mCTFCounter, mFileFetcher->getNLoops());
+  LOGP(info, "CTF reading total timing: Cpu: {:.3f} Real: {:.3f} s for {} TFs ({} accepted) in {} loops, spent {:.2} s in {} data waiting states",
+       mTimer.CpuTime(), mTimer.RealTime(), mCTFCounter, mCTFCounterAcc, mFileFetcher->getNLoops(), 1e-6 * mTotalWaitTime, mNWaits);
   mRunning = false;
   mFileFetcher->stop();
   mFileFetcher.reset();
@@ -139,13 +159,135 @@ void CTFReaderSpec::stopReader()
 void CTFReaderSpec::init(InitContext& ic)
 {
   mInput.ctfIDs = o2::RangeTokenizer::tokenize(ic.options().get("select-ctf-ids"));
+  if ((mInput.reverseCTFIDs = ic.options().get("reverse-select-ctf-ids"))) {
+    std::reverse(mInput.ctfIDs.begin(), mInput.ctfIDs.end());
+  }
   mUseLocalTFCounter = ic.options().get("local-tf-counter");
   mImposeRunStartMS = ic.options().get("impose-run-start-timstamp");
+  mInput.checkTFLimitBeforeReading = ic.options().get("limit-tf-before-reading");
+  mInput.maxTFs = ic.options().get("max-tf");
+  mInput.maxTFs = mInput.maxTFs > 0 ? mInput.maxTFs : 0x7fffffff;
+  mInput.maxTFsPerFile = ic.options().get("max-tf-per-file");
+  mInput.maxTFsPerFile = mInput.maxTFsPerFile > 0 ? mInput.maxTFsPerFile : 0x7fffffff;
   mRunning = true;
   mFileFetcher = std::make_unique(mInput.inpdata, mInput.tffileRegex, mInput.remoteRegex, mInput.copyCmd);
   mFileFetcher->setMaxFilesInQueue(mInput.maxFileCache);
   mFileFetcher->setMaxLoops(mInput.maxLoops);
+  mFileFetcher->setFailThreshold(ic.options().get("fetch-failure-threshold"));
   mFileFetcher->start();
+  if (!mInput.fileIRFrames.empty()) {
+    mIRFrameSelector.loadIRFrames(mInput.fileIRFrames);
+    const auto& hbfu = o2::raw::HBFUtils::Instance();
+    mTFLength = hbfu.nHBFPerTF;
+    LOGP(info, "IRFrames will be selected from {}, assumed TF length: {} HBF", mInput.fileIRFrames, mTFLength);
+    mIFRamesOut = true;
+  }
+  if (!mInput.fileRunTimeSpans.empty()) {
+    loadRunTimeSpans(mInput.fileRunTimeSpans);
+    mIFRamesOut = true;
+  }
+}
+
+void CTFReaderSpec::runTimeRangesToIRFrameSelector(const o2::framework::TimingInfo& timingInfo)
+{
+  // convert entries in the runTimeRanges to IRFrameSelector, if needed, convert time to orbit
+  mIRFrameSelector.clear();
+  auto ent = mRunTimeRanges.find(timingInfo.runNumber);
+  if (ent == mRunTimeRanges.end()) {
+    LOGP(info, "RunTimeRanges selection was provided but run {} has no entries, all TFs will be processed", timingInfo.runNumber);
+    return;
+  }
+  o2::parameters::AggregatedRunInfo rinfo;
+  auto& ccdb = o2::ccdb::BasicCCDBManager::instance();
+  rinfo = o2::parameters::AggregatedRunInfo::buildAggregatedRunInfo(ccdb, timingInfo.runNumber);
+  if (rinfo.runNumber != timingInfo.runNumber || rinfo.orbitsPerTF < 1) {
+    LOGP(fatal, "failed to extract AggregatedRunInfo for run {}", timingInfo.runNumber);
+  }
+  mTFLength = rinfo.orbitsPerTF;
+  std::vector frames;
+  for (const auto& rng : ent->second) {
+    long orbMin = 0, orbMax = 0;
+    if (mConvRunTimeRangesToOrbits > 0) {
+      orbMin = rinfo.orbitSOR + (rng.first - rinfo.sor) / (o2::constants::lhc::LHCOrbitMUS * 0.001);
+      orbMax = rinfo.orbitSOR + (rng.second - rinfo.sor) / (o2::constants::lhc::LHCOrbitMUS * 0.001);
+    } else {
+      orbMin = rng.first;
+      orbMax = rng.second;
+    }
+    if (orbMin < 0) {
+      orbMin = 0;
+    }
+    if (orbMax < 0) {
+      orbMax = 0;
+    }
+    if (timingInfo.runNumber > 523897) {
+      orbMin = (orbMin / rinfo.orbitsPerTF) * rinfo.orbitsPerTF;
+      orbMax = (orbMax / rinfo.orbitsPerTF + 1) * rinfo.orbitsPerTF - 1;
+    }
+    LOGP(info, "TFs overlapping with orbits {}:{} will be {}", orbMin, orbMax, mInput.invertIRFramesSelection ? "rejected" : "selected");
+    frames.emplace_back(InteractionRecord{0, uint32_t(orbMin)}, InteractionRecord{o2::constants::lhc::LHCMaxBunches, uint32_t(orbMax)});
+  }
+  mIRFrameSelector.setOwnList(frames, true);
+}
+
+void CTFReaderSpec::loadRunTimeSpans(const std::string& flname)
+{
+  std::ifstream inputFile(flname);
+  if (!inputFile) {
+    LOGP(fatal, "Failed to open selected run/timespans file {}", flname);
+  }
+  std::string line;
+  size_t cntl = 0, cntr = 0;
+  while (std::getline(inputFile, line)) {
+    cntl++;
+    for (char& ch : line) { // Replace semicolons and tabs with spaces for uniform processing
+      if (ch == ';' || ch == '\t' || ch == ',') {
+        ch = ' ';
+      }
+    }
+    o2::utils::Str::trim(line);
+    if (line.size() < 1 || line[0] == '#') {
+      continue;
+    }
+    auto tokens = o2::utils::Str::tokenize(line, ' ');
+    auto logError = [&cntl, &line]() { LOGP(error, "Expected format for selection is tripplet   , failed on line#{}: {}", cntl, line); };
+    if (tokens.size() >= 3) {
+      int run = 0;
+      long rmin, rmax;
+      try {
+        run = std::stoi(tokens[0]);
+        rmin = std::stol(tokens[1]);
+        rmax = std::stol(tokens[2]);
+      } catch (...) {
+        logError();
+        continue;
+      }
+
+      constexpr long ISTimeStamp = 1514761200000L;
+      int convmn = rmin > ISTimeStamp ? 1 : 0, convmx = rmax > ISTimeStamp ? 1 : 0; // values above ISTimeStamp are timestamps (need to be converted to orbits)
+      if (rmin > rmax) {
+        LOGP(fatal, "Provided range limits are not in increasing order, entry is {}", line);
+      }
+      if (mConvRunTimeRangesToOrbits == -1) {
+        if (convmn != convmx) {
+          LOGP(fatal, "Provided range limits should be both consistent either with orbit number or with unix timestamp in ms, entry is {}", line);
+        }
+        mConvRunTimeRangesToOrbits = convmn; // need to convert to orbit if time
+        LOGP(info, "Interpret selected time-spans input as {}", mConvRunTimeRangesToOrbits == 1 ? "timstamps(ms)" : "orbits");
+      } else {
+        if (mConvRunTimeRangesToOrbits != convmn || mConvRunTimeRangesToOrbits != convmx) {
+          LOGP(fatal, "Provided range limits should are not consistent with previously determined {} input, entry is {}", mConvRunTimeRangesToOrbits == 1 ? "timestamps" : "orbits", line);
+        }
+      }
+
+      mRunTimeRanges[run].emplace_back(rmin, rmax);
+      cntr++;
+    } else {
+      logError();
+    }
+  }
+  LOGP(info, "Read {} time-spans for {} runs from {}", cntr, mRunTimeRanges.size(), flname);
+  inputFile.close();
 }
 
 ///_______________________________________
@@ -155,11 +297,35 @@ void CTFReaderSpec::openCTFFile(const std::string& flname)
     mFilesRead++;
     mCTFFile.reset(TFile::Open(flname.c_str()));
     if (!mCTFFile || !mCTFFile->IsOpen() || mCTFFile->IsZombie()) {
-      throw std::runtime_error("failed to open CTF file");
+      throw std::runtime_error(fmt::format("failed to open CTF file {}, skipping", flname));
     }
     mCTFTree.reset((TTree*)mCTFFile->Get(std::string(o2::base::NameConf::CTFTREENAME).c_str()));
     if (!mCTFTree) {
-      throw std::runtime_error("failed to load CTF tree from");
+      throw std::runtime_error(fmt::format("failed to load CTF tree from {}, skipping", flname));
+    }
+    if (mCTFTree->GetEntries() < 1) {
+      throw std::runtime_error(fmt::format("CTF tree in {} has 0 entries, skipping", flname));
+    }
+    if (mInput.shuffle) {
+      if (mInput.ctfIDs.empty()) {
+        int entries = mCTFTree->GetEntries();
+        if (mInput.maxTFs > 0) {
+          entries = std::min(entries, mInput.maxTFs);
+        }
+        if (mInput.maxTFsPerFile > 0) {
+          entries = std::min(entries, mInput.maxTFsPerFile);
+        }
+        mInput.ctfIDs.clear();
+        mInput.ctfIDs.resize(entries);
+        std::iota(mInput.ctfIDs.begin(), mInput.ctfIDs.end(), 0);
+      }
+      std::random_device dev;
+      std::mt19937 gen{dev()};
+      std::shuffle(mInput.ctfIDs.begin(), mInput.ctfIDs.end(), gen);
+      LOGP(info, "will shuffle reading of CTF entries in this order:");
+      for (int i{0}; i < (int)mInput.ctfIDs.size(); ++i) {
+        LOGP(info, "\tTF {:02} -> {:02}", i, mInput.ctfIDs[i]);
+      }
     }
   } catch (const std::exception& e) {
     LOG(error) << "Cannot process " << flname << ", reason: " << e.what();
@@ -176,30 +342,30 @@ void CTFReaderSpec::openCTFFile(const std::string& flname)
 ///_______________________________________
 void CTFReaderSpec::run(ProcessingContext& pc)
 {
-  static bool initOnceDone = false;
-  if (!initOnceDone) {
+  if (mInput.tfRateLimit == -999) {
     mInput.tfRateLimit = std::stoi(pc.services().get().device()->fConfig->GetValue("timeframes-rate-limit"));
   }
-
   std::string tfFileName;
-  if (mCTFCounter >= mInput.maxTFs || (!mInput.ctfIDs.empty() && mSelIDEntry >= mInput.ctfIDs.size())) { // done
-    LOG(info) << "All CTFs from selected range were injected, stopping";
-    mRunning = false;
-  }
+  bool waitAcknowledged = false;
+  long startWait = 0;
 
   while (mRunning) {
-    if (mCTFTree) { // there is a tree open with multiple CTF
-      if (mInput.ctfIDs.empty() || mInput.ctfIDs[mSelIDEntry] == mCTFCounter) { // no selection requested or matching CTF ID is found
+    if (mCTFTree) {                                                                                                       // there is a tree open with multiple CTF
+      if (mInput.ctfIDs.empty() || mInput.ctfIDs[mSelIDEntry] == mCTFCounter || mInput.shuffle || mInput.reverseCTFIDs) { // no selection requested or matching CTF ID is found
         LOG(debug) << "TF " << mCTFCounter << " of " << mInput.maxTFs << " loop " << mFileFetcher->getNLoops();
+        if (mInput.shuffle || mInput.reverseCTFIDs) {
+          mCurrTreeEntry = mInput.ctfIDs[mSelIDEntry];
+        }
         mSelIDEntry++;
-        processTF(pc);
-        break;
-      } else { // explict CTF ID selection list was provided and current entry is not selected
-        LOGP(info, "Skipping CTF${} ({} of {} in {})", mCTFCounter, mCurrTreeEntry, mCTFTree->GetEntries(), mCTFFile->GetName());
-        checkTreeEntries();
-        mCTFCounter++;
-        continue;
+        if (processTF(pc)) {
+          break;
+        }
       }
+      // explict CTF ID selection list or IRFrame was provided and current entry is not selected
+      LOGP(info, "Skipping CTF#{} ({} of {} in {})", mCTFCounter, mCurrTreeEntry, mCTFTree->GetEntries(), mCTFFile->GetName());
+      checkTreeEntries();
+      mCTFCounter++;
+      continue;
     }
     //
     tfFileName = mFileFetcher->getNextFileInQueue();
@@ -208,29 +374,57 @@ void CTFReaderSpec::run(ProcessingContext& pc)
         mRunning = false;
         break;
       }
+      if (!waitAcknowledged) {
+        startWait = std::chrono::time_point_cast(std::chrono::system_clock::now()).time_since_epoch().count();
+        waitAcknowledged = true;
+      }
       pc.services().get().waitFor(5);
       continue;
     }
+    if (waitAcknowledged) {
+      long waitTime = std::chrono::time_point_cast(std::chrono::system_clock::now()).time_since_epoch().count() - startWait;
+      mTotalWaitTime += waitTime;
+      if (++mNWaits > 1) {
+        LOGP(warn, "Resuming reading after waiting for data {:.2} s (accumulated {:.2} s delay in {} waits)", 1e-6 * waitTime, 1e-6 * mTotalWaitTime, mNWaits);
+      }
+      waitAcknowledged = false;
+    }
     LOG(info) << "Reading CTF input " << ' ' << tfFileName;
     openCTFFile(tfFileName);
   }
 
+  if (mCTFCounter >= mInput.maxTFs || (!mInput.ctfIDs.empty() && mSelIDEntry >= mInput.ctfIDs.size())) { // done
+    LOGP(info, "All CTFs from selected range were injected, stopping");
+    mRunning = false;
+  } else if (mRunning && !mCTFTree && mFileFetcher->getNextFileInQueue().empty() && !mFileFetcher->isRunning()) { // previous tree was done, can we read more?
+    mRunning = false;
+  }
+
   if (!mRunning) {
     pc.services().get().endOfStream();
     pc.services().get().readyToQuit(QuitRequest::Me);
     stopReader();
+    const std::string dummy{"ctf_read_ntf.txt"};
+    if (mCTFCounterAcc == 0) {
+      LOGP(warn, "No TF passed selection, writing a 0 to file {}", dummy);
+    }
+    try {
+      std::ofstream outfile;
+      outfile.open(dummy, std::ios::out | std::ios::trunc);
+      outfile << mCTFCounterAcc << std::endl;
+    } catch (...) {
+      LOGP(error, "Failed to write {}", dummy);
+    }
   }
 }
 
 ///_______________________________________
-void CTFReaderSpec::processTF(ProcessingContext& pc)
+bool CTFReaderSpec::processTF(ProcessingContext& pc)
 {
   auto cput = mTimer.CpuTime();
   mTimer.Start(false);
 
   static RateLimiter limiter;
-  limiter.check(pc, mInput.tfRateLimit, mInput.minSHM);
-
   CTFHeader ctfHeader;
   if (!readFromTree(*(mCTFTree.get()), "CTFHeader", ctfHeader, mCurrTreeEntry)) {
     throw std::runtime_error("did not find CTFHeader");
@@ -243,7 +437,7 @@ void CTFReaderSpec::processTF(ProcessingContext& pc)
   }
 
   if (mUseLocalTFCounter) {
-    ctfHeader.tfCounter = mCTFCounter;
+    ctfHeader.tfCounter = mCTFCounterAcc;
   }
 
   LOG(info) << ctfHeader;
@@ -254,6 +448,35 @@ void CTFReaderSpec::processTF(ProcessingContext& pc)
   timingInfo.tfCounter = ctfHeader.tfCounter;
   timingInfo.runNumber = ctfHeader.run;
 
+  if (mRunTimeRanges.size() && timingInfo.runNumber != mRunNumberPrev) {
+    runTimeRangesToIRFrameSelector(timingInfo);
+  }
+  mRunNumberPrev = timingInfo.runNumber;
+  gsl::span irSpan{};
+  if (mIRFrameSelector.isSet()) {
+    o2::InteractionRecord ir0(0, timingInfo.firstTForbit);
+    o2::InteractionRecord ir1(o2::constants::lhc::LHCMaxBunches - 1, timingInfo.firstTForbit < 0xffffffff - (mTFLength - 1) ? timingInfo.firstTForbit + (mTFLength - 1) : 0xffffffff);
+    irSpan = mIRFrameSelector.getMatchingFrames({ir0, ir1});
+    bool acc = true;
+    if (mInput.skipSkimmedOutTF) {
+      acc = (irSpan.size() > 0) ? !mInput.invertIRFramesSelection : mInput.invertIRFramesSelection;
+      LOGP(info, "IRFrame selection contains {} frames for TF [{}] : [{}]: {}use this TF (selection inversion mode is {})",
+           irSpan.size(), ir0.asString(), ir1.asString(), acc ? "" : "do not ", mInput.invertIRFramesSelection ? "ON" : "OFF");
+    }
+    if (!acc) {
+      return false;
+    }
+    if (mInput.checkTFLimitBeforeReading) {
+      limiter.check(pc, mInput.tfRateLimit, mInput.minSHM);
+    }
+  } else {
+    if (mInput.checkTFLimitBeforeReading) {
+      limiter.check(pc, mInput.tfRateLimit, mInput.minSHM);
+    }
+  }
+  if (mIFRamesOut) {
+    auto outVec = pc.outputs().make>(OutputRef{"selIRFrames"}, irSpan.begin(), irSpan.end());
+  }
   // send CTF Header
   pc.outputs().snapshot({"header", mInput.subspec}, ctfHeader);
 
@@ -273,6 +496,7 @@ void CTFReaderSpec::processTF(ProcessingContext& pc)
   processDetector(DetID::CPV, ctfHeader, pc);
   processDetector(DetID::ZDC, ctfHeader, pc);
   processDetector(DetID::CTP, ctfHeader, pc);
+  mCTFCounterAcc++;
 
   // send sTF acknowledge message
   if (!mInput.sup0xccdb) {
@@ -286,25 +510,34 @@ void CTFReaderSpec::processTF(ProcessingContext& pc)
   checkTreeEntries();
   mTimer.Stop();
 
-  // do we need to way to respect the delay ?
+  // do we need to wait to respect the delay ?
   long tNow = std::chrono::time_point_cast(std::chrono::system_clock::now()).time_since_epoch().count();
-  auto tDiff = tNow - mLastSendTime;
   if (mCTFCounter) {
+    auto tDiff = tNow - mLastSendTime;
     if (tDiff < mInput.delay_us) {
       pc.services().get().waitFor((mInput.delay_us - tDiff) / 1000); // respect requested delay before sending
     }
   }
+  if (!mInput.checkTFLimitBeforeReading) {
+    limiter.check(pc, mInput.tfRateLimit, mInput.minSHM);
+  }
   tNow = std::chrono::time_point_cast(std::chrono::system_clock::now()).time_since_epoch().count();
-  LOGP(info, "Read CTF {} {} in {:.3f} s, {:.4f} s elapsed from previous CTF", mCTFCounter, entryStr, mTimer.CpuTime() - cput, 1e-6 * (tNow - mLastSendTime));
+  LOGP(info, "Read CTF {} {} in {:.3f} s, {:.4f} s elapsed from previous CTF", mCTFCounter, entryStr, mTimer.CpuTime() - cput, mCTFCounter ? 1e-6 * (tNow - mLastSendTime) : 0.);
   mLastSendTime = tNow;
   mCTFCounter++;
+  return true;
 }
 
 ///_______________________________________
 void CTFReaderSpec::checkTreeEntries()
 {
-  // check if the tree has entries left, if needed, close current tree/file
-  if (++mCurrTreeEntry >= mCTFTree->GetEntries()) { // this file is done, check if there are other files
+  bool reachedEnd{false};
+  if (mInput.shuffle || mInput.reverseCTFIDs) { // last entry is last id
+    reachedEnd = (mCurrTreeEntry == mInput.ctfIDs.back());
+  } else { // check if the tree has entries left, if needed, close current tree/file
+    reachedEnd = (++mCurrTreeEntry >= mCTFTree->GetEntries());
+  }
+  if (reachedEnd || (mInput.maxTFsPerFile > 0 && mCurrTreeEntry >= mInput.maxTFsPerFile)) { // this file is done, check if there are other files
     mCTFTree.reset();
     mCTFFile->Close();
     mCTFFile.reset();
@@ -395,6 +628,7 @@ void CTFReaderSpec::tryToFixCTFHeader(CTFHeader& ctfHeader) const
 ///_______________________________________
 DataProcessorSpec getCTFReaderSpec(const CTFReaderInp& inp)
 {
+  std::vector inputs;
   std::vector outputs;
   std::vector options;
 
@@ -405,20 +639,28 @@ DataProcessorSpec getCTFReaderSpec(const CTFReaderInp& inp)
       outputs.emplace_back(OutputLabel{det.getName()}, det.getDataOrigin(), "CTFDATA", inp.subspec, Lifetime::Timeframe);
     }
   }
+  if (!inp.fileIRFrames.empty() || !inp.fileRunTimeSpans.empty()) {
+    outputs.emplace_back(OutputLabel{"selIRFrames"}, "CTF", "SELIRFRAMES", 0, Lifetime::Timeframe);
+  }
   if (!inp.sup0xccdb) {
     outputs.emplace_back(OutputSpec{{"TFDist"}, o2::header::gDataOriginFLP, o2::header::gDataDescriptionDISTSTF, 0xccdb});
   }
-
   options.emplace_back(ConfigParamSpec{"select-ctf-ids", VariantType::String, "", {"comma-separated list CTF IDs to inject (from cumulative counter of CTFs seen)"}});
+  options.emplace_back(ConfigParamSpec{"reverse-select-ctf-ids", VariantType::Bool, false, {"reverse order of to inject CTF IDs"}});
   options.emplace_back(ConfigParamSpec{"impose-run-start-timstamp", VariantType::Int64, 0L, {"impose run start time stamp (ms), ignored if 0"}});
   options.emplace_back(ConfigParamSpec{"local-tf-counter", VariantType::Bool, false, {"reassign header.tfCounter from local TF counter"}});
+  options.emplace_back(ConfigParamSpec{"fetch-failure-threshold", VariantType::Float, 0.f, {"Fail if too many failures( >0: fraction, <0: abs number, 0: no threshold)"}});
+  options.emplace_back(ConfigParamSpec{"limit-tf-before-reading", VariantType::Bool, false, {"Check TF limiting before reading new TF, otherwhise before injecting it"}});
+  options.emplace_back(ConfigParamSpec{"max-tf", VariantType::Int, -1, {"max CTFs to process (<= 0 : infinite)"}});
+  options.emplace_back(ConfigParamSpec{"max-tf-per-file", VariantType::Int, -1, {"max TFs to process per ctf file (<= 0 : infinite)"}});
+
   if (!inp.metricChannel.empty()) {
     options.emplace_back(ConfigParamSpec{"channel-config", VariantType::String, inp.metricChannel, {"Out-of-band channel config for TF throttling"}});
   }
 
   return DataProcessorSpec{
     "ctf-reader",
-    Inputs{},
+    inputs,
     outputs,
     AlgorithmSpec{adaptFromTask(inp)},
     options};
diff --git a/Detectors/CTF/workflow/src/CTFWriterSpec.cxx b/Detectors/CTF/workflow/src/CTFWriterSpec.cxx
index c5dabb85d3b04..ba4542969a712 100644
--- a/Detectors/CTF/workflow/src/CTFWriterSpec.cxx
+++ b/Detectors/CTF/workflow/src/CTFWriterSpec.cxx
@@ -44,7 +44,10 @@
 #include "DataFormatsCPV/CTF.h"
 #include "DataFormatsZDC/CTF.h"
 #include "DataFormatsCTP/CTF.h"
-#include "rANS/rans.h"
+
+#include "rANS/histogram.h"
+#include "rANS/compat.h"
+
 #include 
 #include 
 #include 
@@ -59,6 +62,7 @@
 #include 
 #include 
 #include 
+#include 
 
 using namespace o2::framework;
 
@@ -88,13 +92,13 @@ size_t appendToTree(TTree& tree, const std::string brname, T& ptr)
 }
 
 using DetID = o2::detectors::DetID;
-using FTrans = o2::rans::FrequencyTable;
+using FTrans = o2::rans::DenseHistogram;
 
 class CTFWriterSpec : public o2::framework::Task
 {
  public:
   CTFWriterSpec() = delete;
-  CTFWriterSpec(DetID::mask_t dm, uint64_t r, const std::string& outType, int verbosity, int reportInterval);
+  CTFWriterSpec(DetID::mask_t dm, const std::string& outType, int verbosity, int reportInterval);
   ~CTFWriterSpec() final { finalize(); }
   void init(o2::framework::InitContext& ic) final;
   void run(o2::framework::ProcessingContext& pc) final;
@@ -124,23 +128,30 @@ class CTFWriterSpec : public o2::framework::Task
   bool mCreateRunEnvDir = true;
   bool mStoreMetaFile = false;
   bool mRejectCurrentTF = false;
+  bool mFallBackDirUsed = false;
+  bool mFallBackDirProvided = false;
   int mReportInterval = -1;
   int mVerbosity = 0;
   int mSaveDictAfter = 0;          // if positive and mWriteCTF==true, save dictionary after each mSaveDictAfter TFs processed
   uint32_t mPrevDictTimeStamp = 0; // timestamp of the previously stored dictionary
   uint32_t mDictTimeStamp = 0;     // timestamp of the currently stored dictionary
-  size_t mMinSize = 0;               // if > 0, accumulate CTFs in the same tree until the total size exceeds this minimum
-  size_t mMaxSize = 0;               // if > MinSize, and accumulated size will exceed this value, stop accumulation (even if mMinSize is not reached)
-  size_t mChkSize = 0;               // if > 0 and fallback storage provided, reserve this size per CTF file in production on primary storage
-  size_t mAccCTFSize = 0;            // so far accumulated size (if any)
-  size_t mCurrCTFSize = 0;           // size of currently processed CTF
-  size_t mNCTF = 0;                  // total number of CTFs written
-  size_t mNCTFPrevDict = 0;          // total number of CTFs used for previous dictionary version
-  size_t mNAccCTF = 0;               // total number of CTFs accumulated in the current file
-  size_t mCTFAutoSave = 0;           // if > 0, autosave after so many TFs
-  size_t mNCTFFiles = 0;             // total number of CTF files written
-  int mMaxCTFPerFile = 0;            // max CTFs per files to store
-  int mRejRate = 0;                  // CTF rejection rule (>0: percentage to reject randomly, <0: reject if timeslice%|value|!=0)
+  size_t mMinSize = 0;             // if > 0, accumulate CTFs in the same tree until the total size exceeds this minimum
+  size_t mMaxSize = 0;             // if > MinSize, and accumulated size will exceed this value, stop accumulation (even if mMinSize is not reached)
+  size_t mChkSize = 0;             // if > 0 and fallback storage provided, reserve this size per CTF file in production on primary storage
+  size_t mAccCTFSize = 0;          // so far accumulated size (if any)
+  size_t mCurrCTFSize = 0;         // size of currently processed CTF
+  size_t mNCTF = 0;                // total number of CTFs written
+  size_t mNCTFPrevDict = 0;        // total number of CTFs used for previous dictionary version
+  size_t mNAccCTF = 0;             // total number of CTFs accumulated in the current file
+  int mWaitDiskFull = 0;           // if mCheckDiskFull triggers, pause for this amount of ms before new attempt
+  int mWaitDiskFullMax = -1;       // produce fatal mCheckDiskFull block the workflow for more than this time (in ms)
+  float mCheckDiskFull = 0.;       // wait for if available abs. disk space is < mCheckDiskFull (if >0) or if its fraction is < -mCheckDiskFull (if <0)
+  long mCTFAutoSave = 0;           // if > 0, autosave after so many TFs
+  size_t mNCTFFiles = 0;           // total number of CTF files written
+  int mMaxCTFPerFile = 0;          // max CTFs per files to store
+  int mRejRate = 0;                // CTF rejection rule (>0: percentage to reject randomly, <0: reject if timeslice%|value|!=0)
+  int mCTFFileCompression = 0;     // CTF file compression level (if >= 0)
+  bool mFillMD5 = false;
   std::vector mTFOrbits{}; // 1st orbits of TF accumulated in current file
   o2::framework::DataTakingContext mDataTakingContext{};
   o2::framework::TimingInfo mTimingInfo{};
@@ -179,7 +190,7 @@ class CTFWriterSpec : public o2::framework::Task
 const std::string CTFWriterSpec::TMPFileEnding{".part"};
 
 //___________________________________________________________________
-CTFWriterSpec::CTFWriterSpec(DetID::mask_t dm, uint64_t r, const std::string& outType, int verbosity, int reportInterval)
+CTFWriterSpec::CTFWriterSpec(DetID::mask_t dm, const std::string& outType, int verbosity, int reportInterval)
   : mDets(dm), mOutputType(outType), mReportInterval(reportInterval), mVerbosity(verbosity)
 {
   std::for_each(mIsSaturatedFrequencyTable.begin(), mIsSaturatedFrequencyTable.end(), [](auto& bitset) { bitset.reset(); });
@@ -209,17 +220,26 @@ void CTFWriterSpec::init(InitContext& ic)
   }
 
   mSaveDictAfter = ic.options().get("save-dict-after");
-  mCTFAutoSave = ic.options().get("save-ctf-after");
-  mDictDir = o2::utils::Str::rectifyDirectory(ic.options().get("ctf-dict-dir"));
-  mCTFDir = o2::utils::Str::rectifyDirectory(ic.options().get("output-dir"));
-  mCTFDirFallBack = ic.options().get("output-dir-alt");
-  if (mCTFDirFallBack != "/dev/null") {
-    mCTFDirFallBack = o2::utils::Str::rectifyDirectory(mCTFDirFallBack);
-  }
+  mCTFAutoSave = ic.options().get("save-ctf-after");
+  mCTFFileCompression = ic.options().get("ctf-file-compression");
   mCTFMetaFileDir = ic.options().get("meta-output-dir");
   if (mCTFMetaFileDir != "/dev/null") {
     mCTFMetaFileDir = o2::utils::Str::rectifyDirectory(mCTFMetaFileDir);
     mStoreMetaFile = true;
+    mFillMD5 = ic.options().get("md5-for-meta");
+  }
+  mDictDir = o2::utils::Str::rectifyDirectory(ic.options().get("ctf-dict-dir"));
+  mCTFDir = ic.options().get("output-dir");
+  if (mCTFDir != "/dev/null") {
+    mCTFDir = o2::utils::Str::rectifyDirectory(mCTFDir);
+  } else {
+    mWriteCTF = false;
+    mStoreMetaFile = false;
+  }
+  mCTFDirFallBack = ic.options().get("output-dir-alt");
+  if (mCTFDirFallBack != "/dev/null") {
+    mCTFDirFallBack = o2::utils::Str::rectifyDirectory(mCTFDirFallBack);
+    mFallBackDirProvided = true;
   }
   mCreateRunEnvDir = !ic.options().get("ignore-partition-run-dir");
   mMinSize = ic.options().get("min-file-size");
@@ -240,6 +260,11 @@ void CTFWriterSpec::init(InitContext& ic)
       }
     }
   }
+
+  mCheckDiskFull = ic.options().get("require-free-disk");
+  mWaitDiskFull = 1000 * ic.options().get("wait-for-free-disk");
+  mWaitDiskFullMax = 1000 * ic.options().get("max-wait-for-free-disk");
+
   mChkSize = std::max(size_t(mMinSize * 1.1), mMaxSize);
   o2::utils::createDirectoriesIfAbsent(LOCKFileDir);
 
@@ -260,15 +285,13 @@ void CTFWriterSpec::init(InitContext& ic)
 //___________________________________________________________________
 void CTFWriterSpec::updateTimeDependentParams(ProcessingContext& pc)
 {
-  static bool initOnceDone = false;
   namespace GRPECS = o2::parameters::GRPECS;
-  if (!initOnceDone) {
-    initOnceDone = true;
+  mTimingInfo = pc.services().get();
+  if (mTimingInfo.globalRunNumberChanged) {
     mDataTakingContext = pc.services().get();
     // determine the output type for the CTF metadata
     mMetaDataType = GRPECS::getRawDataPersistencyMode(mDataTakingContext.runType, mDataTakingContext.forcedRaw);
   }
-  mTimingInfo = pc.services().get();
 }
 
 //___________________________________________________________________
@@ -276,54 +299,74 @@ void CTFWriterSpec::updateTimeDependentParams(ProcessingContext& pc)
 template 
 size_t CTFWriterSpec::processDet(o2::framework::ProcessingContext& pc, DetID det, CTFHeader& header, TTree* tree)
 {
+  static bool warnedEmpty = false;
   size_t sz = 0;
   if (!isPresent(det) || !pc.inputs().isValid(det.getName())) {
     mSizeReport += fmt::format(" {}:N/A", det.getName());
     return sz;
   }
   auto ctfBuffer = pc.inputs().get>(det.getName());
-  const auto ctfImage = C::getImage(ctfBuffer.data());
-  ctfImage.print(o2::utils::Str::concat_string(det.getName(), ": "), mVerbosity);
-  if (mWriteCTF && !mRejectCurrentTF) {
-    sz = ctfImage.appendToTree(*tree, det.getName());
-    header.detectors.set(det);
-  } else {
-    sz = ctfBuffer.size();
-  }
-  if (mCreateDict) {
-    if (!mFreqsAccumulation[det].size()) {
-      mFreqsAccumulation[det].resize(C::getNBlocks());
-      mFreqsMetaData[det].resize(C::getNBlocks());
+  const o2::ctf::BufferType* bdata = ctfBuffer.data();
+  if (bdata) {
+    if (warnedEmpty) {
+      throw std::runtime_error(fmt::format("Non-empty input was seen at {}-th TF after empty one for {}, this will lead to misalignment of detectors in CTF", mNCTF, det.getName()));
     }
-    if (!mHeaders[det]) { // store 1st header
-      mHeaders[det] = ctfImage.cloneHeader();
-      auto& hb = *static_cast(mHeaders[det].get());
-      hb.det = det;
+    const auto ctfImage = C::getImage(bdata);
+    ctfImage.print(o2::utils::Str::concat_string(det.getName(), ": "), mVerbosity);
+    if (mWriteCTF && !mRejectCurrentTF) {
+      sz = ctfImage.appendToTree(*tree, det.getName());
+      header.detectors.set(det);
+    } else {
+      sz = ctfBuffer.size();
     }
-    for (int ib = 0; ib < C::getNBlocks(); ib++) {
-      if (!mIsSaturatedFrequencyTable[det][ib]) {
-        const auto& bl = ctfImage.getBlock(ib);
-        if (bl.getNDict()) {
-          auto freq = mFreqsAccumulation[det][ib];
-          auto& mdSave = mFreqsMetaData[det][ib];
-          const auto& md = ctfImage.getMetadata(ib);
-          if ([&, this]() {
-                try {
-                  freq.addFrequencies(bl.getDict(), bl.getDict() + bl.getNDict(), md.min);
-                } catch (const std::overflow_error& e) {
-                  LOGP(warning, "unable to frequency table for {}, block {} due to overflow", det.getName(), ib);
-                  mIsSaturatedFrequencyTable[det][ib] = true;
-                  return false;
-                }
-                return true;
-              }()) {
-            auto newProbBits = static_cast(o2::rans::computeRenormingPrecision(freq));
-            mdSave = o2::ctf::Metadata{0, 0, md.messageWordSize, md.coderType, md.streamSize, newProbBits, md.opt, freq.getMinSymbol(), freq.getMaxSymbol(), static_cast(freq.size()), 0, 0};
-            mFreqsAccumulation[det][ib] = std::move(freq);
+    if (mCreateDict) {
+      if (mFreqsAccumulation[det].empty()) {
+        mFreqsAccumulation[det].resize(C::getNBlocks());
+        mFreqsMetaData[det].resize(C::getNBlocks());
+      }
+      if (!mHeaders[det]) { // store 1st header
+        mHeaders[det] = ctfImage.cloneHeader();
+        auto& hb = *static_cast(mHeaders[det].get());
+        hb.det = det;
+      }
+      for (int ib = 0; ib < C::getNBlocks(); ib++) {
+        if (!mIsSaturatedFrequencyTable[det][ib]) {
+          const auto& bl = ctfImage.getBlock(ib);
+          if (bl.getNDict()) {
+            auto freq = mFreqsAccumulation[det][ib];
+            auto& mdSave = mFreqsMetaData[det][ib];
+            const auto& md = ctfImage.getMetadata(ib);
+            if ([&, this]() {
+                  try {
+                    freq.addFrequencies(bl.getDict(), bl.getDict() + bl.getNDict(), md.min);
+                  } catch (const std::overflow_error& e) {
+                    LOGP(warning, "unable to add frequency table for {}, block {} due to overflow", det.getName(), ib);
+                    mIsSaturatedFrequencyTable[det][ib] = true;
+                    return false;
+                  }
+                  return true;
+                }()) {
+              auto newProbBits = static_cast(o2::rans::compat::computeRenormingPrecision(countNUsedAlphabetSymbols(freq)));
+              auto histogramView = o2::rans::trim(o2::rans::makeHistogramView(freq));
+              mdSave = ctf::detail::makeMetadataRansDict(newProbBits,
+                                                         static_cast(histogramView.getMin()),
+                                                         static_cast(histogramView.getMax()),
+                                                         static_cast(histogramView.size()),
+                                                         md.opt);
+              mFreqsAccumulation[det][ib] = std::move(freq);
+            }
           }
         }
       }
     }
+  } else {
+    if (!warnedEmpty) {
+      if (mNCTF) {
+        throw std::runtime_error(fmt::format("Empty input was seen at {}-th TF after non-empty one for {}, this will lead to misalignment of detectors in CTF", mNCTF, det.getName()));
+      }
+      LOGP(important, "Empty CTF provided for {}, skipping and will not report anymore", det.getName());
+      warnedEmpty = true;
+    }
   }
   mSizeReport += fmt::format(" {}:{}", det.getName(), fmt::group_digits(sz));
   return sz;
@@ -386,6 +429,9 @@ size_t CTFWriterSpec::estimateCTFSize(ProcessingContext& pc)
 void CTFWriterSpec::run(ProcessingContext& pc)
 {
   const std::string NAStr = "NA";
+  if (pc.services().get().globalRunNumberChanged) {
+    mTimer.Reset();
+  }
   auto cput = mTimer.CpuTime();
   mTimer.Start(false);
   updateTimeDependentParams(pc);
@@ -393,35 +439,72 @@ void CTFWriterSpec::run(ProcessingContext& pc)
   mCurrCTFSize = estimateCTFSize(pc);
   if (mWriteCTF && !mRejectCurrentTF) {
     prepareTFTreeAndFile();
+
+    int totalWait = 0, nwaitCycles = 0;
+    while ((mFallBackDirUsed || !mFallBackDirProvided) && mCheckDiskFull) { // we are on the physical disk and not on the RAM disk
+      constexpr size_t MB = 1024 * 1024;
+      constexpr int showFirstN = 10, prsecaleWarnings = 50;
+      try {
+        const auto si = std::filesystem::space(mCTFFileOut->GetName());
+        std::string wmsg{};
+        if (mCheckDiskFull > 0.f && si.available < mCheckDiskFull) {
+          nwaitCycles++;
+          wmsg = fmt::format("Disk has {} MB available while at least {} MB is requested, wait for {} ms (on top of {} ms)", si.available / MB, size_t(mCheckDiskFull) / MB, mWaitDiskFull, totalWait);
+        } else if (mCheckDiskFull < 0.f && float(si.available) / si.capacity < -mCheckDiskFull) { // relative margin requested
+          nwaitCycles++;
+          wmsg = fmt::format("Disk has {:.3f}% available while at least {:.3f}% is requested, wait for {} ms (on top of {} ms)", si.capacity ? float(si.available) / si.capacity * 100.f : 0., -mCheckDiskFull, mWaitDiskFull, totalWait);
+        } else {
+          nwaitCycles = 0;
+        }
+        if (nwaitCycles) {
+          if (mWaitDiskFullMax > 0 && totalWait > mWaitDiskFullMax) {
+            closeTFTreeAndFile(); // try to save whatever we have
+            LOGP(fatal, "Disk has {} MB available out of {} MB after waiting for {} ms", si.available / MB, si.capacity / MB, mWaitDiskFullMax);
+          }
+          if (nwaitCycles < showFirstN + 1 || (prsecaleWarnings && (nwaitCycles % prsecaleWarnings) == 0)) {
+            LOG(alarm) << wmsg;
+          }
+          pc.services().get().waitFor((unsigned int)(mWaitDiskFull));
+          totalWait += mWaitDiskFull;
+          continue;
+        }
+      } catch (std::exception const& e) {
+        LOG(fatal) << "unable to query disk space info for path " << mCurrentCTFFileNameFull << ", reason: " << e.what();
+      }
+      break;
+    }
   }
   // create header
   CTFHeader header{mTimingInfo.runNumber, mTimingInfo.creation, mTimingInfo.firstTForbit, mTimingInfo.tfCounter};
   size_t szCTF = 0;
   mSizeReport = "";
-  szCTF += processDet(pc, DetID::ITS, header, mCTFTreeOut.get());
-  szCTF += processDet(pc, DetID::TPC, header, mCTFTreeOut.get());
-  szCTF += processDet(pc, DetID::TRD, header, mCTFTreeOut.get());
-  szCTF += processDet(pc, DetID::TOF, header, mCTFTreeOut.get());
-  szCTF += processDet(pc, DetID::PHS, header, mCTFTreeOut.get());
-  szCTF += processDet(pc, DetID::CPV, header, mCTFTreeOut.get());
-  szCTF += processDet(pc, DetID::EMC, header, mCTFTreeOut.get());
-  szCTF += processDet(pc, DetID::HMP, header, mCTFTreeOut.get());
-  szCTF += processDet(pc, DetID::MFT, header, mCTFTreeOut.get());
-  szCTF += processDet(pc, DetID::MCH, header, mCTFTreeOut.get());
-  szCTF += processDet(pc, DetID::MID, header, mCTFTreeOut.get());
-  szCTF += processDet(pc, DetID::ZDC, header, mCTFTreeOut.get());
-  szCTF += processDet(pc, DetID::FT0, header, mCTFTreeOut.get());
-  szCTF += processDet(pc, DetID::FV0, header, mCTFTreeOut.get());
-  szCTF += processDet(pc, DetID::FDD, header, mCTFTreeOut.get());
-  szCTF += processDet(pc, DetID::CTP, header, mCTFTreeOut.get());
+  std::array szCTFperDet{0}; // DetID::TST is between FDD and CTP and remains empty
+  szCTFperDet[DetID::ITS] = processDet(pc, DetID::ITS, header, mCTFTreeOut.get());
+  szCTFperDet[DetID::TPC] = processDet(pc, DetID::TPC, header, mCTFTreeOut.get());
+  szCTFperDet[DetID::TRD] = processDet(pc, DetID::TRD, header, mCTFTreeOut.get());
+  szCTFperDet[DetID::TOF] = processDet(pc, DetID::TOF, header, mCTFTreeOut.get());
+  szCTFperDet[DetID::PHS] = processDet(pc, DetID::PHS, header, mCTFTreeOut.get());
+  szCTFperDet[DetID::CPV] = processDet(pc, DetID::CPV, header, mCTFTreeOut.get());
+  szCTFperDet[DetID::EMC] = processDet(pc, DetID::EMC, header, mCTFTreeOut.get());
+  szCTFperDet[DetID::HMP] = processDet(pc, DetID::HMP, header, mCTFTreeOut.get());
+  szCTFperDet[DetID::MFT] = processDet(pc, DetID::MFT, header, mCTFTreeOut.get());
+  szCTFperDet[DetID::MCH] = processDet(pc, DetID::MCH, header, mCTFTreeOut.get());
+  szCTFperDet[DetID::MID] = processDet(pc, DetID::MID, header, mCTFTreeOut.get());
+  szCTFperDet[DetID::ZDC] = processDet(pc, DetID::ZDC, header, mCTFTreeOut.get());
+  szCTFperDet[DetID::FT0] = processDet(pc, DetID::FT0, header, mCTFTreeOut.get());
+  szCTFperDet[DetID::FV0] = processDet(pc, DetID::FV0, header, mCTFTreeOut.get());
+  szCTFperDet[DetID::FDD] = processDet(pc, DetID::FDD, header, mCTFTreeOut.get());
+  szCTFperDet[DetID::CTP] = processDet(pc, DetID::CTP, header, mCTFTreeOut.get());
+  szCTF = std::accumulate(szCTFperDet.begin(), szCTFperDet.end(), 0);
   if (mReportInterval > 0 && (mTimingInfo.tfCounter % mReportInterval) == 0) {
-    LOGP(important, "CTF {} size report:{}", mTimingInfo.tfCounter, mSizeReport);
+    LOGP(important, "CTF {} size report:{} - Total:{}", mTimingInfo.tfCounter, mSizeReport, fmt::group_digits(szCTF));
   }
 
   mTimer.Stop();
 
   if (mWriteCTF && !mRejectCurrentTF) {
     szCTF += appendToTree(*mCTFTreeOut.get(), "CTFHeader", header);
+    size_t prevSizeMB = mAccCTFSize / (1 << 20);
     mAccCTFSize += szCTF;
     mCTFTreeOut->SetEntries(++mNAccCTF);
     mTFOrbits.push_back(mTimingInfo.firstTForbit);
@@ -439,7 +522,7 @@ void CTFWriterSpec::run(ProcessingContext& pc)
 
     if (mAccCTFSize >= mMinSize || (mMaxCTFPerFile > 0 && mNAccCTF >= mMaxCTFPerFile)) {
       closeTFTreeAndFile();
-    } else if (mCTFAutoSave > 0 && mNAccCTF % mCTFAutoSave == 0) {
+    } else if ((mCTFAutoSave > 0 && mNAccCTF % mCTFAutoSave == 0) || (mCTFAutoSave < 0 && int(prevSizeMB / (-mCTFAutoSave)) != size_t(mAccCTFSize / (1 << 20)) / (-mCTFAutoSave))) {
       mCTFTreeOut->AutoSave("override");
     }
   } else {
@@ -450,6 +533,9 @@ void CTFWriterSpec::run(ProcessingContext& pc)
   if (mCreateDict && mSaveDictAfter > 0 && (mNCTF % mSaveDictAfter) == 0) {
     storeDictionaries();
   }
+  int dummy = 0;
+  pc.outputs().snapshot({"ctfdone", 0}, dummy);
+  pc.outputs().snapshot(Output{"CTF", "SIZES", 0}, szCTFperDet);
 }
 
 //___________________________________________________________________
@@ -467,6 +553,8 @@ void CTFWriterSpec::finalize()
   LOGF(info, "CTF writing total timing: Cpu: %.3e Real: %.3e s in %d slots",
        mTimer.CpuTime(), mTimer.RealTime(), mTimer.Counter() - 1);
   mFinalized = true;
+  mNCTF = 0;
+  mNCTFFiles = 0;
 }
 
 //___________________________________________________________________
@@ -488,14 +576,16 @@ void CTFWriterSpec::prepareTFTreeAndFile()
   }
   if (needToOpen) {
     closeTFTreeAndFile();
+    mFallBackDirUsed = false;
     auto ctfDir = mCTFDir.empty() ? o2::utils::Str::rectifyDirectory("./") : mCTFDir;
-    if (mChkSize > 0 && (mCTFDirFallBack != "/dev/null")) {
+    if (mChkSize > 0 && mFallBackDirProvided) {
       createLockFile(0);
       auto sz = getAvailableDiskSpace(ctfDir, 0); // check main storage
       if (sz < mChkSize) {
         removeLockFile();
         LOG(warning) << "Primary CTF output device has available size " << sz << " while " << mChkSize << " is requested: will write on secondary one";
         ctfDir = mCTFDirFallBack;
+        mFallBackDirUsed = true;
       }
     }
     if (mCreateRunEnvDir && !mDataTakingContext.envId.empty() && (mDataTakingContext.envId != o2::framework::DataTakingContext::UNKNOWN)) {
@@ -508,6 +598,9 @@ void CTFWriterSpec::prepareTFTreeAndFile()
     mCurrentCTFFileName = o2::base::NameConf::getCTFFileName(mTimingInfo.runNumber, mTimingInfo.firstTForbit, mTimingInfo.tfCounter, mHostName);
     mCurrentCTFFileNameFull = fmt::format("{}{}", ctfDir, mCurrentCTFFileName);
     mCTFFileOut.reset(TFile::Open(fmt::format("{}{}", mCurrentCTFFileNameFull, TMPFileEnding).c_str(), "recreate")); // to prevent premature external usage, use temporary name
+    if (mCTFFileCompression >= 0) {
+      mCTFFileOut->SetCompressionLevel(mCTFFileCompression);
+    }
     mCTFTreeOut = std::make_unique(std::string(o2::base::NameConf::CTFTREENAME).c_str(), "O2 CTF tree");
 
     mNCTFFiles++;
@@ -524,16 +617,16 @@ void CTFWriterSpec::closeTFTreeAndFile()
       mCTFTreeOut.reset();
       mCTFFileOut->Close();
       mCTFFileOut.reset();
-      if (!TMPFileEnding.empty()) {
-        std::filesystem::rename(o2::utils::Str::concat_string(mCurrentCTFFileNameFull, TMPFileEnding), mCurrentCTFFileNameFull);
-      }
       // write CTF file metaFile data
+      auto actualFileName = TMPFileEnding.empty() ? mCurrentCTFFileNameFull : o2::utils::Str::concat_string(mCurrentCTFFileNameFull, TMPFileEnding);
       if (mStoreMetaFile) {
         o2::dataformats::FileMetaData ctfMetaData;
-        ctfMetaData.fillFileData(mCurrentCTFFileNameFull);
+        if (!ctfMetaData.fillFileData(actualFileName, mFillMD5, TMPFileEnding)) {
+          throw std::runtime_error("metadata file was requested but not created");
+        }
         ctfMetaData.setDataTakingContext(mDataTakingContext);
         ctfMetaData.type = mMetaDataType;
-        ctfMetaData.priority = "high";
+        ctfMetaData.priority = mFallBackDirUsed ? "low" : "high";
         ctfMetaData.tfOrbits.swap(mTFOrbits);
         auto metaFileNameTmp = fmt::format("{}{}.tmp", mCTFMetaFileDir, mCurrentCTFFileName);
         auto metaFileName = fmt::format("{}{}.done", mCTFMetaFileDir, mCurrentCTFFileName);
@@ -541,10 +634,15 @@ void CTFWriterSpec::closeTFTreeAndFile()
           std::ofstream metaFileOut(metaFileNameTmp);
           metaFileOut << ctfMetaData;
           metaFileOut.close();
+          if (!TMPFileEnding.empty()) {
+            std::filesystem::rename(actualFileName, mCurrentCTFFileNameFull);
+          }
           std::filesystem::rename(metaFileNameTmp, metaFileName);
         } catch (std::exception const& e) {
           LOG(error) << "Failed to store CTF meta data file " << metaFileName << ", reason: " << e.what();
         }
+      } else if (!TMPFileEnding.empty()) {
+        std::filesystem::rename(actualFileName, mCurrentCTFFileNameFull);
       }
     } catch (std::exception const& e) {
       LOG(error) << "Failed to finalize CTF file " << mCurrentCTFFileNameFull << ", reason: " << e.what();
@@ -690,7 +788,7 @@ size_t CTFWriterSpec::getAvailableDiskSpace(const std::string& path, int level)
 }
 
 //___________________________________________________________________
-DataProcessorSpec getCTFWriterSpec(DetID::mask_t dets, uint64_t run, const std::string& outType, int verbosity, int reportInterval)
+DataProcessorSpec getCTFWriterSpec(DetID::mask_t dets, const std::string& outType, int verbosity, int reportInterval)
 {
   std::vector inputs;
   LOG(debug) << "Detectors list:";
@@ -703,19 +801,25 @@ DataProcessorSpec getCTFWriterSpec(DetID::mask_t dets, uint64_t run, const std::
   return DataProcessorSpec{
     "ctf-writer",
     inputs,
-    Outputs{},
-    AlgorithmSpec{adaptFromTask(dets, run, outType, verbosity, reportInterval)}, // RS FIXME once global/local options clash is solved, --output-type will become device option
-    Options{                                                                                    //{"output-type", VariantType::String, "ctf", {"output types: ctf (per TF) or dict (create dictionaries) or both or none"}},
-            {"save-ctf-after", VariantType::Int, 0, {"if > 0, autosave CTF tree with multiple CTFs after every N CTFs"}},
+    Outputs{{OutputLabel{"ctfdone"}, "CTF", "DONE", 0, Lifetime::Timeframe},
+            {"CTF", "SIZES", 0, Lifetime::Timeframe}},
+    AlgorithmSpec{adaptFromTask(dets, outType, verbosity, reportInterval)}, // RS FIXME once global/local options clash is solved, --output-type will become device option
+    Options{                                                                               //{"output-type", VariantType::String, "ctf", {"output types: ctf (per TF) or dict (create dictionaries) or both or none"}},
+            {"save-ctf-after", VariantType::Int64, 0ll, {"autosave CTF tree with multiple CTFs after every N CTFs if >0 or every -N MBytes if < 0"}},
             {"save-dict-after", VariantType::Int, 0, {"if > 0, in dictionary generation mode save it dictionary after certain number of TFs processed"}},
             {"ctf-dict-dir", VariantType::String, "none", {"CTF dictionary directory, must exist"}},
             {"output-dir", VariantType::String, "none", {"CTF output directory, must exist"}},
             {"output-dir-alt", VariantType::String, "/dev/null", {"Alternative CTF output directory, must exist (if not /dev/null)"}},
             {"meta-output-dir", VariantType::String, "/dev/null", {"CTF metadata output directory, must exist (if not /dev/null)"}},
+            {"md5-for-meta", VariantType::Bool, false, {"fill CTF file MD5 sum in the metadata file"}},
             {"min-file-size", VariantType::Int64, 0l, {"accumulate CTFs until given file size reached"}},
             {"max-file-size", VariantType::Int64, 0l, {"if > 0, try to avoid exceeding given file size, also used for space check"}},
             {"max-ctf-per-file", VariantType::Int, 0, {"if > 0, avoid storing more than requested CTFs per file"}},
             {"ctf-rejection", VariantType::Int, 0, {">0: percentage to reject randomly, <0: reject if timeslice%|value|!=0"}},
+            {"ctf-file-compression", VariantType::Int, 0, {"if >= 0: impose CTF file compression level"}},
+            {"require-free-disk", VariantType::Float, 0.f, {"pause writing op. if available disk space is below this margin, in bytes if >0, as a fraction of total if <0"}},
+            {"wait-for-free-disk", VariantType::Float, 10.f, {"if paused due to the low disk space, recheck after this time (in s)"}},
+            {"max-wait-for-free-disk", VariantType::Float, 60.f, {"produce fatal if paused due to the low disk space for more than this amount in s."}},
             {"ignore-partition-run-dir", VariantType::Bool, false, {"Do not creare partition-run directory in output-dir"}}}};
 }
 
diff --git a/Detectors/CTF/workflow/src/ctf-reader-workflow.cxx b/Detectors/CTF/workflow/src/ctf-reader-workflow.cxx
index bd982a382cada..fc50c971c5d20 100644
--- a/Detectors/CTF/workflow/src/ctf-reader-workflow.cxx
+++ b/Detectors/CTF/workflow/src/ctf-reader-workflow.cxx
@@ -18,10 +18,10 @@
 #include "Framework/InputSpec.h"
 #include "CommonUtils/NameConf.h"
 #include "CTFWorkflow/CTFReaderSpec.h"
-#include "DataFormatsParameters/GRPObject.h"
 #include "DetectorsCommonDataFormats/DetID.h"
 #include "CommonUtils/ConfigurableParam.h"
 #include "Algorithm/RangeTokenizer.h"
+#include "DetectorsBase/DPLWorkflowUtils.h"
 
 // Specific detectors specs
 #include "ITSMFTWorkflow/EntropyDecoderSpec.h"
@@ -39,6 +39,9 @@
 #include "CPVWorkflow/EntropyDecoderSpec.h"
 #include "ZDCWorkflow/EntropyDecoderSpec.h"
 #include "CTPWorkflow/EntropyDecoderSpec.h"
+#ifdef WITH_OPENMP
+#include 
+#endif
 
 using namespace o2::framework;
 using DetID = o2::detectors::DetID;
@@ -49,11 +52,12 @@ void customize(std::vector& workflowOptions)
   // option allowing to set parameters
   std::vector options;
   options.push_back(ConfigParamSpec{"ctf-input", VariantType::String, "none", {"comma-separated list CTF input files"}});
+  options.push_back(ConfigParamSpec{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}});
   options.push_back(ConfigParamSpec{"onlyDet", VariantType::String, std::string{DetID::ALL}, {"comma-separated list of detectors to accept. Overrides skipDet"}});
   options.push_back(ConfigParamSpec{"skipDet", VariantType::String, std::string{DetID::NONE}, {"comma-separate list of detectors to skip"}});
-  options.push_back(ConfigParamSpec{"max-tf", VariantType::Int, -1, {"max CTFs to process (<= 0 : infinite)"}});
   options.push_back(ConfigParamSpec{"loop", VariantType::Int, 0, {"loop N times (infinite for N<0)"}});
   options.push_back(ConfigParamSpec{"delay", VariantType::Float, 0.f, {"delay in seconds between consecutive TFs sending"}});
+  options.push_back(ConfigParamSpec{"shuffle", VariantType::Bool, false, {"shuffle TF sending order (for debug)"}});
   options.push_back(ConfigParamSpec{"copy-cmd", VariantType::String, "alien_cp ?src file://?dst", {"copy command for remote files or no-copy to avoid copying"}}); // Use "XrdSecPROTOCOL=sss,unix xrdcp -N root://eosaliceo2.cern.ch/?src ?dst" for direct EOS access
   options.push_back(ConfigParamSpec{"ctf-file-regex", VariantType::String, ".*o2_ctf_run.+\\.root$", {"regex string to identify CTF files"}});
   options.push_back(ConfigParamSpec{"remote-regex", VariantType::String, "^(alien://|)/alice/data/.+", {"regex string to identify remote files"}}); // Use "^/eos/aliceo2/.+" for direct EOS access
@@ -63,12 +67,19 @@ void customize(std::vector& workflowOptions)
   options.push_back(ConfigParamSpec{"ctf-reader-verbosity", VariantType::Int, 0, {"verbosity level (0: summary per detector, 1: summary per block"}});
   options.push_back(ConfigParamSpec{"ctf-data-subspec", VariantType::Int, 0, {"subspec to use for decoded CTF messages (use non-0 if CTF writer will be attached downstream)"}});
   options.push_back(ConfigParamSpec{"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}});
+  options.push_back(ConfigParamSpec{"ir-frames-files", VariantType::String, "", {"If non empty, inject selected IRFrames from this file"}});
+  options.push_back(ConfigParamSpec{"run-time-span-file", VariantType::String, "", {"If non empty, inject selected IRFrames from this text file (run, min/max orbit or unix time)"}});
+  options.push_back(ConfigParamSpec{"skip-skimmed-out-tf", VariantType::Bool, false, {"Do not process TFs with empty IR-Frame coverage"}});
+  options.push_back(ConfigParamSpec{"invert-irframe-selection", VariantType::Bool, false, {"Select only frames mentioned in ir-frames-file (skip-skimmed-out-tf applied to TF not selected!)"}});
   //
   options.push_back(ConfigParamSpec{"its-digits", VariantType::Bool, false, {"convert ITS clusters to digits"}});
   options.push_back(ConfigParamSpec{"mft-digits", VariantType::Bool, false, {"convert MFT clusters to digits"}});
-
+  //
+  options.push_back(ConfigParamSpec{"emcal-decoded-subspec", VariantType::Int, 0, {"subspec to use for decoded EMCAL data"}});
+  //
   options.push_back(ConfigParamSpec{"timeframes-shm-limit", VariantType::String, "0", {"Minimum amount of SHM required in order to publish data"}});
   options.push_back(ConfigParamSpec{"metric-feedback-channel-format", VariantType::String, "name=metric-feedback,type=pull,method=connect,address=ipc://{}metric-feedback-{},transport=shmem,rateLogging=0", {"format for the metric-feedback channel for TF rate limiting"}});
+  options.push_back(ConfigParamSpec{"combine-devices", VariantType::Bool, false, {"combine multiple DPL devices (entropy decoders)"}});
   std::swap(workflowOptions, options);
 }
 
@@ -92,6 +103,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const& configcontext)
   ctfInput.detMask &= DetID::getMask(allowedDetectors);
   ctfInput.inpdata = configcontext.options().get("ctf-input");
   ctfInput.subspec = (unsigned int)configcontext.options().get("ctf-data-subspec");
+  ctfInput.decSSpecEMC = (unsigned int)configcontext.options().get("emcal-decoded-subspec");
   if (ctfInput.inpdata.empty() || ctfInput.inpdata == "none") {
     if (!configcontext.helpOnCommandLine()) {
       throw std::runtime_error("--ctf-input  is not provided");
@@ -107,75 +119,148 @@ WorkflowSpec defineDataProcessing(ConfigContext const& configcontext)
   if (ctfInput.delay_us < 0) {
     ctfInput.delay_us = 0;
   }
-  int n = configcontext.options().get("max-tf");
-  ctfInput.maxTFs = n > 0 ? n : 0x7fffffff;
 
   ctfInput.maxFileCache = std::max(1, configcontext.options().get("max-cached-files"));
 
+  ctfInput.shuffle = configcontext.options().get("shuffle");
   ctfInput.copyCmd = configcontext.options().get("copy-cmd");
   ctfInput.tffileRegex = configcontext.options().get("ctf-file-regex");
   ctfInput.remoteRegex = configcontext.options().get("remote-regex");
   ctfInput.allowMissingDetectors = configcontext.options().get("allow-missing-detectors");
   ctfInput.sup0xccdb = !configcontext.options().get("send-diststf-0xccdb");
   ctfInput.minSHM = std::stoul(configcontext.options().get("timeframes-shm-limit"));
+  ctfInput.fileIRFrames = configcontext.options().get("ir-frames-files");
+  ctfInput.fileRunTimeSpans = configcontext.options().get("run-time-span-file");
+  ctfInput.skipSkimmedOutTF = configcontext.options().get("skip-skimmed-out-tf");
+  ctfInput.invertIRFramesSelection = configcontext.options().get("invert-irframe-selection");
+  ctfInput.dictOpt = configcontext.options().get("ctf-dict");
   int verbosity = configcontext.options().get("ctf-reader-verbosity");
 
   int rateLimitingIPCID = std::stoi(configcontext.options().get("timeframes-rate-limit-ipcid"));
   std::string chanFmt = configcontext.options().get("metric-feedback-channel-format");
   if (rateLimitingIPCID > -1 && !chanFmt.empty()) {
-    ctfInput.metricChannel = fmt::format(chanFmt, o2::framework::ChannelSpecHelpers::defaultIPCFolder(), rateLimitingIPCID);
+    ctfInput.metricChannel = fmt::format(fmt::runtime(chanFmt), o2::framework::ChannelSpecHelpers::defaultIPCFolder(), rateLimitingIPCID);
+  }
+  if (!ctfInput.fileRunTimeSpans.empty()) {
+    ctfInput.skipSkimmedOutTF = true;
+  }
+  if (!ctfInput.fileIRFrames.empty() && !ctfInput.fileRunTimeSpans.empty()) {
+    LOGP(fatal, "One cannot provide --ir-frames-files and --run-time-span-file options simultaneously");
   }
 
   specs.push_back(o2::ctf::getCTFReaderSpec(ctfInput));
 
+  auto pipes = configcontext.options().get("pipeline");
+  std::unordered_map plines;
+  auto ptokens = o2::utils::Str::tokenize(pipes, ',');
+  for (auto& token : ptokens) {
+    auto split = token.find(":");
+    if (split == std::string::npos) {
+      throw std::runtime_error("bad pipeline definition. Syntax :");
+    }
+    auto key = token.substr(0, split);
+    token.erase(0, split + 1);
+    size_t error;
+    auto value = std::stoll(token, &error, 10);
+    if (token[error] != '\0') {
+      throw std::runtime_error("Bad pipeline definition. Expecting integer");
+    }
+    if (value > 1) {
+      plines[key] = value;
+    }
+  }
+
+  std::vector decSpecsV;
+
+  auto addSpecs = [&decSpecsV, &plines](DataProcessorSpec&& s) {
+    auto entry = plines.find(s.name);
+    size_t mult = (entry == plines.end() || entry->second < 2) ? 1 : entry->second;
+    if (mult > decSpecsV.size()) {
+      decSpecsV.resize(mult);
+    }
+    decSpecsV[mult - 1].push_back(s);
+  };
+
   // add decoders for all allowed detectors.
   if (ctfInput.detMask[DetID::ITS]) {
-    specs.push_back(o2::itsmft::getEntropyDecoderSpec(DetID::getDataOrigin(DetID::ITS), verbosity, configcontext.options().get("its-digits"), ctfInput.subspec));
+    addSpecs(o2::itsmft::getEntropyDecoderSpec(DetID::getDataOrigin(DetID::ITS), verbosity, configcontext.options().get("its-digits"), ctfInput.subspec, ctfInput.dictOpt));
   }
   if (ctfInput.detMask[DetID::MFT]) {
-    specs.push_back(o2::itsmft::getEntropyDecoderSpec(DetID::getDataOrigin(DetID::MFT), verbosity, configcontext.options().get("mft-digits"), ctfInput.subspec));
+    addSpecs(o2::itsmft::getEntropyDecoderSpec(DetID::getDataOrigin(DetID::MFT), verbosity, configcontext.options().get("mft-digits"), ctfInput.subspec, ctfInput.dictOpt));
   }
   if (ctfInput.detMask[DetID::TPC]) {
-    specs.push_back(o2::tpc::getEntropyDecoderSpec(verbosity, ctfInput.subspec));
+    addSpecs(o2::tpc::getEntropyDecoderSpec(verbosity, ctfInput.subspec, ctfInput.dictOpt));
   }
   if (ctfInput.detMask[DetID::TRD]) {
-    specs.push_back(o2::trd::getEntropyDecoderSpec(verbosity, ctfInput.subspec));
+    addSpecs(o2::trd::getEntropyDecoderSpec(verbosity, ctfInput.subspec, ctfInput.dictOpt));
   }
   if (ctfInput.detMask[DetID::TOF]) {
-    specs.push_back(o2::tof::getEntropyDecoderSpec(verbosity, ctfInput.subspec));
+    addSpecs(o2::tof::getEntropyDecoderSpec(verbosity, ctfInput.subspec, ctfInput.dictOpt));
   }
   if (ctfInput.detMask[DetID::FT0]) {
-    specs.push_back(o2::ft0::getEntropyDecoderSpec(verbosity, ctfInput.subspec));
+    addSpecs(o2::ft0::getEntropyDecoderSpec(verbosity, ctfInput.subspec, ctfInput.dictOpt));
   }
   if (ctfInput.detMask[DetID::FV0]) {
-    specs.push_back(o2::fv0::getEntropyDecoderSpec(verbosity, ctfInput.subspec));
+    addSpecs(o2::fv0::getEntropyDecoderSpec(verbosity, ctfInput.subspec, ctfInput.dictOpt));
   }
   if (ctfInput.detMask[DetID::FDD]) {
-    specs.push_back(o2::fdd::getEntropyDecoderSpec(verbosity, ctfInput.subspec));
+    addSpecs(o2::fdd::getEntropyDecoderSpec(verbosity, ctfInput.subspec, ctfInput.dictOpt));
   }
   if (ctfInput.detMask[DetID::MID]) {
-    specs.push_back(o2::mid::getEntropyDecoderSpec(verbosity, ctfInput.subspec));
+    addSpecs(o2::mid::getEntropyDecoderSpec(verbosity, ctfInput.subspec, ctfInput.dictOpt));
   }
   if (ctfInput.detMask[DetID::MCH]) {
-    specs.push_back(o2::mch::getEntropyDecoderSpec(verbosity, "mch-entropy-decoder", ctfInput.subspec));
+    addSpecs(o2::mch::getEntropyDecoderSpec(verbosity, "mch-entropy-decoder", ctfInput.subspec, ctfInput.dictOpt));
   }
   if (ctfInput.detMask[DetID::EMC]) {
-    specs.push_back(o2::emcal::getEntropyDecoderSpec(verbosity, ctfInput.subspec));
+    addSpecs(o2::emcal::getEntropyDecoderSpec(verbosity, ctfInput.subspec, ctfInput.decSSpecEMC, ctfInput.dictOpt));
   }
   if (ctfInput.detMask[DetID::PHS]) {
-    specs.push_back(o2::phos::getEntropyDecoderSpec(verbosity, ctfInput.subspec));
+    addSpecs(o2::phos::getEntropyDecoderSpec(verbosity, ctfInput.subspec, ctfInput.dictOpt));
   }
   if (ctfInput.detMask[DetID::CPV]) {
-    specs.push_back(o2::cpv::getEntropyDecoderSpec(verbosity, ctfInput.subspec));
+    addSpecs(o2::cpv::getEntropyDecoderSpec(verbosity, ctfInput.subspec, ctfInput.dictOpt));
   }
   if (ctfInput.detMask[DetID::ZDC]) {
-    specs.push_back(o2::zdc::getEntropyDecoderSpec(verbosity, ctfInput.subspec));
+    addSpecs(o2::zdc::getEntropyDecoderSpec(verbosity, ctfInput.subspec, ctfInput.dictOpt));
   }
   if (ctfInput.detMask[DetID::HMP]) {
-    specs.push_back(o2::hmpid::getEntropyDecoderSpec(verbosity, ctfInput.subspec));
+    addSpecs(o2::hmpid::getEntropyDecoderSpec(verbosity, ctfInput.subspec, ctfInput.dictOpt));
   }
   if (ctfInput.detMask[DetID::CTP]) {
-    specs.push_back(o2::ctp::getEntropyDecoderSpec(verbosity, ctfInput.subspec));
+    addSpecs(o2::ctp::getEntropyDecoderSpec(verbosity, ctfInput.subspec, ctfInput.dictOpt));
+  }
+
+  bool combine = configcontext.options().get("combine-devices");
+  if (!combine) {
+    for (auto& decSpecs : decSpecsV) {
+      for (auto& s : decSpecs) {
+        specs.push_back(s);
+      }
+    }
+  } else {
+    std::vector remaining;
+    if (decSpecsV.size() && decSpecsV[0].size()) {
+      specs.push_back(specCombiner("EntropyDecoders", decSpecsV[0], remaining)); // processing w/o pipelining
+    }
+    bool updatePipelines = false;
+    for (size_t i = 1; i < decSpecsV.size(); i++) { // add pipelined processes separately, consider combining them to separate groups (need to have modify argument of pipeline option)
+      if (decSpecsV[i].size() > 1) {
+        specs.push_back(specCombiner(fmt::format("EntropyDecodersP{}", i + 1), decSpecsV[i], remaining)); // processing pipelining multiplicity i+1
+        updatePipelines = true;
+        pipes += fmt::format(",EntropyDecodersP{}:{}", i + 1, i + 1);
+      } else {
+        for (auto& s : decSpecsV[i]) {
+          specs.push_back(s);
+        }
+      }
+    }
+    for (auto& s : remaining) {
+      specs.push_back(s);
+    }
+    if (updatePipelines) {
+      configcontext.options().override("pipeline", pipes);
+    }
   }
 
   return std::move(specs);
diff --git a/Detectors/CTF/workflow/src/ctf-writer-workflow.cxx b/Detectors/CTF/workflow/src/ctf-writer-workflow.cxx
index 2fdf76e08854d..2757192727521 100644
--- a/Detectors/CTF/workflow/src/ctf-writer-workflow.cxx
+++ b/Detectors/CTF/workflow/src/ctf-writer-workflow.cxx
@@ -18,7 +18,6 @@
 #include "Framework/InputSpec.h"
 #include "CommonUtils/NameConf.h"
 #include "CTFWorkflow/CTFWriterSpec.h"
-#include "DataFormatsParameters/GRPObject.h"
 #include "DetectorsCommonDataFormats/DetID.h"
 #include "CommonUtils/ConfigurableParam.h"
 
@@ -32,8 +31,6 @@ void customize(std::vector& workflowOptions)
   std::vector options;
   options.push_back(ConfigParamSpec{"onlyDet", VariantType::String, std::string{DetID::NONE}, {"comma separated list of detectors to accept. Overrides skipDet"}});
   options.push_back(ConfigParamSpec{"skipDet", VariantType::String, std::string{DetID::NONE}, {"comma separate list of detectors to skip"}});
-  options.push_back(ConfigParamSpec{"grpfile", VariantType::String, o2::base::NameConf::getGRPFileName(), {"name of the grp file"}});
-  options.push_back(ConfigParamSpec{"no-grp", VariantType::Bool, false, {"do not read GRP file"}});
   options.push_back(ConfigParamSpec{"output-type", VariantType::String, "ctf", {"output types: ctf (per TF) or dict (create dictionaries) or both or none"}});
   options.push_back(ConfigParamSpec{"ctf-writer-verbosity", VariantType::Int, 0, {"verbosity level (0: summary per detector, 1: summary per block"}});
   options.push_back(ConfigParamSpec{"report-data-size-interval", VariantType::Int, 200, {"report sizes per detector for every N-th timeframe"}});
@@ -54,34 +51,22 @@ WorkflowSpec defineDataProcessing(ConfigContext const& configcontext)
 {
   DetID::mask_t dets = 0;
   o2::conf::ConfigurableParam::updateFromString(configcontext.options().get("configKeyValues"));
-  long run = 0;
-  bool doCTF = true, doDict = false, dictPerDet = false;
-  size_t szMin = 0, szMax = 0;
   std::string outType{}; // RS FIXME once global/local options clash is solved, --output-type will become device option
   if (!configcontext.helpOnCommandLine()) {
-    bool noGRP = configcontext.options().get("no-grp");
-    auto onlyDet = configcontext.options().get("onlyDet");
-    if (!noGRP) {
-      std::unique_ptr grp(o2::parameters::GRPObject::loadFrom(configcontext.options().get("grpfile")));
-      dets = grp->getDetsReadOut(onlyDet, configcontext.options().get("skipDet"));
-      run = grp->getRun();
+    dets.set(); // by default read all
+    auto mskOnly = DetID::getMask(configcontext.options().get("onlyDet"));
+    auto mskSkip = DetID::getMask(configcontext.options().get("skipDet"));
+    if (mskOnly.any()) {
+      dets &= mskOnly;
     } else {
-      dets.set(); // by default read all
-      auto mskOnly = DetID::getMask(configcontext.options().get("onlyDet"));
-      auto mskSkip = DetID::getMask(configcontext.options().get("skipDet"));
-      if (mskOnly.any()) {
-        dets &= mskOnly;
-      } else {
-        dets ^= mskSkip;
-      }
-      run = 0;
+      dets ^= mskSkip;
     }
     if (dets.none()) {
       throw std::invalid_argument("Invalid workflow: no detectors found");
     }
     outType = configcontext.options().get("output-type");
   }
-  WorkflowSpec specs{o2::ctf::getCTFWriterSpec(dets, run, outType,
+  WorkflowSpec specs{o2::ctf::getCTFWriterSpec(dets, outType,
                                                configcontext.options().get("ctf-writer-verbosity"),
                                                configcontext.options().get("report-data-size-interval"))};
   return std::move(specs);
diff --git a/Detectors/CTP/macro/CMakeLists.txt b/Detectors/CTP/macro/CMakeLists.txt
index decffe8de9525..8608c1a8b7846 100644
--- a/Detectors/CTP/macro/CMakeLists.txt
+++ b/Detectors/CTP/macro/CMakeLists.txt
@@ -25,3 +25,63 @@ o2_add_test_root_macro(SaveInputsConfig.C
                        PUBLIC_LINK_LIBRARIES O2::DataFormatsCTP
                                              O2::CCDB
                        LABELS ctp)
+o2_add_test_root_macro(CheckCTPDigits.C
+                       PUBLIC_LINK_LIBRARIES O2::DataFormatsCTP
+                                             O2::CCDB
+                       LABELS ctp)
+o2_add_test_root_macro(CheckAOD2CTPDigits.C
+                       PUBLIC_LINK_LIBRARIES O2::DataFormatsCTP
+                                             O2::CCDB
+                       LABELS ctp)
+o2_add_test_root_macro(CheckAOD2CTPDigitsII.C
+                       PUBLIC_LINK_LIBRARIES O2::DataFormatsCTP
+                                             O2::CCDB
+                       LABELS ctp)
+o2_add_test_root_macro(TestConfig.C
+                       PUBLIC_LINK_LIBRARIES O2::DataFormatsCTP
+                                             O2::CCDB
+                       LABELS ctp)
+o2_add_test_root_macro(dumpCTPRO.C
+                       PUBLIC_LINK_LIBRARIES O2::DataFormatsCTP
+                                             O2::CCDB
+                       LABELS ctp)
+o2_add_test_root_macro(GetAndSave.C
+                       PUBLIC_LINK_LIBRARIES O2::DataFormatsCTP
+                                             O2::CCDB
+                       LABELS ctp)
+o2_add_test_root_macro(ReadCTPRunScalersFromFile.C
+                       PUBLIC_LINK_LIBRARIES O2::DataFormatsCTP
+                                             O2::CCDB
+                       LABELS ctp)
+o2_add_test_root_macro(GetScalersForRun.C
+                       PUBLIC_LINK_LIBRARIES O2::DataFormatsCTP
+                                             O2::CCDB
+                       LABELS ctp)
+o2_add_test_root_macro(PlotPbLumi.C
+                       PUBLIC_LINK_LIBRARIES O2::DataFormatsCTP
+                                             O2::CCDB
+                       LABELS ctp)
+o2_add_test_root_macro(PlotOrbit.C
+                       PUBLIC_LINK_LIBRARIES O2::DataFormatsCTP
+                                             O2::CCDB
+                       LABELS ctp)
+o2_add_test_root_macro(TestFetcher.C
+                       PUBLIC_LINK_LIBRARIES O2::DataFormatsCTP
+                                             O2::CCDB
+                       LABELS ctp)
+o2_add_test_root_macro(CreateBKForRun.C
+                       PUBLIC_LINK_LIBRARIES O2::DataFormatsCTP
+                                             O2::CCDB
+                       LABELS ctp)
+o2_add_test_root_macro(CheckCTPConfig.C
+                       PUBLIC_LINK_LIBRARIES O2::DataFormatsCTP
+                                             O2::CCDB
+                       LABELS ctp)
+o2_add_test_root_macro(GetRates.C
+                       PUBLIC_LINK_LIBRARIES O2::DataFormatsCTP
+                                             O2::CCDB
+                       LABELS ctp)
+o2_add_test_root_macro(TestGetRates.C
+                       PUBLIC_LINK_LIBRARIES O2::DataFormatsCTP
+                                             O2::CCDB
+                       LABELS ctp)
\ No newline at end of file
diff --git a/Detectors/CTP/macro/CheckAOD2CTPDigits.C b/Detectors/CTP/macro/CheckAOD2CTPDigits.C
new file mode 100644
index 0000000000000..df9406fec62c1
--- /dev/null
+++ b/Detectors/CTP/macro/CheckAOD2CTPDigits.C
@@ -0,0 +1,109 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file checkAOD2CTPDigits.C
+/// \brief create CTP config, test it and add to database
+/// \author Roman Lietava
+
+#if !defined(__CLING__) || defined(__ROOTCLING__)
+
+#include 
+#include "TFile.h"
+#include "TTree.h"
+#include 
+#include 
+#include 
+#include "TKey.h"
+#include "TTreeReader.h"
+#include "TTreeReaderValue.h"
+#endif
+// To produce CTP digits:
+// o2-raw-tf-reader-workflow -b --onlyDet CTP,FT0  --input-data list.txt  | o2-ctp-reco-workflow -b  --use-verbose-mode
+// To produce AOD:
+// WORKFLOW_PARAMETERS="AOD" WORKFLOW_DETECTORS="CTP,FT0,FV0" IGNORE_EXISTING_SHMFILES=1 $O2_ROOT/prodtests/full-system-test/run-workflow-on-inputlist.sh TF list.txt
+using namespace o2::ctp;
+int CheckAOD2CTPDigits(bool files = 0)
+{
+  if (files == 0) {
+    return 0;
+  }
+  int fRunNumber;
+  ULong64_t fGlobalBC;
+  ULong64_t fTriggerMask;
+  std::unique_ptr file(TFile::Open("AO2D.root"));
+  file->ls();
+  TIter keyList(file->GetListOfKeys());
+  TKey* key;
+  TTree* tree;
+  ;
+  std::map bc2classmask;
+  // Find aod trigger info
+  while ((key = (TKey*)keyList())) {
+    std::cout << key->GetName() << std::endl;
+    tree = (TTree*)file->Get(Form("%s/O2bc", key->GetName()));
+    if (tree != 0) {
+      std::cout << "found O2bc" << std::endl;
+      tree->SetBranchAddress("fRunNumber", &fRunNumber);
+      tree->SetBranchAddress("fGlobalBC", &fGlobalBC);
+      tree->SetBranchAddress("fTriggerMask", &fTriggerMask);
+      std::cout << "# of entries:" << tree->GetEntries() << std::endl;
+      int NN = tree->GetEntries();
+      int Nloop = NN;
+      for (int n{0}; n < Nloop; ++n) {
+        tree->GetEvent(n);
+        if (fTriggerMask) {
+          bc2classmask[fGlobalBC] = fTriggerMask;
+          // std::cout << std::dec <<  n << " Run:" << fRunNumber << " GBC:" << std::hex << fGlobalBC << " TM:0x" << std::hex << fTriggerMask << " count:" << bc2classmask.count(fGlobalBC) << std::endl;
+        }
+      }
+    } else {
+      return 1;
+    }
+  }
+  // Read CTP digits and check if every class mask in digits is in AOD
+  // CTP digits
+  TFile* fileDigits = TFile::Open("ctpdigits.root");
+  //
+  fileDigits->ls();
+  o2::ctp::CTPDigit* dig = new o2::ctp::CTPDigit;
+  //
+  // tree->Print();
+  TTreeReader reader("o2sim", fileDigits);
+  TTreeReaderValue> ctpdigs(reader, "CTPDigits");
+  // TTreeReaderArray ctpdigs(reader,"CTPDigits");
+  bool firstE = true;
+  //
+  std::bitset<48> tvxmask;
+  tvxmask.set(2);
+  int ncls = 0;
+  int nnotf = 0;
+  while (reader.Next()) {
+    if (ctpdigs.GetSetupStatus() < 0) {
+      std::cout << "Error:" << std::dec << ctpdigs.GetSetupStatus() << " for:" << ctpdigs.GetBranchName() << std::endl;
+      return 1;
+    }
+    // std::cout << "size:" << std::dec << ctpdigs.GetSize() << std::endl;
+    std::cout << "size:" << std::dec << ctpdigs->size() << std::endl;
+    for (auto const& dig : *ctpdigs) {
+      if (dig.CTPClassMask.count()) {
+        ULong64_t gbc = dig.intRecord.toLong();
+        // int del = 280+17;
+        if (bc2classmask.count(gbc)) {
+          // std::cout << std::hex << gbc << "aod clsmask:" << bc2classmask[gbc] << " " << dig.CTPClassMask.to_ullong() << " inps:" << dig.CTPInputMask.to_ullong() << std::endl;
+          // dig.printStream(std::cout);
+        } else {
+          std::cout << std::dec << dig.intRecord.orbit << " " << dig.intRecord.bc << " " << std::hex << dig.intRecord.toLong() << " not found " << bc2classmask.count(gbc) << std::endl;
+        }
+      }
+    }
+  }
+  return 0;
+}
diff --git a/Detectors/CTP/macro/CheckAOD2CTPDigitsII.C b/Detectors/CTP/macro/CheckAOD2CTPDigitsII.C
new file mode 100644
index 0000000000000..14e60194bdd82
--- /dev/null
+++ b/Detectors/CTP/macro/CheckAOD2CTPDigitsII.C
@@ -0,0 +1,183 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file checkAOD2CTPDigits.C
+/// \brief create CTP config, test it and add to database
+/// \author Roman Lietava
+
+#if !defined(__CLING__) || defined(__ROOTCLING__)
+
+#include 
+#include "TFile.h"
+#include "TTree.h"
+#include 
+#include 
+#include 
+#include "TKey.h"
+#include "TTreeReader.h"
+#include "TTreeReaderValue.h"
+#include "CommonDataFormat/InteractionRecord.h"
+#endif
+// To produce CTP digits:
+// o2-raw-tf-reader-workflow -b --onlyDet CTP,FT0  --input-data list.txt  | o2-ctp-reco-workflow -b  --use-verbose-mode
+// To produce AOD:
+// WORKFLOW_PARAMETERS="AOD" WORKFLOW_DETECTORS="CTP,FT0,FV0" IGNORE_EXISTING_SHMFILES=1 $O2_ROOT/prodtests/full-system-test/run-workflow-on-inputlist.sh TF list.txt
+using namespace o2::ctp;
+int CheckAOD2CTPDigits(bool files = 1)
+{
+  if (files == 0) {
+    return 0;
+  }
+  int fRunNumber;
+  ULong64_t fGlobalBC;
+  ULong64_t fTriggerMask;
+  ULong64_t fInputMask;
+  std::unique_ptr file(TFile::Open("AO2D.root"));
+  if (file == nullptr) {
+    std::cout << "Can not open file AO2D.root" << std::endl;
+    return 1;
+  }
+  file->ls();
+  TIter keyList(file->GetListOfKeys());
+  TKey* key;
+  TTree* tree;
+  std::map bc2classmask;
+  std::map bc2inputmask;
+  // Find aod trigger info
+  int i = 0;
+  while ((key = (TKey*)keyList())) {
+    std::cout << "loop:" << i << " " << key->GetName() << std::endl;
+    i++;
+    std::string name = key->GetName();
+    if (name.find("metaData") != std::string::npos) {
+      std::cout << "Skipping:" << name << std::endl;
+      continue;
+    }
+    tree = (TTree*)file->Get(Form("%s/O2bc_001", key->GetName()));
+    if (tree != 0) {
+      std::cout << "found O2bc" << std::endl;
+      tree->SetBranchAddress("fRunNumber", &fRunNumber);
+      tree->SetBranchAddress("fGlobalBC", &fGlobalBC);
+      tree->SetBranchAddress("fTriggerMask", &fTriggerMask);
+      tree->SetBranchAddress("fInputMask", &fInputMask);
+      std::cout << "# of entries:" << tree->GetEntries() << std::endl;
+      int NN = tree->GetEntries();
+      int Nloop = NN;
+      for (int n{0}; n < Nloop; ++n) {
+        tree->GetEvent(n);
+        if (fTriggerMask) {
+          bc2classmask[fGlobalBC] = fTriggerMask;
+          // std::cout << std::dec <<  n << " Run:" << fRunNumber << " GBC:" << std::hex << fGlobalBC << " TM:0x" << std::hex << fTriggerMask << " count:" << bc2classmask.count(fGlobalBC) << std::endl;
+        }
+        if (fInputMask) {
+          bc2inputmask[fGlobalBC] = fInputMask;
+        }
+        if (1) {
+          if (fInputMask || fTriggerMask) {
+            auto ir = o2::InteractionRecord::long2IR(fGlobalBC);
+            // auto bcc = ir.orbit;
+            // if(fTriggerMask) std::cout << "===>";
+            // std::cout << std::hex << ir.orbit << " " << ir.bc << " " << fInputMask << " " << fTriggerMask << std::dec << " (" << ir.orbit << " " << ir.bc << ")"  << std::endl;
+          }
+        }
+      }
+    } else {
+      std::cout << "return 1" << std::endl;
+      return 1;
+    }
+  }
+  // return 0;
+  //  Read CTP digits and check if every class mask in digits is in AOD
+  //  CTP digits
+  TFile* fileDigits = TFile::Open("ctpdigitsWOD.root");
+  //
+  fileDigits->ls();
+  o2::ctp::CTPDigit* dig = new o2::ctp::CTPDigit;
+  //
+  // tree->Print();
+  TTreeReader reader("o2sim", fileDigits);
+  TTreeReaderValue> ctpdigs(reader, "CTPDigits");
+  // TTreeReaderArray ctpdigs(reader,"CTPDigits");
+  bool firstE = true;
+  //
+  std::cout << "Doing ctpdigits" << std::endl;
+  std::map bc2classmaskD;
+  std::map bc2inputmaskD;
+  std::bitset<48> tvxmask;
+  tvxmask.set(2);
+  int nnotfinp = 0;
+  int nokinp = 0;
+  int nnotf = 0;
+  int nok = 0;
+  while (reader.Next()) {
+    if (ctpdigs.GetSetupStatus() < 0) {
+      std::cout << "Error:" << std::dec << ctpdigs.GetSetupStatus() << " for:" << ctpdigs.GetBranchName() << std::endl;
+      return 1;
+    }
+    // std::cout << "size:" << std::dec << ctpdigs.GetSize() << std::endl;
+    // std::cout << "size:" << std::dec << ctpdigs->size() << std::endl;
+    for (auto const& dig : *ctpdigs) {
+      ULong64_t gbc = dig.intRecord.toLong();
+      if (dig.CTPClassMask.count()) {
+        bc2classmaskD[gbc] = dig.CTPClassMask.to_ullong();
+        // int del = 280+17;
+        // auto it = bc2classmask.find (gbc);
+        // if(it != bc2classmask.end()) {
+        if (bc2classmask.count(gbc)) {
+          // bc2classmask.erase(gbc);
+          nok++;
+          // std::cout << std::hex << gbc << " tc aod clsmask:" << bc2classmask[gbc] << " " << dig.CTPClassMask.to_ullong()  << std::endl;
+          //  dig.printStream(std::cout);
+        } else {
+          std::cout << std::dec << dig.intRecord.orbit << " " << dig.intRecord.bc << " " << std::hex << dig.intRecord.toLong() << " not found " << bc2classmask.count(gbc) << std::endl;
+          nnotf++;
+        }
+      }
+      if (dig.CTPInputMask.count()) {
+        bc2inputmaskD[gbc] = dig.CTPInputMask.to_ullong();
+        // auto it = bc2inputmask.find (gbc);
+        // if(it != bc2inputmask.end()) {
+        if (bc2inputmask.count(gbc)) {
+          // bc2inputmask.erase(gbc);
+          nokinp++;
+          // std::cout << std::hex << gbc << " ir aod inpmask:" << bc2inputmask[gbc] << " " << dig.CTPInputMask.to_ullong() << std::endl;
+        } else {
+          std::cout << std::dec << dig.intRecord.orbit << " " << dig.intRecord.bc << " " << std::hex << dig.intRecord.toLong() << " not found " << bc2inputmask.count(gbc) << std::endl;
+          nnotfinp++;
+        }
+      }
+    }
+  }
+  std::cout << "TClasses ===> nok:" << nok << " NOT found in digits:" << nnotf << " left:" << bc2classmask.size() << std::endl;
+  std::cout << "Inputs   ===> nok:" << nokinp << " NOT found in digits:" << nnotfinp << " left:" << bc2inputmask.size() << std::endl;
+  nok = 0;
+  nokinp = 0;
+  nnotf = 0;
+  nnotfinp = 0;
+  for (auto const& tm : bc2classmask) {
+    if (bc2classmaskD.count(tm.first)) {
+      nok++;
+    } else {
+      nnotf++;
+    }
+  }
+  for (auto const& tm : bc2inputmask) {
+    if (bc2inputmaskD.count(tm.first)) {
+      nokinp++;
+    } else {
+      nnotfinp++;
+    }
+  }
+
+  std::cout << "TClasses ===> nok:" << nok << " NOT found in aod:" << nnotf << " left:" << bc2classmaskD.size() << std::endl;
+  std::cout << "Inputs   ===> nok:" << nokinp << " NOT found in aod:" << nnotfinp << " left:" << bc2inputmaskD.size() << std::endl;
+  return 0;
+}
diff --git a/Detectors/CTP/macro/CheckCTPConfig.C b/Detectors/CTP/macro/CheckCTPConfig.C
new file mode 100644
index 0000000000000..24a5e354f3fcd
--- /dev/null
+++ b/Detectors/CTP/macro/CheckCTPConfig.C
@@ -0,0 +1,58 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file CreateCTPConfig.C
+/// \brief create CTP config, test it and add to database
+/// \author Roman Lietava
+
+#if !defined(__CLING__) || defined(__ROOTCLING__)
+
+#include 
+#include "CCDB/CcdbApi.h"
+#include "CCDB/BasicCCDBManager.h"
+#include "DataFormatsCTP/Configuration.h"
+#include 
+#include 
+#include 
+#endif
+using namespace o2::ctp;
+int CheckCTPConfig(std::string cfgRun3str = "/home/rl/backup24/runs/559781.rcfg2", int writeToFile = 0)
+{
+  //
+  // run3 config
+  //
+  if (cfgRun3str.find(".rcfg") == std::string::npos) {
+    std::cout << "No file name:" << cfgRun3str << std::endl;
+    return 1;
+  } else {
+    std::string filename = cfgRun3str;
+    std::ifstream in;
+    in.open(filename);
+    if (!in) {
+      std::cout << "Can not open file:" << filename << std::endl;
+      return 2;
+    }
+    std::stringstream buffer;
+    buffer << in.rdbuf();
+    cfgRun3str = buffer.str();
+  }
+  //
+  CTPConfiguration ctpcfg;
+  int ret = ctpcfg.loadConfigurationRun3(cfgRun3str);
+  ctpcfg.printStream(std::cout);
+  std::cout << "CTP config done" << std::endl;
+  // ctpcfg.checkConfigConsistency();
+  auto ctpclasses = ctpcfg.getCTPClasses();
+  for (auto const& cls : ctpclasses) {
+    std::cout << cls.descriptor->name << ":" << std::hex << cls.descriptor->getInputsMask() << std::endl;
+  }
+  return ret;
+}
diff --git a/Detectors/CTP/macro/CheckCTPDigits.C b/Detectors/CTP/macro/CheckCTPDigits.C
new file mode 100644
index 0000000000000..15cf6df7cf364
--- /dev/null
+++ b/Detectors/CTP/macro/CheckCTPDigits.C
@@ -0,0 +1,100 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file checkCTPDigits.C
+/// \brief create CTP config, test it and add to database
+/// \author Roman Lietava
+
+#if !defined(__CLING__) || defined(__ROOTCLING__)
+
+#include 
+#include "TFile.h"
+#include "TTree.h"
+#include 
+#include 
+#include 
+#include "TKey.h"
+#include "TTreeReader.h"
+#include "TTreeReaderValue.h"
+#endif
+// Check if trigger class mask has corresponding input class mask
+// Tp be generalised to use CTP config
+using namespace o2::ctp;
+int CheckCTPDigits(bool files = 0)
+{
+  if (files == 0) {
+    return 0;
+  }
+  // CTP digits
+  TFile* fileDigits = TFile::Open("ctpdigits.root");
+  //
+  fileDigits->ls();
+  o2::ctp::CTPDigit* dig = new o2::ctp::CTPDigit;
+  //
+  // tree->Print();
+  TTreeReader reader("o2sim", fileDigits);
+  // TTreeReaderValue> ctpdigs(reader,"CTPDigits");
+  TTreeReaderArray ctpdigs(reader, "CTPDigits");
+  bool firstE = true;
+  //
+  std::bitset<48> tvxmask;
+  tvxmask.set(2);
+  int ncls = 0;
+  int nnotf = 0;
+  while (reader.Next()) {
+    if (ctpdigs.GetSetupStatus() < 0) {
+      std::cout << "Error:" << std::dec << ctpdigs.GetSetupStatus() << " for:" << ctpdigs.GetBranchName() << std::endl;
+      return 1;
+    }
+    std::cout << "size:" << std::dec << ctpdigs.GetSize() << std::endl;
+    int del = 280 + 14;
+    int i, j, bc;
+    for (i = del; i < ctpdigs.GetSize(); i++) {
+      o2::ctp::CTPDigit* dig = &ctpdigs[i];
+      if (dig->CTPClassMask.count() > 0) {
+        std::cout << std::dec << "=======> BC tm:" << dig->intRecord.bc << " O:" << dig->intRecord.orbit << std::hex << " 0b" << dig->CTPClassMask << " gbc:" << dig->intRecord.toLong() << std::endl;
+        int found = 0;
+        ncls++;
+        std::stringstream ss;
+        for (j = 0; j < del; j++) {
+          ss << std::dec << ctpdigs[i - j].intRecord.bc << " O:" << ctpdigs[i - j].intRecord.orbit << " " << std::hex << ctpdigs[i - j].CTPInputMask.to_ullong() << std::endl;
+          bc = (dig->intRecord.bc - del) % 3564;
+          if (bc < 0)
+            bc += 3564;
+          if (bc == ctpdigs[i - j].intRecord.bc) {
+            std::cout << " found:" << std::dec << j << " 0b" << ctpdigs[i - j].CTPInputMask << std::endl;
+            found = 1;
+            auto istvx = ctpdigs[i - j].CTPInputMask & tvxmask;
+            if (istvx.count() == 0)
+              std::cout << "error: tcx missing " << std::endl;
+            break;
+          }
+        }
+        // std::cout << std::endl;
+        if (0) {
+          if (found == 0) {
+            std::cout << " NOT FOUND:" << std::dec << bc;
+            std::cout << std::dec << " =======> BC tm:" << dig->intRecord.bc << ""
+                      << " O:" << dig->intRecord.orbit << std::hex << " 0b" << dig->CTPClassMask << std::endl;
+            std::cout << ss.str() << std::endl;
+            nnotf++;
+          } else {
+            std::cout << " FOUND:" << std::dec << bc << " " << ctpdigs[i - j].intRecord.bc;
+            std::cout << std::dec << " =======> BC tm:" << dig->intRecord.bc << ""
+                      << " O:" << dig->intRecord.orbit << std::hex << " 0b" << dig->CTPClassMask << std::endl;
+          }
+        }
+      }
+    }
+  }
+  std::cout << "# of cls masks:" << std::dec << ncls++ << " NOT found:" << nnotf << std::endl;
+  return 0;
+}
diff --git a/Detectors/CTP/macro/CreateBKForRun.C b/Detectors/CTP/macro/CreateBKForRun.C
new file mode 100644
index 0000000000000..06ef9eac5f900
--- /dev/null
+++ b/Detectors/CTP/macro/CreateBKForRun.C
@@ -0,0 +1,74 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#if !defined(__CLING__) || defined(__ROOTCLING__)
+#include 
+#include 
+#include 
+#include 
+#include 
+#endif
+using namespace o2::ctp;
+
+void CreateBKForRun()
+{
+  std::vector runs = {558124, 558126, 558215, 558217, 558221, 558244, 558247};
+  std::string mCCDBPathCTPScalers = "CTP/Calib/Scalers";
+  std::string mCCDBPathCTPConfig = "CTP/Config/Config";
+  //
+  std::string filename = "BKcounters.txt";
+  std::ofstream outfile(filename);
+  if (!outfile) {
+    Error("", "Failed to open file %s", filename.c_str());
+    return;
+  }
+  auto& ccdbMgr = o2::ccdb::BasicCCDBManager::instance();
+  for (auto const& runNumber : runs) {
+    auto soreor = ccdbMgr.getRunDuration(runNumber);
+    uint64_t timeStamp = (soreor.second - soreor.first) / 2 + soreor.first;
+    std::cout << runNumber << " Timestamp:" << timeStamp << std::endl;
+    //
+    std::string srun = std::to_string(runNumber);
+    std::map metadata;
+    metadata["runNumber"] = srun;
+    auto ctpscalers = ccdbMgr.getSpecific(mCCDBPathCTPScalers, timeStamp, metadata);
+    if (ctpscalers == nullptr) {
+      LOG(info) << "CTPRunScalers not in database, timestamp:" << timeStamp;
+    }
+    auto ctpcfg = ccdbMgr.getSpecific(mCCDBPathCTPConfig, timeStamp, metadata);
+    if (ctpcfg == nullptr) {
+      LOG(info) << "CTPRunConfig not in database, timestamp:" << timeStamp;
+    }
+    //
+    ctpscalers->convertRawToO2();
+    std::vector& ctpcls = ctpcfg->getCTPClasses();
+    std::vector clslist = ctpcfg->getTriggerClassList();
+    auto times = ctpscalers->getTimeLimit();
+    for (size_t i = 0; i < clslist.size(); i++) {
+      // std::cout << i << " " << ctpcls[i].name ;
+      std::array cnts = ctpscalers->getIntegralForClass(i);
+      if (clslist[i] != (int)cnts[0]) {
+        LOG(fatal) << "cls list incompatible with counters";
+      }
+      std::cout << std::setw(21) << ctpcls[cnts[0]].name;
+      outfile << runNumber << ", " << ctpcls[i].name << ", " << std::get<1>(times) / 1000;
+      for (int j = 1; j < 7; j++) {
+        // std::cout << std::setw(21) << " " << cnts[j];
+        std::cout << ", " << cnts[j];
+        outfile << ", " << cnts[j];
+      }
+      std::cout << std::endl;
+      outfile << std::endl;
+    }
+  }
+  // ctpscalers->printFromZero(std::cout);
+  outfile.close();
+}
diff --git a/Detectors/CTP/macro/CreateCTPConfig.C b/Detectors/CTP/macro/CreateCTPConfig.C
index 467db02ef7f0b..a3e4b0c8c35f1 100644
--- a/Detectors/CTP/macro/CreateCTPConfig.C
+++ b/Detectors/CTP/macro/CreateCTPConfig.C
@@ -15,10 +15,11 @@
 
 #if !defined(__CLING__) || defined(__ROOTCLING__)
 
-#include "FairLogger.h"
+#include 
 #include "CCDB/CcdbApi.h"
 #include "CCDB/BasicCCDBManager.h"
 #include "DataFormatsCTP/Configuration.h"
+#include "CTPWorkflowScalers/ctpCCDBManager.h"
 #include 
 #include 
 #include 
@@ -33,8 +34,9 @@ CTPConfiguration CreateCTPConfig(std::string cfgRun3str = "", int writeToFile =
   //
   if (cfgRun3str.find(".rcfg") == std::string::npos) {
     cfgRun3str =
-      "bcm TOF 100 1288 2476 \n \
-bcm PHYS 1226 \n\
+      "run 123 \n \
+bcm TOF 100 1288 2476 \n \
+bcm bcmPHYS 1226 \n\
 bcd10 1khz \n\
 bcd20 0 \n\
 bcd2m 45khz \n\
@@ -50,14 +52,16 @@ LTG mch \n\
 ferst 1 \n\
 # 3 clusters for CRU, TRD and oldTTC detectors: \n\
 0 cluster clu1 fv0 ft0 fdd its mft mid mch tpc zdc tst tof \n\
-0 cl_ph PHYS \n\
+0 cl_ph bcmPHYS 3\n\
 # \n\
 1 cluster clu2 trd \n\
 1 cl_45khz bcd2m \n\
 2 cluster clu3 hmp phs \n\
 2 cl_1khz bcd10 \n \
 3 cluster clu4 emc cpv \n \
-4 cl_5khz bcd20 \n";
+4 cl_5khz bcd20 \n \
+5 cl3 3 5\n \
+6 clMTVX MTVX\n";
   } else {
     std::string filename = cfgRun3str;
     std::ifstream in;
@@ -73,6 +77,15 @@ ferst 1 \n\
   //
   ctpcfg.loadConfigurationRun3(cfgRun3str);
   ctpcfg.printStream(std::cout);
+  std::cout << "CTP config done" << std::endl;
+  ctpcfg.checkConfigConsistency();
+  if (0) {
+    o2::ctp::ctpCCDBManager* man = new ctpCCDBManager;
+    man->setCCDBHost("http://ccdb-test.cern.ch:8080");
+    man->saveRunConfigToCCDB(&ctpcfg, 1665784953);
+    // uint64_t classmask = ctpcfg.getClassMaskForInputMask(0x4);
+    // std::cout << "classmask:" << std::hex << classmask << std::dec << std::endl;
+  }
   if (writeToFile == 1) {
     std::unique_ptr myFile(TFile::Open("CTPConfig.root", "RECREATE"));
     myFile->WriteObject(&ctpcfg, "CTPConfig");
diff --git a/Detectors/CTP/macro/GetAndSave.C b/Detectors/CTP/macro/GetAndSave.C
new file mode 100644
index 0000000000000..ff70a3055c957
--- /dev/null
+++ b/Detectors/CTP/macro/GetAndSave.C
@@ -0,0 +1,62 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file TestCTPScalers.C
+/// \brief create CTP scalers, test it and add to database
+/// \author Roman Lietava
+#if !defined(__CLING__) || defined(__ROOTCLING__)
+
+#include 
+#include "CCDB/CcdbApi.h"
+#include "CCDB/BasicCCDBManager.h"
+#include "DataFormatsCTP/Scalers.h"
+#include "DataFormatsCTP/Configuration.h"
+#include "CTPWorkflowScalers/ctpCCDBManager.h"
+#include "TFile.h"
+#include "TString.h"
+#include 
+#include 
+#include 
+#endif
+using namespace o2::ctp;
+void GetAndSave(std::string ccdbHost = "http://ccdb-test.cern.ch:8080")
+{
+  std::string CCDBPathCTPScalers = "CTP/Calib/Scalers";
+  // std::vector runs = {"518541","518543","518546","518547"};
+  // std::vector timestamps = {1655116302316,1655118513690,1655121997478,1655123792911};
+  std::vector runs = {"519903", "519904", "519905", "519906"};
+  std::vector timestamps = {1656658674161, 1656660737184, 1656667772462, 1656669421115};
+  // std::vector runs = {"518543"};
+  // std::vector timestamps = {1655118513690};
+  int i = 0;
+  ctpCCDBManager mng;
+  // mng.setCCDBHost(ccdbHost);
+  auto& mgr = o2::ccdb::BasicCCDBManager::instance();
+  mgr.setURL(ccdbHost);
+  for (auto const& run : runs) {
+    CTPConfiguration ctpcfg;
+    CTPRunScalers scl;
+    std::map metadata; // can be empty
+    metadata["runNumber"] = run;
+    CTPRunScalers* ctpscalers = mgr.getSpecific(CCDBPathCTPScalers, timestamps[i], metadata);
+    if (ctpscalers == nullptr) {
+      std::cout << run << " CTPRunScalers not in database, timestamp:" << timestamps[i] << std::endl;
+    } else {
+      // ctpscalers->printStream(std::cout);
+      std::string name = run + ".root";
+      TFile* myFile = TFile::Open(name.c_str(), "RECREATE");
+      myFile->WriteObject(ctpscalers, "CTPRunScalers");
+      // myFile->Write();
+      std::cout << run << " ok" << std::endl;
+    }
+    i++;
+  }
+}
diff --git a/Detectors/CTP/macro/GetRates.C b/Detectors/CTP/macro/GetRates.C
new file mode 100644
index 0000000000000..8894d7935b99e
--- /dev/null
+++ b/Detectors/CTP/macro/GetRates.C
@@ -0,0 +1,55 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#if !defined(__CLING__) || defined(__ROOTCLING__)
+#include 
+#include 
+#include "CTPWorkflowScalers/ctpCCDBManager.h"
+#include "Framework/Logger.h"
+#endif
+using namespace o2::ctp;
+
+void GetRates(int run = 559617)
+{
+  uint64_t inputmaskCum = 0, classmackCum = 0;
+  int ntrigSel = 0;
+
+  auto& cmb = o2::ccdb::BasicCCDBManager::instance();
+  auto ctpcfg = cmb.getSpecificForRun("CTP/Config/Config", run);
+  if (!ctpcfg) {
+    LOGP(error, "Can not get config for run {}", run);
+    return;
+  }
+  CTPConfiguration ctpconfig;
+  ctpconfig.loadConfigurationRun3(ctpcfg->getConfigString());
+  ctpconfig.printStream(std::cout);
+  auto& triggerclasses = ctpconfig.getCTPClasses();
+  LOGP(info, "Found {} trigger classes", triggerclasses.size());
+  int indexInList = 0;
+  for (const auto& trgclass : triggerclasses) {
+    uint64_t inputmask = 0;
+    if (trgclass.descriptor != nullptr) {
+      inputmask = trgclass.descriptor->getInputsMask();
+      // LOGP(info, "inputmask: {:#x}", inputmask);
+    }
+    trgclass.printStream(std::cout);
+    //    std::cout << indexInList << ": " << trgclass.name << ", input mask 0x" << std::hex << inputmask << ", class mask 0x" << trgclass.classMask << std::dec << std::endl;
+    indexInList++;
+    if (trgclass.cluster->getClusterDetNames().find("TRD") != std::string::npos || trgclass.cluster->getClusterDetNames().find("trd") != std::string::npos) {
+      LOGP(info, "Found TRD trigger cluster, class mask: {:#x}, input mask: {:#x}", trgclass.classMask, inputmask);
+      inputmaskCum |= inputmask;
+      classmackCum |= trgclass.classMask;
+      ntrigSel++;
+    }
+  }
+
+  LOGP(info, "Found {} triggers with TRD: classMasks: {:#x}  inputMasks: {:#x}", ntrigSel, classmackCum, inputmaskCum);
+}
diff --git a/Detectors/CTP/macro/GetScalers.C b/Detectors/CTP/macro/GetScalers.C
index 2cfb1fd0c177f..1f104850c8c39 100644
--- a/Detectors/CTP/macro/GetScalers.C
+++ b/Detectors/CTP/macro/GetScalers.C
@@ -12,13 +12,16 @@
 /// \file TestCTPScalers.C
 /// \brief create CTP scalers, test it and add to database
 /// \author Roman Lietava
+// root -b -q "GetScalers.C(\"519499\", 1656286373953)"
 #if !defined(__CLING__) || defined(__ROOTCLING__)
 
-#include "FairLogger.h"
+#include 
 #include "CCDB/CcdbApi.h"
 #include "CCDB/BasicCCDBManager.h"
 #include "DataFormatsCTP/Scalers.h"
 #include "DataFormatsCTP/Configuration.h"
+// #include "BookkeepingApi/BkpClientFactory.h"
+#include "CTPWorkflowScalers/ctpCCDBManager.h"
 #include 
 #include 
 #include 
@@ -26,12 +29,6 @@
 using namespace o2::ctp;
 void GetScalers(std::string srun, long time, std::string ccdbHost = "http://ccdb-test.cern.ch:8080")
 {
-  o2::ccdb::CcdbApi cdb;
-  cdb.init("http://alice-ccdb.cern.ch");
-  int runNumber = 518420;
-  // std::string srun = "519044";
-  // std::string srun = "519045";
-  // std::string srun = "519502";
   std::map metadata;
   metadata["runNumber"] = srun;
   // auto hd = cdb.retrieveHeaders("RCT/Info/RunInformation", {}, runNumber);
@@ -39,17 +36,31 @@ void GetScalers(std::string srun, long time, std::string ccdbHost = "http://ccdb
   // std::cout << stol(hd["SOR"]) << "\n";
   CTPConfiguration ctpcfg;
   CTPRunScalers scl;
-  CTPRunManager mng;
-  mng.setCCDBHost("http://ccdb-test.cern.ch:8080");
-  // mng.setCCDBPathScalers("CTP/Scalers");
-  scl = mng.getScalersFromCCDB(time, srun);
-  scl.convertRawToO2();
-  // scl.printStream(std::cout);
-  // scl.printRates();
-  scl.printIntegrals();
-  ctpcfg = mng.getConfigFromCCDB(time, srun);
-  // std::vector clsses;
-  // clsses = ctpcfg.getTriggerClassList();
-  // std::cout << clsses.size() << std::endl;
-  // for(auto const& i : clsses) std::cout << i << std::endl;
+  o2::ctp::ctpCCDBManager mng;
+  mng.setCCDBHost(ccdbHost);
+  bool ok;
+  // ctpcfg = mng.getConfigFromCCDB(time, srun);
+  // ctpcfg.printStream(std::cout);
+  // return;
+  scl = mng.getScalersFromCCDB(time, srun, ok);
+  if (ok == 1) {
+    scl.printStream(std::cout);
+    scl.convertRawToO2();
+    // scl.printO2(std::cout);
+    // scl.printFromZero(std::cout);
+    // scl.printIntegrals();
+    // scl.printRates();
+    std::cout << "TVX,TSC,TCE,ZNC:" << std::endl;
+    scl.printInputRateAndIntegral(3);
+    scl.printInputRateAndIntegral(4);
+    scl.printInputRateAndIntegral(5);
+    scl.printInputRateAndIntegral(26);
+    std::cout << " TVX,TVX&TCE,TVX&TSC,TVX&VCH:" << std::endl;
+    scl.printClassBRateAndIntegral(3);
+    scl.printClassBRateAndIntegral(4);
+    scl.printClassBRateAndIntegral(5);
+    scl.printClassBRateAndIntegral(6);
+  } else {
+    std::cout << "Can not find run, please, check parameters" << std::endl;
+  }
 }
diff --git a/Detectors/CTP/macro/GetScalersForRun.C b/Detectors/CTP/macro/GetScalersForRun.C
new file mode 100644
index 0000000000000..a96d79d569213
--- /dev/null
+++ b/Detectors/CTP/macro/GetScalersForRun.C
@@ -0,0 +1,141 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#if !defined(__CLING__) || defined(__ROOTCLING__)
+#include 
+#include 
+#include 
+#include 
+#include "Common/CCDB/ctpRateFetcher.h"
+#endif
+using namespace o2::ctp;
+
+void GetScalersForRun(int runNumber = 0, int fillN = 0, bool test = 1)
+{
+  if (test == 0) {
+    return;
+  }
+  std::string mCCDBPathCTPScalers = "CTP/Calib/Scalers";
+  std::string mCCDBPathCTPConfig = "CTP/Config/Config";
+
+  auto& ccdbMgr = o2::ccdb::BasicCCDBManager::instance();
+  auto soreor = ccdbMgr.getRunDuration(runNumber);
+  uint64_t timeStamp = (soreor.second - soreor.first) / 2 + soreor.first;
+  std::cout << "Timestamp:" << timeStamp << std::endl;
+  //
+  std::string sfill = std::to_string(fillN);
+  std::map metadata;
+  metadata["fillNumber"] = sfill;
+  auto lhcifdata = ccdbMgr.getSpecific("GLO/Config/GRPLHCIF", timeStamp);
+  auto bfilling = lhcifdata->getBunchFilling();
+  std::vector bcs = bfilling.getFilledBCs();
+  std::cout << "Number of interacting bc:" << bcs.size() << std::endl;
+  //
+  std::string srun = std::to_string(runNumber);
+  metadata.clear(); // can be empty
+  metadata["runNumber"] = srun;
+  auto ctpscalers = ccdbMgr.getSpecific(mCCDBPathCTPScalers, timeStamp, metadata);
+  if (ctpscalers == nullptr) {
+    LOG(info) << "CTPRunScalers not in database, timestamp:" << timeStamp;
+  }
+  auto ctpcfg = ccdbMgr.getSpecific(mCCDBPathCTPConfig, timeStamp, metadata);
+  if (ctpcfg == nullptr) {
+    LOG(info) << "CTPRunConfig not in database, timestamp:" << timeStamp;
+  }
+  std::cout << "all good" << std::endl;
+  ctpscalers->convertRawToO2();
+  std::vector ctpcls = ctpcfg->getCTPClasses();
+  // std::vector clslist = ctpcfg->getTriggerClassList();
+  std::vector clslist = ctpscalers->getClassIndexes();
+  std::map clsIndexToScaler;
+  std::cout << "Classes:";
+  int i = 0;
+  for (auto const& cls : clslist) {
+    std::cout << cls << " ";
+    clsIndexToScaler[cls] = i;
+    i++;
+  }
+  std::cout << std::endl;
+  int tsc = 255;
+  int tce = 255;
+  int vch = 255;
+  int iznc = 255;
+  for (auto const& cls : ctpcls) {
+    if (cls.name.find("CMTVXTSC-B-NOPF-CRU") != std::string::npos) {
+      tsc = cls.getIndex();
+      std::cout << cls.name << ":" << tsc << std::endl;
+    }
+    if (cls.name.find("CMTVXTCE-B-NOPF-CRU") != std::string::npos) {
+      tce = cls.getIndex();
+      std::cout << cls.name << ":" << tce << std::endl;
+    }
+    if (cls.name.find("CMTVXVCH-B-NOPF-CRU") != std::string::npos) {
+      vch = cls.getIndex();
+      std::cout << cls.name << ":" << vch << std::endl;
+    }
+    // if (cls.name.find("C1ZNC-B-NOPF-CRU") != std::string::npos) {
+    if (cls.name.find("C1ZNC-B-NOPF") != std::string::npos) {
+      iznc = cls.getIndex();
+      std::cout << cls.name << ":" << iznc << std::endl;
+    }
+  }
+  std::vector recs = ctpscalers->getScalerRecordO2();
+  if (recs[0].scalersInps.size() == 48) {
+    std::cout << "ZNC:";
+    int inp = 26;
+    double_t nbc = bcs.size();
+    double_t frev = 11245;
+    double_t sigmaratio = 28.;
+    double_t time0 = recs[0].epochTime;
+    double_t timeL = recs[recs.size() - 1].epochTime;
+    double_t Trun = timeL - time0;
+    double_t integral = recs[recs.size() - 1].scalersInps[inp - 1] - recs[0].scalersInps[inp - 1];
+    double_t rate = integral / Trun;
+    double_t rat = integral / Trun / nbc / frev;
+    double_t mu = -TMath::Log(1 - rat);
+    double_t pp = 1 - mu / (TMath::Exp(mu) - 1);
+    double_t ratepp = mu * nbc * frev;
+    double_t integralpp = ratepp * Trun;
+    std::cout << "Rate:" << rate / sigmaratio << " Integral:" << integral << " mu:" << mu << " Pileup prob:" << pp;
+    std::cout << " Integralpp:" << integralpp << " Ratepp:" << ratepp / sigmaratio << std::endl;
+    // ctpscalers->printInputRateAndIntegral(26);
+  } else {
+    std::cout << "Inputs not available" << std::endl;
+  }
+  //
+  if (tsc != 255) {
+    std::cout << "TSC:";
+    ctpscalers->printClassBRateAndIntegral(clsIndexToScaler[tsc] + 1);
+  }
+  if (tce != 255) {
+    std::cout << "TCE:";
+    ctpscalers->printClassBRateAndIntegral(clsIndexToScaler[tce] + 1);
+  }
+  // std::cout << "TCE input:" << ctpscalers->printInputRateAndIntegral(5) << std::endl;;
+  if (vch != 255) {
+    std::cout << "VCH:";
+    ctpscalers->printClassBRateAndIntegral(clsIndexToScaler[vch] + 1);
+  }
+  if (iznc != 255) {
+    std::cout << "ZNC class:";
+    // uint64_t integral = recs[recs.size() - 1].scalers[iznc].l1After - recs[0].scalers[iznc].l1After;
+    auto zncrate = ctpscalers->getRateGivenT(timeStamp * 1.e-3, iznc, 6);
+    std::cout << "ZNC class rate:" << zncrate.first / 28. << std::endl;
+  } else {
+    std::cout << "ZNC class not available, rate from input:" << std::endl;
+  }
+  auto zncinprate = ctpscalers->getRateGivenT((timeStamp + 100) * 1.e-3, 25, 7);
+  std::cout << "ZNC inp rate:" << zncinprate.first / 28. << " " << zncinprate.second / 28. << std::endl;
+
+  // ctpRateFetcher ctprate;
+  // auto fetcherRate = ctprate.fetch(&ccdbMgr, timeStamp, runNumber, "ZNChadronic");
+  // std::cout << "Result from the fetcher (ZNC): " << fetcherRate << std::endl;
+}
diff --git a/Detectors/CTP/macro/PlotOrbit.C b/Detectors/CTP/macro/PlotOrbit.C
new file mode 100644
index 0000000000000..9f70341ecc0da
--- /dev/null
+++ b/Detectors/CTP/macro/PlotOrbit.C
@@ -0,0 +1,100 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file TestCTPScalers.C
+/// \brief create CTP scalers, test it and add to database
+/// \author Roman Lietava
+// root -b -q "GetScalers.C(\"519499\", 1656286373953)"
+#if !defined(__CLING__) || defined(__ROOTCLING__)
+
+#include 
+#include "CCDB/CcdbApi.h"
+#include "CCDB/BasicCCDBManager.h"
+#include "DataFormatsCTP/Scalers.h"
+#include "DataFormatsCTP/Configuration.h"
+#include 
+#include 
+#include 
+#endif
+//"http://ccdb-test.cern.ch:8080"
+using namespace o2::ctp;
+void PlotOrbit(int runNumber)
+{ //
+  std::string mCCDBPathCTPScalers = "CTP/Calib/Scalers";
+  auto& ccdbMgr = o2::ccdb::BasicCCDBManager::instance();
+  // Timestamp
+  auto soreor = ccdbMgr.getRunDuration(runNumber);
+  // uint64_t timeStamp = (soreor.second - soreor.first) / 2 + soreor.first;
+  uint64_t timeStamp = soreor.first + 60 * 1000;
+  std::cout << "Timestamp:" << timeStamp << std::endl;
+  //
+  // Scalers
+  std::string srun = std::to_string(runNumber);
+  std::map metadata;
+  metadata["runNumber"] = srun;
+  // ccdbMgr.setURL("http://ccdb-test.cern.ch:8080");
+  auto scl = ccdbMgr.getSpecific(mCCDBPathCTPScalers, timeStamp, metadata);
+  if (scl == nullptr) {
+    LOG(info) << "CTPRunScalers not in database, timestamp:" << timeStamp;
+    return;
+  }
+  scl->convertRawToO2();
+  // return ;
+  std::vector recs = scl->getScalerRecordRaw();
+  std::cout << "raw rec size:" << recs.size() << std::endl;
+  //
+  //
+  // Anal
+  //
+  // Times
+  int64_t time0 = recs[0].epochTime;
+  int64_t timeL = recs[recs.size() - 1].epochTime;
+  double_t Trun = timeL - time0;
+  // double_t orbit0 = recs[0].intRecord.orbit;
+  int64_t orbit0 = scl->getOrbitLimitFromRaw().first;
+  int64_t orbitL = scl->getOrbitLimitFromRaw().second;
+  int n = recs.size() - 1;
+  std::cout << " Run duration:" << Trun << " Scalers size:" << n + 1 << std::endl;
+  std::cout << "Orbit0:" << orbit0 << " orbitL:" << orbitL << std::endl;
+  return;
+  Double_t x[n], orbit[n];
+  // Double_t tcetsctoznc[n], tcetoznc[n], vchtoznc[n];
+  for (int i = 0; i < n; i++) {
+    // x[i] = i;
+    x[i] = recs[i + 1].epochTime - time0;
+    orbit[i] = recs[i + 1].intRecord.orbit - orbit0;
+  }
+  //
+  gStyle->SetMarkerSize(0.5);
+  TGraph* gr1 = new TGraph(n, x, orbit);
+  // TGraph* gr2 = new TGraph(n, x, tcetsctoznc);
+  // TGraph* gr3 = new TGraph(n, x, tcetoznc);
+  // TGraph* gr4 = new TGraph(n, x, vchtoznc);
+  gr1->SetMarkerStyle(20);
+  // gr2->SetMarkerStyle(21);
+  // gr3->SetMarkerStyle(23);
+  // gr4->SetMarkerStyle(23);
+  std::string title = "Orbit vs EpochTime for run " + srun + " ;EpochTime[s]; Orbit";
+  gr1->SetTitle(title.c_str());
+  // gr2->SetTitle("R=(TSC+TCE)*TVTX*B*28/ZNC; time[sec]; R");
+  //  gr2->GetHistogram()->SetMaximum(1.1);
+  //  gr2->GetHistogram()->SetMinimum(0.9);
+  // gr3->SetTitle("R=(TCE)*TVTX*B*28/ZNC; time[sec]; R");
+  //  gr3->GetHistogram()->SetMaximum(0.6);
+  //  gr3->GetHistogram()->SetMinimum(0.4);
+  // gr4->SetTitle("R=(VCH)*TVTX*B*28/ZNC; time[sec]; R");
+  //  gr4->GetHistogram()->SetMaximum(0.6);
+  //  gr4->GetHistogram()->SetMinimum(0.4);
+  TCanvas* c1 = new TCanvas("c1", srun.c_str(), 200, 10, 800, 500);
+  gr1->Draw("AP");
+  std::cout << "epoch0:" << time0 << " epochL:" << timeL << " T:" << Trun << std::endl;
+  std::cout << "orbit0:" << orbit0 << " orbitL:" << orbitL << " T:" << (orbitL - orbit0) * 88e-6 << std::endl;
+}
diff --git a/Detectors/CTP/macro/PlotPbLumi.C b/Detectors/CTP/macro/PlotPbLumi.C
new file mode 100644
index 0000000000000..4bda8d25e006e
--- /dev/null
+++ b/Detectors/CTP/macro/PlotPbLumi.C
@@ -0,0 +1,265 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file PlotPbLumi.C
+/// \brief create CTP scalers, test it and add to database
+/// \author Roman Lietava
+// root "PLotPbLumi.C(519499)"
+#if !defined(__CLING__) || defined(__ROOTCLING__)
+
+#include 
+#include "CCDB/CcdbApi.h"
+#include "CCDB/BasicCCDBManager.h"
+#include "DataFormatsCTP/Scalers.h"
+#include "DataFormatsCTP/Configuration.h"
+#include "DataFormatsParameters/GRPLHCIFData.h"
+#include "TGraph.h"
+#include "TMath.h"
+#include "TCanvas.h"
+#include "TStyle.h"
+#include 
+#include 
+#include 
+#endif
+using namespace o2::ctp;
+//
+// sum = 0: TCE and TSC separatelly otherwise TCE and (TCE+TSC)
+// qc = 0: takes scalers from CCDB (available only for finished runs) otherwise from QCCDB (available for active runs)
+// t0-tlast: window in seconds counted from beginning of run
+//
+void PlotPbLumi(int runNumber = 567905, bool sum = 0, bool qc = 0, Double_t t0 = 0., Double_t tlast = 0.)
+{ //
+  // PLots in one canvas
+  // znc rate/28
+  // R = (TCE+TSC)*TVX*B*/ZNC*28
+  // R = TCE*TVX*B/ZNC*28
+  // R = VCH*TVX*B/ZNC*28
+  std::string ccdbHost = "http://alice-ccdb.cern.ch";
+  std::string mCCDBPathCTPScalers = "/CTP/Calib/Scalers";
+  std::string mCCDBPathCTPScalersQC = "qc/CTP/Scalers";
+  std::string mCCDBPathCTPConfig = "CTP/Config/Config";
+  auto& ccdbMgr = o2::ccdb::BasicCCDBManager::instance();
+  // Timestamp
+  auto soreor = ccdbMgr.getRunDuration(runNumber);
+  uint64_t timeStamp = (soreor.second - soreor.first) / 2 + soreor.first;
+  std::cout << "Timestamp:" << timeStamp << std::endl;
+  // Filling
+  auto lhcifdata = ccdbMgr.getForRun("GLO/Config/GRPLHCIF", runNumber);
+  // auto lhcifdata = ccdbMgr.getSpecific("GLO/Config/GRPLHCIF", timeStamp, metadata);
+  if (!lhcifdata) {
+    throw std::runtime_error("No GRPLHCIFData for run " + std::to_string(runNumber));
+  }
+  auto bfilling = lhcifdata->getBunchFilling();
+  std::vector bcs = bfilling.getFilledBCs();
+  int nbc = bcs.size();
+  std::cout << "Number of interacting bc:" << nbc << std::endl;
+  // Scalers
+  std::string srun = std::to_string(runNumber);
+  std::map metadata;
+  metadata["runNumber"] = srun;
+  CTPRunScalers* scl = nullptr;
+  if (qc) {
+    ccdbMgr.setURL("http://ali-qcdb-gpn.cern.ch:8083");
+    scl = ccdbMgr.getSpecific(mCCDBPathCTPScalersQC, timeStamp, metadata);
+  } else {
+    scl = ccdbMgr.getSpecific(mCCDBPathCTPScalers, timeStamp, metadata);
+  }
+  if (scl == nullptr) {
+    LOG(info) << "CTPRunScalers not in database, timestamp:" << timeStamp;
+    return;
+  }
+  scl->convertRawToO2();
+  std::vector recs = scl->getScalerRecordO2();
+  //
+  // CTPConfiguration ctpcfg;
+  ccdbMgr.setURL("http://alice-ccdb.cern.ch");
+  auto ctpcfg = ccdbMgr.getSpecific(mCCDBPathCTPConfig, timeStamp, metadata);
+  if (ctpcfg == nullptr) {
+    LOG(info) << "CTPRunConfig not in database, timestamp:" << timeStamp;
+    return;
+  }
+  std::vector clslist = ctpcfg->getTriggerClassList();
+  // std::vector clslist = scl->getClassIndexes();
+  std::map clsIndexToScaler;
+  std::cout << "Classes:";
+  int i = 0;
+  for (auto const& cls : clslist) {
+    std::cout << cls << " ";
+    clsIndexToScaler[cls] = i;
+    i++;
+  }
+  std::cout << std::endl;
+  std::vector ctpcls = ctpcfg->getCTPClasses();
+  int tsc = 255;
+  int tce = 255;
+  int vch = 255;
+  int zncclsi = 255;
+  for (auto const& cls : ctpcls) {
+    if (cls.name.find("CMTVXTSC-B-NOPF") != std::string::npos && tsc == 255) {
+      int itsc = cls.getIndex();
+      tsc = clsIndexToScaler[itsc];
+      // tsc = scl->getScalerIndexForClass(itsc);
+      std::cout << cls.name << ":" << tsc << ":" << itsc << std::endl;
+    }
+    if (cls.name.find("CMTVXTCE-B-NOPF-CRU") != std::string::npos) {
+      int itce = cls.getIndex();
+      tce = clsIndexToScaler[itce];
+      // tce = scl->getScalerIndexForClass(itce);
+      std::cout << cls.name << ":" << tce << ":" << itce << std::endl;
+    }
+    if (cls.name.find("CMTVXVCH-B-NOPF-CRU") != std::string::npos) {
+      int ivch = cls.getIndex();
+      vch = clsIndexToScaler[ivch];
+      // vch = scl->getScalerIndexForClass(ivch);
+      std::cout << cls.name << ":" << vch << ":" << ivch << std::endl;
+    }
+    if (cls.name.find("C1ZNC-B-NOPF-CRU") != std::string::npos) {
+      int iznc = cls.getIndex();
+      zncclsi = clsIndexToScaler[iznc];
+      // vch = scl->getScalerIndexForClass(ivch);
+      std::cout << cls.name << ":" << zncclsi << ":" << iznc << std::endl;
+    }
+  }
+  if (tsc == 255 || tce == 255 || vch == 255) {
+    std::cout << " One of dcalers not available, check config to find alternative)" << std::endl;
+    return;
+  }
+  //
+  // Anal
+  //
+  // Times
+  double_t frev = 11245;
+  double_t time0 = recs[0].epochTime;
+  double_t timeL = recs[recs.size() - 1].epochTime;
+  double_t Trun = timeL - time0;
+  double_t orbit0 = recs[0].intRecord.orbit;
+  int n = recs.size() - 1;
+  std::cout << " Run duration:" << Trun << " Scalers size:" << n + 1 << std::endl;
+  //
+  int i0 = 0;
+  int ilast = 0;
+  if (t0 != 0. || tlast != 0.) {
+    for (int i = 0; i < n; i++) {
+      double_t ttime = recs[i].epochTime - time0;
+      if (!i0 && t0 < ttime) {
+        i0 = i;
+      }
+      if (!ilast && tlast < ttime) {
+        ilast = i;
+      }
+    }
+  } else {
+    ilast = n;
+  }
+  n = ilast - i0;
+  std::cout << "i0:" << i0 << " ilast:" << ilast << std::endl;
+  // Double_t x[n], znc[n], zncpp[n];
+  std::vector xvec(n), zncvec(n), zncppvec(n), zncclassvec(n);
+  Double_t* x = xvec.data();
+  Double_t* znc = zncvec.data();
+  Double_t* zncpp = zncppvec.data();
+  Double_t* zncclass = zncclassvec.data();
+  // Double_t tcetsctoznc[n], tcetoznc[n], vchtoznc[n];
+  std::vector tcetsctozncvec(n), tcetozncvec(n), vchtozncvec(n);
+  Double_t* tcetsctoznc = tcetsctozncvec.data();
+  Double_t* tcetoznc = tcetozncvec.data();
+  Double_t* vchtoznc = vchtozncvec.data();
+  for (int i = i0; i < ilast; i++) {
+    int iv = i - i0;
+    x[iv] = (double_t)(recs[i + 1].intRecord.orbit + recs[i].intRecord.orbit) / 2. - orbit0;
+    x[iv] *= 88e-6;
+    // x[i] = (double_t)(recs[i+1].epochTime + recs[i].epochTime)/2.;
+    double_t tt = (double_t)(recs[i + 1].intRecord.orbit - recs[i].intRecord.orbit);
+    tt = tt * 88e-6;
+    //
+    // std::cout << recs[i+1].scalersInps[25] << std::endl;
+    double_t znci = (double_t)(recs[i + 1].scalersInps[25] - recs[i].scalersInps[25]);
+    double_t mu = -TMath::Log(1. - znci / tt / nbc / frev);
+    double_t zncipp = mu * nbc * frev;
+    zncpp[iv] = zncipp / 28.;
+    znc[iv] = znci / 28. / tt;
+    // znc class
+    znci = recs[i + 1].scalers[zncclsi].l1Before - recs[i].scalers[zncclsi].l1Before;
+    zncclass[iv] = znci / 28. / tt;
+    // std::cout << znc[i]/zncclass[i] << std::endl;
+    //
+    double_t had = 0;
+    if (sum) {
+      had += recs[i + 1].scalers[tce].lmBefore - recs[i].scalers[tce].lmBefore;
+    }
+    double_t mutce = -TMath::Log(1. - had / tt / nbc / frev);
+    // std::cout << recs[i+1].scalers[tce].lmBefore << std::endl;
+    had += recs[i + 1].scalers[tsc].lmBefore - recs[i].scalers[tsc].lmBefore;
+    // rat = (double_t)(had)/double_t(recs[i+1].scalersInps[25] - recs[i].scalersInps[25])*28;
+    tcetsctoznc[iv] = (double_t)(had) / zncpp[iv] / tt;
+    had = recs[i + 1].scalers[tce].lmBefore - recs[i].scalers[tce].lmBefore;
+    // rat = (double_t)(had)/double_t(recs[i+1].scalersInps[25] - recs[i].scalersInps[25])*28;
+    tcetoznc[iv] = (double_t)(had) / zncpp[iv] / tt;
+    had = recs[i + 1].scalers[vch].lmBefore - recs[i].scalers[vch].lmBefore;
+    double_t muvch = -TMath::Log(1. - had / tt / nbc / frev);
+
+    // rat = (double_t)(had)/double_t(recs[i+1].scalersInps[25] - recs[i].scalersInps[25])*28;
+    vchtoznc[iv] = (double_t)(had) / zncpp[iv] / tt;
+    // std::cout << "muzdc:" << mu << " mu tce:" << mutce << " muvch:" << muvch << std::endl;
+  }
+  //
+  gStyle->SetMarkerSize(0.5);
+  TGraph* gr1 = new TGraph(n, x, znc);
+  TGraph* gr11 = new TGraph(n, x, zncpp);    // PileuP corrected
+  TGraph* gr12 = new TGraph(n, x, zncclass); // NOT PileuP corrected
+  TGraph* gr2 = new TGraph(n, x, tcetsctoznc);
+  TGraph* gr3 = new TGraph(n, x, tcetoznc);
+  TGraph* gr4 = new TGraph(n, x, vchtoznc);
+  gr1->SetMarkerStyle(20);
+  gr11->SetMarkerStyle(20);
+  gr12->SetMarkerStyle(20);
+  gr11->SetMarkerColor(kRed);
+  gr12->SetMarkerColor(kBlue);
+  gr2->SetMarkerStyle(21);
+  gr3->SetMarkerStyle(23);
+  gr4->SetMarkerStyle(23);
+  if (sum) {
+    gr2->SetTitle("R=(TSC+TCE)*TVTX*B*28/ZNC; time[sec]; R");
+  } else {
+    gr2->SetTitle("R=(TSC)*TVTX*B*28/ZNC; time[sec]; R");
+  }
+  // gr2->GetHistogram()->SetMaximum(1.1);
+  // gr2->GetHistogram()->SetMinimum(0.9);
+  gr3->SetTitle("R=(TCE)*TVTX*B*28/ZNC; time[sec]; R");
+  // gr3->GetHistogram()->SetMaximum(0.6);
+  // gr3->GetHistogram()->SetMinimum(0.4);
+  gr4->SetTitle("R=(VCH)*TVTX*B*28/ZNC; time[sec]; R");
+  // gr4->GetHistogram()->SetMaximum(0.6);
+  // gr4->GetHistogram()->SetMinimum(0.4);
+  TMultiGraph* mg1 = new TMultiGraph();
+  mg1->SetTitle("R=ZNC/28 rate [Hz] (red=PilUp Corrected); time[sec]; R");
+  mg1->Add(gr1);
+  mg1->Add(gr11);
+  mg1->Add(gr12);
+  TCanvas* c1 = new TCanvas("c1", srun.c_str(), 200, 10, 800, 500);
+  std::string title = "RUN " + std::to_string(runNumber);
+  c1->SetTitle(title.c_str());
+  c1->Divide(2, 2);
+  c1->cd(1);
+  mg1->Draw("AP");
+  c1->cd(2);
+  gr2->Draw("AP");
+  c1->cd(3);
+  gr3->Draw("AP");
+  c1->cd(4);
+  gr4->Draw("AP");
+  // getRate test:
+  double tt = timeStamp / 1000.;
+  std::pair r1 = scl->getRateGivenT(tt, 25, 7);
+  std::cout << "ZDC input getRateGivetT:" << r1.first / 28. << " " << r1.second / 28. << std::endl;
+  std::pair r2 = scl->getRateGivenT(tt, tce, 1);
+  std::cout << "LM before TCE class getRateGivetT:" << r2.first << " " << r2.second << std::endl;
+}
diff --git a/Detectors/CTP/macro/ReadCTPRunScalersFromFile.C b/Detectors/CTP/macro/ReadCTPRunScalersFromFile.C
new file mode 100644
index 0000000000000..385a8c7e91e15
--- /dev/null
+++ b/Detectors/CTP/macro/ReadCTPRunScalersFromFile.C
@@ -0,0 +1,54 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file TestCTPScalers.C
+/// \brief create CTP scalers, test it and add to database
+/// \author Roman Lietava
+#if !defined(__CLING__) || defined(__ROOTCLING__)
+
+#include 
+#include "CCDB/CcdbApi.h"
+#include "CCDB/BasicCCDBManager.h"
+#include "DataFormatsCTP/Scalers.h"
+#include "DataFormatsCTP/Configuration.h"
+#include "TFile.h"
+#include "TString.h"
+#include 
+#include 
+#include 
+#endif
+using namespace o2::ctp;
+void ReadCTPRunScalersFromFile(std::string name = "s.root")
+{
+  std::cout << "Reading file:" << name << std::endl;
+  TFile* myFile = TFile::Open(name.c_str());
+  bool doscalers = 1;
+  bool doconfig = 0;
+  if (doscalers) {
+    // CTPRunScalers* ctpscalers = myFile->Get("ccdb_object");
+    CTPRunScalers* ctpscalers = myFile->Get("CTPRunScalers");
+    if (ctpscalers != nullptr) {
+      ctpscalers->printStream(std::cout);
+      ctpscalers->convertRawToO2();
+      ctpscalers->printIntegrals();
+    } else {
+      std::cout << "Scalers not there ?" << std::endl;
+    }
+  }
+  if (doconfig) {
+    CTPConfiguration* ctpconfig = myFile->Get("CTPConfig");
+    if (ctpconfig != nullptr) {
+      ctpconfig->printStream(std::cout);
+    } else {
+      std::cout << "Config not there ?" << std::endl;
+    }
+  }
+}
diff --git a/Detectors/CTP/macro/SaveInputsConfig.C b/Detectors/CTP/macro/SaveInputsConfig.C
index fc6bcc40a5100..459fbf4024c95 100644
--- a/Detectors/CTP/macro/SaveInputsConfig.C
+++ b/Detectors/CTP/macro/SaveInputsConfig.C
@@ -14,7 +14,7 @@
 /// \author Roman Lietava
 #if !defined(__CLING__) || defined(__ROOTCLING__)
 
-#include "FairLogger.h"
+#include 
 #include "CCDB/CcdbApi.h"
 #include "CCDB/BasicCCDBManager.h"
 #include "DataFormatsCTP/Scalers.h"
@@ -41,7 +41,7 @@ void SaveInputsConfig(std::string filename = "inputs.cfg", std::string ccdbHost
   long tmax = timeStamp + time365days;
   o2::ccdb::CcdbApi api;
   api.init(ccdbHost);           // or http://localhost:8080 for a local installation
-  map metadata; // can be empty
+  std::map metadata; // can be empty
   // store abitrary user object in strongly typed manner
   api.storeAsTFileAny(&ctpcfginps, "CTP/Calib/Inputs", metadata, tmin, tmax);
 }
diff --git a/Detectors/CTP/macro/TestCTPScalers.C b/Detectors/CTP/macro/TestCTPScalers.C
index 8591dcd1b5483..062a8e4385b73 100644
--- a/Detectors/CTP/macro/TestCTPScalers.C
+++ b/Detectors/CTP/macro/TestCTPScalers.C
@@ -14,7 +14,7 @@
 /// \author Roman Lietava
 #if !defined(__CLING__) || defined(__ROOTCLING__)
 
-#include "FairLogger.h"
+#include 
 #include "CCDB/CcdbApi.h"
 #include "CCDB/BasicCCDBManager.h"
 #include "DataFormatsCTP/Scalers.h"
diff --git a/Detectors/CTP/macro/TestConfig.C b/Detectors/CTP/macro/TestConfig.C
new file mode 100644
index 0000000000000..38da821ea4807
--- /dev/null
+++ b/Detectors/CTP/macro/TestConfig.C
@@ -0,0 +1,59 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#if !defined(__CLING__) || defined(__ROOTCLING__)
+#include 
+#include 
+#include "CTPWorkflowScalers/ctpCCDBManager.h"
+#endif
+using namespace o2::ctp;
+
+void TestConfig(bool test = 0)
+{
+  if (test == 0) {
+    return;
+  }
+  uint64_t timestamp = 1660196771632;
+  std::string run = "523148";
+  o2::ctp::ctpCCDBManager::setCCDBHost("https://alice-ccdb.cern.ch");
+  bool ok;
+  auto ctpcfg = o2::ctp::ctpCCDBManager::getConfigFromCCDB(timestamp, run, ok);
+  if (ok == 0) {
+    std::cout << "Can not get config for run:" << run << std::endl;
+  }
+  CTPConfiguration ctpconfig;
+  ctpconfig.loadConfigurationRun3(ctpcfg.getConfigString());
+  // ctpconfig.printStream(std::cout);
+  // return;
+  // ctpconfig.assignDescriptors();
+  // ctpconfig.createInputsInDecriptorsFromNames();
+  ctpconfig.printStream(std::cout);
+  auto& triggerclasses = ctpconfig.getCTPClasses();
+  std::cout << "Found " << triggerclasses.size() << " trigger classes" << std::endl;
+  int indexInList = 0;
+  for (const auto& trgclass : triggerclasses) {
+    uint64_t inputmask = 0;
+    // auto desc = ctpconfig.getDescriptor(trgclass.descriptorIndex);
+    // std::cout << "desc index:" << trgclass.descriptorIndex << " " << desc << std::endl;
+    if (trgclass.descriptor != nullptr) {
+      inputmask = trgclass.descriptor->getInputsMask();
+      std::cout << "inputmask:" << std::hex << inputmask << std::dec << std::endl;
+    }
+    trgclass.printStream(std::cout);
+    std::cout << indexInList << ": " << trgclass.name << ", input mask 0x" << std::hex << inputmask << ", class mask 0x" << trgclass.classMask << std::dec << std::endl;
+    indexInList++;
+    if (trgclass.cluster->getClusterDetNames().find("EMC") != std::string::npos) {
+      std::cout << "Found EMCAL trigger cluster, class mask: 0x" << std::hex << trgclass.classMask << std::dec << std::endl;
+    }
+  }
+  auto classmask = ctpconfig.getClassMaskForInputMask(0x4);
+  std::cout << "Found class mask " << classmask << std::endl;
+}
diff --git a/Detectors/CTP/macro/TestFetcher.C b/Detectors/CTP/macro/TestFetcher.C
new file mode 100644
index 0000000000000..b2b6912f49911
--- /dev/null
+++ b/Detectors/CTP/macro/TestFetcher.C
@@ -0,0 +1,46 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#if !defined(__CLING__) || defined(__ROOTCLING__)
+#include 
+#include 
+#include 
+#endif
+using namespace o2::ctp;
+
+void TestFetcher(int runNumber = 557251)
+{
+  auto& ccdb = o2::ccdb::BasicCCDBManager::instance();
+  std::pair pp = ccdb.getRunDuration(runNumber);
+  long ts = pp.first + 60;
+  std::cout << "Run duration:" << pp.first << " " << pp.second << std::endl;
+  // Opening run
+  CTPRateFetcher fetcher;
+  fetcher.setupRun(runNumber, &ccdb, ts, 0);
+  ccdb.setURL("http://ali-qcdb-gpn.cern.ch:8083/");
+  std::string QCDBPathCTPScalers = "qc/CTP/Scalers";
+  std::map metadata; // can be empty
+  std::string run = std::to_string(runNumber);
+  metadata["runNumber"] = run;
+  CTPRunScalers* ctpscalers = ccdb.getSpecific(QCDBPathCTPScalers, ts, metadata);
+  auto tt = ctpscalers->getTimeLimitFromRaw();
+  std::cout << "1st scalers duration:" << tt.first << " " << tt.second << std::endl;
+  fetcher.updateScalers(*ctpscalers);
+  auto rate = fetcher.fetchNoPuCorr(&ccdb, ts, runNumber, "T0VTX");
+  std::cout << "1st rate:" << rate << std::endl;
+  // Running on the same run
+  ts = ts + 5 * 1000 * 3600;
+  ctpscalers = ccdb.getSpecific(QCDBPathCTPScalers, ts, metadata);
+  std::cout << "Later scalers duration:" << tt.first << " " << tt.second << std::endl;
+  fetcher.updateScalers(*ctpscalers);
+  rate = fetcher.fetchNoPuCorr(&ccdb, ts, runNumber, "T0VTX");
+  std::cout << "Later rate:" << rate << std::endl;
+}
diff --git a/Detectors/CTP/macro/TestGetRates.C b/Detectors/CTP/macro/TestGetRates.C
new file mode 100644
index 0000000000000..19644853c568b
--- /dev/null
+++ b/Detectors/CTP/macro/TestGetRates.C
@@ -0,0 +1,48 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#if !defined(__CLING__) || defined(__ROOTCLING__)
+#include 
+#include 
+#include 
+#endif
+using namespace o2::ctp;
+
+void TestGetRates(int runN = 0)
+{
+  std::vector runs;
+  std::vector codes = {"T0VTX", "T0VTX", "ZNChadronic", "ZNChadronic", "T0VTX"};
+  if (runN == 0) {
+    runs = {529066, 539218, 544013, 544518, 557251};
+  } else {
+    runs.push_back(runN);
+  }
+  auto& ccdb = o2::ccdb::BasicCCDBManager::instance();
+  int i = 0;
+  for (auto const& runNumber : runs) {
+    // Opening run
+    std::pair pp = ccdb.getRunDuration(runNumber);
+    long ts = pp.first + 60;
+    // std::cout << "Run duration:" << pp.first << " " << pp.second << std::endl;
+    std::cout << "===> RUN:" << runNumber << " duration:" << (pp.second - pp.first) / 1000. << std::endl;
+
+    CTPRateFetcher fetcher;
+    fetcher.setupRun(runNumber, &ccdb, ts, 1);
+    fetcher.setOrbit(1);
+    std::array rates;
+    fetcher.getRates(rates, &ccdb, runNumber, codes[i]);
+    std::cout << "Start:" << rates[0] << " End:" << rates[1] << " Middle:" << rates[2] << " code:" << codes[i] << std::endl;
+    double lumi1 = fetcher.getLumi(&ccdb, runNumber, codes[i], 0);
+    double lumi2 = fetcher.getLumi(&ccdb, runNumber, codes[i], 1);
+    std::cout << " Lumi NO pile up corr:" << lumi1 << " Lumi with pile upcorr:" << lumi2 << " code:" << codes[i] << std::endl;
+    i++;
+  }
+}
diff --git a/Detectors/CTP/macro/compare.py b/Detectors/CTP/macro/compare.py
new file mode 100755
index 0000000000000..70c9326c3d8ca
--- /dev/null
+++ b/Detectors/CTP/macro/compare.py
@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+#import numpy as np
+dir = "/home/rl/tests/digitsNew/"
+def openfile(filename):
+  filename = dir+filename
+  print("Opening:",filename)
+  with open(filename) as f:
+    lines = f.readlines()
+  out = []
+  for line in lines:
+    #print(line)
+    item = line.split("[INFO]")[1]
+    out.append(item[0:-1])
+  #for l in out: print(l)
+  return out
+def compare(file1,file2):
+  f1 = openfile(file1)
+  f2 = openfile(file2)
+  s1 = len(f1)
+  s2 = len(f2)
+  print(file1,":",s1,file2,":",s2)
+  s = s1
+  if s1 > s2: s = s2
+  for i in range(s):
+    if f1[i] != f2[i]:
+      print(i,f1[i])
+      print(i,f2[i])
+if __name__ == "__main__":
+ #compare("outv6","outv6_32")
+ compare("outv6","outv7_32")
diff --git a/Detectors/CTP/macro/dumpCTPRO.C b/Detectors/CTP/macro/dumpCTPRO.C
new file mode 100644
index 0000000000000..e65c246b316e3
--- /dev/null
+++ b/Detectors/CTP/macro/dumpCTPRO.C
@@ -0,0 +1,70 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file checkCTPDigits.C
+/// \brief create CTP config, test it and add to database
+/// \author Roman Lietava
+
+#if !defined(__CLING__) || defined(__ROOTCLING__)
+
+#include 
+#include "TFile.h"
+#include "TTree.h"
+#include 
+#include 
+#include 
+#include "TKey.h"
+#include "TTreeReader.h"
+#include "TTreeReaderValue.h"
+#endif
+// Check if trigger class mask has corresponding input class mask
+// Tp be generalised to use CTP config
+using namespace o2::ctp;
+int dumpCTPRO(bool files = 0)
+{
+  if (files == 0) {
+    return 0;
+  }
+  // CTP digits
+  // TFile* fileDigits = TFile::Open("/home/rl/tests/digits/ctpdigits.root");
+  TFile* fileDigits = TFile::Open("/home/rl/pp/529690/ctpdigits.root");
+  // TFile* fileDigits = TFile::Open("/home/rl/PbPb/529403/1820/ctpdigits.root");
+  // TFile* fileDigits = TFile::Open("/home/rl/PbPb/529403/1820/ctpdigitsNOParams.root");
+  //
+  fileDigits->ls();
+  o2::ctp::CTPDigit* dig = new o2::ctp::CTPDigit;
+  //
+  // tree->Print();
+  TTreeReader reader("o2sim", fileDigits);
+  TTreeReaderArray ctpdigs(reader, "CTPDigits");
+  bool firstE = true;
+  //
+  int ORBIT = 3564;
+  int ninps = 0;
+  int nClass = 0;
+  uint32_t orbitmax = 0;
+  uint32_t orbitmin = 0xffffffff;
+  while (reader.Next()) {
+    if (ctpdigs.GetSetupStatus() < 0) {
+      std::cout << "Error:" << std::dec << ctpdigs.GetSetupStatus() << " for:" << ctpdigs.GetBranchName() << std::endl;
+      return 1;
+    }
+    // std::cout << "size:" << std::dec << ctpdigs.GetSize() << std::endl;
+    int i;
+    for (i = 0; i < ctpdigs.GetSize(); i++) {
+      o2::ctp::CTPDigit* dig = &ctpdigs[i];
+      uint64_t inpMask = dig->CTPInputMask.to_ullong();
+      uint64_t trgMask = dig->CTPClassMask.to_ullong();
+      std::cout << dig->intRecord.orbit << " " << dig->intRecord.bc << " " << std::hex << inpMask << " " << trgMask << std::dec << std::endl;
+    }
+  }
+  return 0;
+}
diff --git a/Detectors/CTP/reconstruction/CMakeLists.txt b/Detectors/CTP/reconstruction/CMakeLists.txt
index e06f1e8ec9909..3aba8c3c2d588 100644
--- a/Detectors/CTP/reconstruction/CMakeLists.txt
+++ b/Detectors/CTP/reconstruction/CMakeLists.txt
@@ -11,7 +11,10 @@
 
 o2_add_library(CTPReconstruction
                SOURCES src/CTFCoder.cxx src/CTFHelper.cxx
+               src/RawDataDecoder.cxx
                PUBLIC_LINK_LIBRARIES O2::DataFormatsCTP
+                                     O2::DPLUtils
+                                     O2::DetectorsRaw
                                      O2::DetectorsBase
                                      O2::CommonDataFormat
                                      O2::DetectorsCommonDataFormats
diff --git a/Detectors/CTP/reconstruction/include/CTPReconstruction/CTFCoder.h b/Detectors/CTP/reconstruction/include/CTPReconstruction/CTFCoder.h
index 9d2c323ec8c67..8dbc5adadbfc5 100644
--- a/Detectors/CTP/reconstruction/include/CTPReconstruction/CTFCoder.h
+++ b/Detectors/CTP/reconstruction/include/CTPReconstruction/CTFCoder.h
@@ -23,8 +23,9 @@
 #include "DataFormatsCTP/CTF.h"
 #include "DetectorsCommonDataFormats/DetID.h"
 #include "DetectorsBase/CTFCoderBase.h"
-#include "rANS/rans.h"
 #include "CTPReconstruction/CTFHelper.h"
+#include "CTPReconstruction/RawDataDecoder.h"
+#include "DataFormatsCTP/Configuration.h"
 
 class TTree;
 
@@ -33,38 +34,69 @@ namespace o2
 namespace ctp
 {
 
-class CTFCoder : public o2::ctf::CTFCoderBase
+class CTFCoder final : public o2::ctf::CTFCoderBase
 {
  public:
-  CTFCoder(o2::ctf::CTFCoderBase::OpType op) : o2::ctf::CTFCoderBase(op, CTF::getNBlocks(), o2::detectors::DetID::CTP) {}
+  CTFCoder(o2::ctf::CTFCoderBase::OpType op, const std::string& ctfdictOpt = "none") : o2::ctf::CTFCoderBase(op, CTF::getNBlocks(), o2::detectors::DetID::CTP, 1.f, ctfdictOpt) {}
   ~CTFCoder() final = default;
 
   /// entropy-encode data to buffer with CTF
   template 
-  o2::ctf::CTFIOSize encode(VEC& buff, const gsl::span& data);
+  o2::ctf::CTFIOSize encode(VEC& buff, const gsl::span& data, const LumiInfo& lumi);
 
   /// entropy decode data from buffer with CTF
   template 
-  o2::ctf::CTFIOSize decode(const CTF::base& ec, VTRG& data);
+  o2::ctf::CTFIOSize decode(const CTF::base& ec, VTRG& data, LumiInfo& lumi);
+
+  /// add CTP related shifts
+  template 
+  bool finaliseCCDB(o2::framework::ConcreteDataMatcher& matcher, void* obj);
 
   void createCoders(const std::vector& bufVec, o2::ctf::CTFCoderBase::OpType op) final;
+  void setDecodeInps(bool decodeinps) { mDecodeInps = decodeinps; }
+  void setCTPConfig(CTPConfiguration cfg) { mCTPConfig = std::move(cfg); }
+  bool getDecodeInps() { return mDecodeInps; }
+  CTPConfiguration& getCTPConfig() { return mCTPConfig; }
+  bool canApplyBCShiftInputs(const o2::InteractionRecord& ir) const { return canApplyBCShift(ir, mBCShiftInputs); }
 
  private:
+  template 
+  o2::ctf::CTFIOSize encode_impl(VEC& buff, const gsl::span& data, const LumiInfo& lumi);
+
   void appendToTree(TTree& tree, CTF& ec);
-  void readFromTree(TTree& tree, int entry, std::vector& data);
+  void readFromTree(TTree& tree, int entry, std::vector& data, LumiInfo& lumi);
+  std::vector mDataFilt;
+  CTPConfiguration mCTPConfig;
+  int mBCShiftInputs = 0;
+  bool mDecodeInps = false;
 };
 
 /// entropy-encode clusters to buffer with CTF
 template 
-o2::ctf::CTFIOSize CTFCoder::encode(VEC& buff, const gsl::span& data)
+o2::ctf::CTFIOSize CTFCoder::encode(VEC& buff, const gsl::span& data, const LumiInfo& lumi)
+{
+  if (mIRFrameSelector.isSet()) { // preselect data
+    mDataFilt.clear();
+    for (const auto& trig : data) {
+      if (mIRFrameSelector.check(trig.intRecord) >= 0) {
+        mDataFilt.push_back(trig);
+      }
+    }
+    return encode_impl(buff, mDataFilt, lumi);
+  }
+  return encode_impl(buff, data, lumi);
+}
+
+template 
+o2::ctf::CTFIOSize CTFCoder::encode_impl(VEC& buff, const gsl::span& data, const LumiInfo& lumi)
 {
   using MD = o2::ctf::Metadata::OptStore;
   // what to do which each field: see o2::ctd::Metadata explanation
   constexpr MD optField[CTF::getNBlocks()] = {
-    MD::EENCODE, // BLC_bcIncTrig
-    MD::EENCODE, // BLC_orbitIncTrig
-    MD::EENCODE, // BLC_bytesInput
-    MD::EENCODE, // BLC_bytesClass
+    MD::EENCODE_OR_PACK, // BLC_bcIncTrig
+    MD::EENCODE_OR_PACK, // BLC_orbitIncTrig
+    MD::EENCODE_OR_PACK, // BLC_bytesInput
+    MD::EENCODE_OR_PACK, // BLC_bytesClass
   };
 
   CTFHelper helper(data);
@@ -76,13 +108,12 @@ o2::ctf::CTFIOSize CTFCoder::encode(VEC& buff, const gsl::span&
   auto ec = CTF::create(buff);
   using ECB = CTF::base;
 
-  ec->setHeader(helper.createHeader());
+  ec->setHeader(helper.createHeader(lumi));
   assignDictVersion(static_cast(ec->getHeader()));
-  ec->getANSHeader().majorVersion = 0;
-  ec->getANSHeader().minorVersion = 1;
+  ec->setANSHeader(mANSVersion);
   // at every encoding the buffer might be autoexpanded, so we don't work with fixed pointer ec
   o2::ctf::CTFIOSize iosize;
-#define ENCODECTP(beg, end, slot, bits) CTF::get(buff.data())->encode(beg, end, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)].get(), getMemMarginFactor());
+#define ENCODECTP(beg, end, slot, bits) CTF::get(buff.data())->encode(beg, end, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)], getMemMarginFactor());
   // clang-format off
   iosize += ENCODECTP(helper.begin_bcIncTrig(),    helper.end_bcIncTrig(),     CTF::BLC_bcIncTrig,    0);
   iosize += ENCODECTP(helper.begin_orbitIncTrig(), helper.end_orbitIncTrig(),  CTF::BLC_orbitIncTrig, 0);
@@ -97,17 +128,17 @@ o2::ctf::CTFIOSize CTFCoder::encode(VEC& buff, const gsl::span&
 
 /// decode entropy-encoded digits
 template 
-o2::ctf::CTFIOSize CTFCoder::decode(const CTF::base& ec, VTRG& data)
+o2::ctf::CTFIOSize CTFCoder::decode(const CTF::base& ec, VTRG& data, LumiInfo& lumi)
 {
   auto header = ec.getHeader();
   checkDictVersion(static_cast(header));
   ec.print(getPrefix(), mVerbosity);
-  std::vector bcInc;
-  std::vector orbitInc;
+  std::vector bcInc;
+  std::vector orbitInc;
   std::vector bytesInput, bytesClass;
 
   o2::ctf::CTFIOSize iosize;
-#define DECODECTP(part, slot) ec.decode(part, int(slot), mCoders[int(slot)].get())
+#define DECODECTP(part, slot) ec.decode(part, int(slot), mCoders[int(slot)])
   // clang-format off
   iosize += DECODECTP(bcInc,       CTF::BLC_bcIncTrig);
   iosize += DECODECTP(orbitInc,    CTF::BLC_orbitIncTrig);
@@ -116,12 +147,19 @@ o2::ctf::CTFIOSize CTFCoder::decode(const CTF::base& ec, VTRG& data)
   // clang-format on
   //
   data.clear();
-
-  uint32_t firstEntry = 0, digCount = 0;
+  std::map digitsMap;
   o2::InteractionRecord ir(header.firstBC, header.firstOrbit);
+  lumi.nHBFCounted = header.lumiNHBFs;
+  lumi.nHBFCountedFV0 = header.lumiNHBFsFV0 ? header.lumiNHBFsFV0 : header.lumiNHBFs;
+  lumi.counts = header.lumiCounts;
+  lumi.countsFV0 = header.lumiCountsFV0;
+  lumi.orbit = header.lumiOrbit;
+  lumi.inp1 = int(header.inp1);
+  lumi.inp2 = int(header.inp2);
   auto itInp = bytesInput.begin();
   auto itCls = bytesClass.begin();
-
+  bool checkIROK = (mBCShift == 0); // need to check if CTP offset correction does not make the local time negative ?
+  bool checkIROKInputs = (mBCShiftInputs == 0); // need to check if CTP offset correction does not make the local time negative ?
   for (uint32_t itrig = 0; itrig < header.nTriggers; itrig++) {
     // restore TrigRecord
     if (orbitInc[itrig]) {  // non-0 increment => new orbit
@@ -130,20 +168,87 @@ o2::ctf::CTFIOSize CTFCoder::decode(const CTF::base& ec, VTRG& data)
     } else {
       ir.bc += bcInc[itrig];
     }
-    auto& dig = data.emplace_back();
-    dig.intRecord = ir;
-    for (int i = 0; i < CTFHelper::CTPInpNBytes; i++) {
-      dig.CTPInputMask |= static_cast(*itInp++) << (8 * i);
+    if (checkIROKInputs || canApplyBCShiftInputs(ir)) { // correction will be ok
+      auto irs = ir - mBCShiftInputs;
+      uint64_t CTPInputMask = 0;
+      for (int i = 0; i < CTFHelper::CTPInpNBytes; i++) {
+        CTPInputMask |= static_cast(*itInp++) << (8 * i);
+      }
+      if (CTPInputMask) {
+        if (digitsMap.count(irs)) {
+          if (digitsMap[irs].isInputEmpty()) {
+            digitsMap[irs].CTPInputMask = CTPInputMask;
+            // LOG(info) << "IR1:";
+            // digitsMap[irs].printStream(std::cout);
+          } else {
+            LOG(error) << "CTPInpurMask already exist:" << irs << " dig.CTPInputMask:" << digitsMap[irs].CTPInputMask << " CTPInputMask:" << CTPInputMask;
+          }
+        } else {
+          CTPDigit dig = {irs, CTPInputMask, 0};
+          digitsMap[irs] = dig;
+          // LOG(info) << "IR2:";
+          // digitsMap[irs].printStream(std::cout);
+        }
+      }
+    } else { // correction would make IR prior to mFirstTFOrbit, skip
+      itInp += CTFHelper::CTPInpNBytes;
     }
-    for (int i = 0; i < CTFHelper::CTPClsNBytes; i++) {
-      dig.CTPClassMask |= static_cast(*itCls++) << (8 * i);
+    if (checkIROK || canApplyBCShift(ir)) { // correction will be ok
+      auto irs = ir - mBCShift;
+      uint64_t CTPClassMask = 0;
+      for (int i = 0; i < CTFHelper::CTPClsNBytes; i++) {
+        CTPClassMask |= static_cast(*itCls++) << (8 * i);
+      }
+      if (CTPClassMask) {
+        if (digitsMap.count(irs)) {
+          if (digitsMap[irs].isClassEmpty()) {
+            digitsMap[irs].CTPClassMask = CTPClassMask;
+            // LOG(info) << "TCM1:";
+            // digitsMap[irs].printStream(std::cout);
+          } else {
+            LOG(error) << "CTPClassMask already exist:" << irs << " dig.CTPClassMask:" << digitsMap[irs].CTPClassMask << " CTPClassMask:" << CTPClassMask;
+          }
+        } else {
+          CTPDigit dig = {irs, 0, CTPClassMask};
+          digitsMap[irs] = dig;
+          // LOG(info) << "TCM2:";
+          // digitsMap[irs].printStream(std::cout);
+        }
+      }
+    } else { // correction would make IR prior to mFirstTFOrbit, skip
+      itCls += CTFHelper::CTPClsNBytes;
+    }
+  }
+  if (mDecodeInps) {
+    uint64_t trgclassmask = 0xffffffffffffffff;
+    if (mCTPConfig.getRunNumber() != 0) {
+      trgclassmask = mCTPConfig.getTriggerClassMask();
+    }
+    // std::cout << "trgclassmask:" << std::hex << trgclassmask << std::dec << std::endl;
+    o2::pmr::vector digits;
+    o2::ctp::RawDataDecoder::shiftInputs(digitsMap, digits, mFirstTFOrbit, trgclassmask);
+    for (auto const& dig : digits) {
+      data.emplace_back(dig);
+    }
+  } else {
+    for (auto const& dig : digitsMap) {
+      data.emplace_back(dig.second);
     }
   }
   assert(itInp == bytesInput.end());
   assert(itCls == bytesClass.end());
-  iosize.rawIn = data.size() * sizeof(CTPDigit);
+  iosize.rawIn = header.nTriggers * sizeof(CTPDigit);
   return iosize;
 }
+///________________________________
+template 
+bool CTFCoder::finaliseCCDB(o2::framework::ConcreteDataMatcher& matcher, void* obj)
+{
+  auto match = o2::ctf::CTFCoderBase::finaliseCCDB(matcher, obj);
+  mBCShiftInputs = -o2::ctp::TriggerOffsetsParam::Instance().globalInputsShift;
+  LOG(info) << "BCShiftInputs:" << mBCShiftInputs;
+  return match;
+}
 
 } // namespace ctp
 } // namespace o2
diff --git a/Detectors/CTP/reconstruction/include/CTPReconstruction/CTFHelper.h b/Detectors/CTP/reconstruction/include/CTPReconstruction/CTFHelper.h
index 2c1909d984715..34228e3acc55f 100644
--- a/Detectors/CTP/reconstruction/include/CTPReconstruction/CTFHelper.h
+++ b/Detectors/CTP/reconstruction/include/CTPReconstruction/CTFHelper.h
@@ -18,6 +18,7 @@
 
 #include "DataFormatsCTP/CTF.h"
 #include "DataFormatsCTP/Digits.h"
+#include "DataFormatsCTP/LumiInfo.h"
 #include 
 
 namespace o2
@@ -29,16 +30,16 @@ class CTFHelper
 {
 
  public:
-  CTFHelper(const gsl::span& data)
-    : mData(data) {}
+  CTFHelper(const gsl::span& data) : mData(data) {}
 
   static constexpr int CTPInpNBytes = CTP_NINPUTS / 8 + (CTP_NINPUTS % 8 > 0);
   static constexpr int CTPClsNBytes = CTP_NCLASSES / 8 + (CTP_NCLASSES % 8 > 0);
 
-  CTFHeader createHeader()
+  CTFHeader createHeader(const LumiInfo& lumi)
   {
     CTFHeader h{o2::detectors::DetID::CTP, 0, 1, 0, // dummy timestamp, version 1.0
-                uint32_t(mData.size()), 0, 0};
+                lumi.counts, lumi.countsFV0, lumi.nHBFCounted, lumi.nHBFCountedFV0, lumi.orbit,
+                uint32_t(mData.size()), 0, 0, uint16_t(lumi.inp1), uint16_t(lumi.inp2)};
     if (mData.size()) {
       h.firstOrbit = mData[0].intRecord.orbit;
       h.firstBC = mData[0].intRecord.bc;
@@ -54,7 +55,7 @@ class CTFHelper
   class _Iter
   {
    public:
-    using difference_type = int64_t;
+    using difference_type = std::ptrdiff_t;
     using value_type = T;
     using pointer = const T*;
     using reference = const T&;
@@ -63,52 +64,98 @@ class CTFHelper
     _Iter(const gsl::span& data, bool end = false) : mData(data), mIndex(end ? M * data.size() : 0){};
     _Iter() = default;
 
-    const I& operator++()
+    inline I& operator++() noexcept
     {
       ++mIndex;
-      return (I&)(*this);
+      return static_cast(*this);
     }
 
-    const I& operator--()
+    inline I operator++(int)
+    {
+      I res = *(static_cast(this));
+      ++mIndex;
+      return res;
+    }
+
+    inline I& operator--() noexcept
     {
       mIndex--;
-      return (I&)(*this);
+      return static_cast(*this);
+    }
+
+    inline I operator--(int)
+    {
+      I res = *(static_cast(this));
+      --mIndex;
+      return res;
     }
 
-    difference_type operator-(const I& other) const { return mIndex - other.mIndex; }
+    I& operator+=(difference_type i) noexcept
+    {
+      mIndex += i;
+      return static_cast(*this);
+    }
 
-    difference_type operator-(size_t idx) const { return mIndex - idx; }
+    I operator+(difference_type i) const
+    {
+      I res = *(const_cast(static_cast(this)));
+      return res += i;
+    }
+
+    I& operator-=(difference_type i) noexcept
+    {
+      mIndex -= i;
+      return static_cast(*this);
+    }
 
-    const I& operator-(size_t idx)
+    I operator-(difference_type i) const
     {
-      mIndex -= idx;
-      return (I&)(*this);
+      I res = *(const_cast(static_cast(this)));
+      return res -= i;
     }
 
-    bool operator!=(const I& other) const { return mIndex != other.mIndex; }
-    bool operator==(const I& other) const { return mIndex == other.mIndex; }
-    bool operator>(const I& other) const { return mIndex > other.mIndex; }
-    bool operator<(const I& other) const { return mIndex < other.mIndex; }
+    difference_type operator-(const I& other) const noexcept { return mIndex - other.mIndex; }
+
+    inline friend I operator+(difference_type i, const I& iter) { return iter + i; };
+
+    bool operator!=(const I& other) const noexcept { return mIndex != other.mIndex; }
+    bool operator==(const I& other) const noexcept { return mIndex == other.mIndex; }
+    bool operator>(const I& other) const noexcept { return mIndex > other.mIndex; }
+    bool operator<(const I& other) const noexcept { return mIndex < other.mIndex; }
+    bool operator>=(const I& other) const noexcept { return mIndex >= other.mIndex; }
+    bool operator<=(const I& other) const noexcept { return mIndex <= other.mIndex; }
 
    protected:
     gsl::span mData{};
-    size_t mIndex = 0;
+    difference_type mIndex = 0;
   };
 
   //_______________________________________________
   // BC difference wrt previous if in the same orbit, otherwise the abs.value.
   // For the very 1st entry return 0 (diff wrt 1st BC in the CTF header)
-  class Iter_bcIncTrig : public _Iter
+  class Iter_bcIncTrig : public _Iter
   {
    public:
-    using _Iter::_Iter;
+    using _Iter::_Iter;
     value_type operator*() const
     {
       if (mIndex) {
         if (mData[mIndex].intRecord.orbit == mData[mIndex - 1].intRecord.orbit) {
-          return mData[mIndex].intRecord.bc - mData[mIndex - 1].intRecord.bc;
+          return value_type(mData[mIndex].intRecord.bc - mData[mIndex - 1].intRecord.bc);
+        } else {
+          return value_type(mData[mIndex].intRecord.bc);
+        }
+      }
+      return 0;
+    }
+    value_type operator[](difference_type i) const
+    {
+      size_t id = mIndex + i;
+      if (id) {
+        if (mData[id].intRecord.orbit == mData[id - 1].intRecord.orbit) {
+          return value_type(mData[id].intRecord.bc - mData[id - 1].intRecord.bc);
         } else {
-          return mData[mIndex].intRecord.bc;
+          return value_type(mData[id].intRecord.bc);
         }
       }
       return 0;
@@ -117,11 +164,16 @@ class CTFHelper
 
   //_______________________________________________
   // Orbit difference wrt previous. For the very 1st entry return 0 (diff wrt 1st BC in the CTF header)
-  class Iter_orbitIncTrig : public _Iter
+  class Iter_orbitIncTrig : public _Iter
   {
    public:
-    using _Iter::_Iter;
-    value_type operator*() const { return mIndex ? mData[mIndex].intRecord.orbit - mData[mIndex - 1].intRecord.orbit : 0; }
+    using _Iter::_Iter;
+    value_type operator*() const { return value_type(mIndex ? mData[mIndex].intRecord.orbit - mData[mIndex - 1].intRecord.orbit : 0); }
+    value_type operator[](difference_type i) const
+    {
+      size_t id = mIndex + i;
+      return value_type(id ? mData[id].intRecord.orbit - mData[id - 1].intRecord.orbit : 0);
+    }
   };
 
   //_______________________________________________
@@ -133,6 +185,11 @@ class CTFHelper
     {
       return static_cast(((mData[mIndex / CTPInpNBytes].CTPInputMask.to_ullong()) >> (8 * (mIndex % CTPInpNBytes))) & 0xff);
     }
+    value_type operator[](difference_type i) const
+    {
+      size_t id = mIndex + i;
+      return static_cast(((mData[id / CTPInpNBytes].CTPInputMask.to_ullong()) >> (8 * (id % CTPInpNBytes))) & 0xff);
+    }
   };
 
   //_______________________________________________
@@ -144,6 +201,11 @@ class CTFHelper
     {
       return static_cast(((mData[mIndex / CTPClsNBytes].CTPClassMask.to_ullong()) >> (8 * (mIndex % CTPClsNBytes))) & 0xff);
     }
+    value_type operator[](difference_type i) const
+    {
+      size_t id = mIndex + i;
+      return static_cast(((mData[id / CTPClsNBytes].CTPClassMask.to_ullong()) >> (8 * (id % CTPClsNBytes))) & 0xff);
+    }
   };
 
   //<<< =========================== ITERATORS ========================================
diff --git a/Detectors/CTP/reconstruction/include/CTPReconstruction/RawDataDecoder.h b/Detectors/CTP/reconstruction/include/CTPReconstruction/RawDataDecoder.h
new file mode 100644
index 0000000000000..53addf32c538f
--- /dev/null
+++ b/Detectors/CTP/reconstruction/include/CTPReconstruction/RawDataDecoder.h
@@ -0,0 +1,104 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file RawDataDecoder.h
+/// \brief Digits  tw Raw translation
+/// \author Roman Lietava
+
+#ifndef ALICEO2_CTP_RAWDATADECODER_H_
+#define ALICEO2_CTP_RAWDATADECODER_H_
+
+#include 
+#include 
+#include 
+#include "Framework/InputRecord.h"
+#include "DataFormatsCTP/Digits.h"
+#include "DataFormatsCTP/LumiInfo.h"
+#include "DataFormatsCTP/Configuration.h"
+
+namespace o2
+{
+namespace ctp
+{
+class RawDataDecoder
+{
+ public:
+  RawDataDecoder() = default;
+  ~RawDataDecoder() = default;
+  static void makeGBTWordInverse(std::vector& diglets, gbtword80_t& GBTWord, gbtword80_t& remnant, uint32_t& size_gbt, uint32_t Npld);
+  int addCTPDigit(uint32_t linkCRU, uint32_t triggerOrbit, gbtword80_t& diglet, gbtword80_t& pldmask, std::map& digits);
+  int decodeRaw(o2::framework::InputRecord& inputs, std::vector& filter, o2::pmr::vector& digits, std::vector& lumiPointsHBF1);
+  int decodeRawFatal(o2::framework::InputRecord& inputs, std::vector& filter);
+  int decodeRaw(o2::framework::InputRecord& inputs, std::vector& filter, std::vector& digits, std::vector& lumiPointsHBF1);
+  void setDecodeInps(bool decodeinps) { mDecodeInps = decodeinps; }
+  void setDoLumi(bool lumi) { mDoLumi = lumi; }
+  void setDoDigits(bool digi) { mDoDigits = digi; }
+  void setVerbose(bool v) { mVerbose = v; }
+  void setMAXErrors(int m) { mErrorMax = m; }
+  int setLumiInp(int lumiinp, std::string inp);
+  void setCTPConfig(CTPConfiguration cfg) { mCTPConfig = std::move(cfg); };
+  void setCheckConsistency(bool check) { mCheckConsistency = check; }
+  uint32_t getIRRejected() const { return mIRRejected; }
+  uint32_t getTCRRejected() const { return mTCRRejected; }
+  std::vector& getTFOrbits() { return mTFOrbits; }
+  int getErrorIR() { return mErrorIR; }
+  int getErrorTCR() { return mErrorTCR; }
+  CTPConfiguration& getCTPConfig() { return mCTPConfig; }
+  int init();
+  static int shiftNew(const o2::InteractionRecord& irin, uint32_t TFOrbit, std::bitset<48>& inpmask, int64_t shift, int level, std::map& digmap);
+  static int shiftInputs(std::map& digitsMap, o2::pmr::vector& digits, uint32_t TFOrbit, uint64_t trgclassmask = 0xffffffffffffffff);
+  int checkReadoutConsistentncy(o2::pmr::vector& digits, uint64_t trgclassmask = 0xffffffffffffffff, uint64_t trigclassmaskNoTrgDets = 0xffffffffffffffff);
+  std::array getClassErrorsA() { return mClassErrorsA; }
+  std::array getClassErrorsB() { return mClassErrorsB; }
+  std::array getClassCountersA() { return mClassCountersA; }
+  std::array getClassCountersB() { return mClassCountersB; }
+  int getLostDueToShiftCls() { return mLostDueToShiftCC; }
+  int getLostDueToShiftInp() { return mLostDueToShiftInps; }
+
+ private:
+  static constexpr uint32_t TF_TRIGGERTYPE_MASK = 0x800;
+  static constexpr uint32_t HB_TRIGGERTYPE_MASK = 0x2;
+  // true: full inps decoding includine latency shifts here; false: latency shifts in CTF decoder
+  bool mDecodeInps = false;
+  bool mCheckConsistency = false;
+  // for digits
+  bool mDoDigits = true;
+  std::vector mOutputDigits;
+  // for lumi
+  bool mDoLumi = true;
+  //
+  static constexpr std::bitset LMMASKInputs = 0xfff;
+  static constexpr std::bitset L0MASKInputs = 0xfff000;
+  static constexpr std::bitset L1MASKInputs = (0xffffffull << 24);
+  gbtword80_t mTVXMask = 0x4;  // TVX is 3rd input
+  gbtword80_t mVBAMask = 0x20; // VBA is 6 th input
+  bool mVerbose = false;
+  int mIRRejected = 0;
+  int mTCRRejected = 0;
+  bool mPadding = true;
+  uint32_t mTFOrbit = 0;
+  std::vector mTFOrbits;
+  // error verbosness
+  int mErrorIR = 0;
+  int mErrorTCR = 0;
+  int mErrorMax = 5;
+  bool mStickyError = false;
+  std::array mClassErrorsA{};
+  std::array mClassErrorsB{}; // from inputs
+  std::array mClassCountersA{};
+  std::array mClassCountersB{}; // from inputs
+  int mLostDueToShiftCC = 0;
+  int mLostDueToShiftInps = 0;
+  CTPConfiguration mCTPConfig;
+};
+} // namespace ctp
+} // namespace o2
+#endif
diff --git a/Detectors/CTP/reconstruction/src/CTFCoder.cxx b/Detectors/CTP/reconstruction/src/CTFCoder.cxx
index 11b235e4a5240..e182032e12605 100644
--- a/Detectors/CTP/reconstruction/src/CTFCoder.cxx
+++ b/Detectors/CTP/reconstruction/src/CTFCoder.cxx
@@ -18,7 +18,6 @@
 #include 
 
 using namespace o2::ctp;
-
 ///___________________________________________________________________________________
 // Register encoded data in the tree (Fill is not called, will be done by caller)
 void CTFCoder::appendToTree(TTree& tree, CTF& ec)
@@ -28,12 +27,12 @@ void CTFCoder::appendToTree(TTree& tree, CTF& ec)
 
 ///___________________________________________________________________________________
 // extract and decode data from the tree
-void CTFCoder::readFromTree(TTree& tree, int entry, std::vector& data)
+void CTFCoder::readFromTree(TTree& tree, int entry, std::vector& data, LumiInfo& lumi)
 {
   assert(entry >= 0 && entry < tree.GetEntries());
   CTF ec;
   ec.readFromTree(tree, mDet.getName(), entry);
-  decode(ec, data);
+  decode(ec, data, lumi);
 }
 
 ///________________________________
@@ -41,10 +40,10 @@ void CTFCoder::createCoders(const std::vector& bufVec, o2::ctf::CTFCoderBa
 {
   const auto ctf = CTF::getImage(bufVec.data());
   // just to get types
-  uint16_t bcInc = 0;
-  uint32_t orbitInc = 0;
+  int16_t bcInc = 0;
+  int32_t orbitInc = 0;
   uint8_t bytesInput = 0, bytesClass = 0;
-#define MAKECODER(part, slot) createCoder(op, ctf.getFrequencyTable(slot), int(slot))
+#define MAKECODER(part, slot) createCoder(op, std::get>(ctf.getDictionary(slot, mANSVersion)), int(slot))
   // clang-format off
   MAKECODER(bcInc,      CTF::BLC_bcIncTrig);
   MAKECODER(orbitInc,   CTF::BLC_orbitIncTrig);
diff --git a/Detectors/CTP/reconstruction/src/RawDataDecoder.cxx b/Detectors/CTP/reconstruction/src/RawDataDecoder.cxx
new file mode 100644
index 0000000000000..a062a262acf62
--- /dev/null
+++ b/Detectors/CTP/reconstruction/src/RawDataDecoder.cxx
@@ -0,0 +1,691 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file RawDataDecoder.cxx
+/// \author Roman Lietava
+#include 
+#include "DetectorsRaw/RDHUtils.h"
+#include "DPLUtils/DPLRawParser.h"
+#include "DataFormatsCTP/TriggerOffsetsParam.h"
+#include "CTPReconstruction/RawDataDecoder.h"
+#include "DataFormatsCTP/Configuration.h"
+#include "DetectorsBase/GRPGeomHelper.h"
+#include 
+
+using namespace o2::ctp;
+
+// Inverse of Digits2Raw::makeGBTWord
+void RawDataDecoder::makeGBTWordInverse(std::vector& diglets, gbtword80_t& GBTWord, gbtword80_t& remnant, uint32_t& size_gbt, uint32_t Npld)
+{
+  gbtword80_t diglet = remnant;
+  uint32_t i = 0;
+  while (i < (NGBT - Npld)) {
+    std::bitset masksize = 0;
+    for (uint32_t j = 0; j < (Npld - size_gbt); j++) {
+      masksize[j] = 1;
+    }
+    diglet |= (GBTWord & masksize) << (size_gbt);
+    diglets.push_back(diglet);
+    diglet = 0;
+    i += Npld - size_gbt;
+    GBTWord = GBTWord >> (Npld - size_gbt);
+    size_gbt = 0;
+  }
+  size_gbt = NGBT - i;
+  remnant = GBTWord;
+}
+int RawDataDecoder::addCTPDigit(uint32_t linkCRU, uint32_t orbit, gbtword80_t& diglet, gbtword80_t& pldmask, std::map& digits)
+{
+  int ret = 0;
+  gbtword80_t pld = (diglet & pldmask);
+  if (pld.count() == 0) {
+    return 0;
+  }
+  pld >>= 12;
+  CTPDigit digit;
+  const gbtword80_t bcidmask = 0xfff;
+  uint16_t bcid = (diglet & bcidmask).to_ulong();
+  LOG(debug) << bcid << "    pld:" << pld;
+  o2::InteractionRecord ir = {bcid, orbit};
+  if (linkCRU == o2::ctp::GBTLinkIDIntRec) {
+    int32_t BCShiftCorrectionInps = -o2::ctp::TriggerOffsetsParam::Instance().globalInputsShift;
+    LOG(debug) << "InputMaskCount:" << digits[ir].CTPInputMask.count();
+    LOG(debug) << "ir ir ori:" << ir;
+    if ((ir.orbit <= mTFOrbit) && ((int32_t)ir.bc < BCShiftCorrectionInps)) {
+      // LOG(warning) << "Loosing ir:" << ir;
+      mIRRejected++;
+      return 0;
+    }
+    ir -= BCShiftCorrectionInps;
+    LOG(debug) << "ir ir corrected:" << ir;
+    digit.intRecord = ir;
+    if (digits.count(ir) == 0) {
+      digit.setInputMask(pld);
+      digits[ir] = digit;
+      LOG(debug) << bcid << " inputs case 0 bcid orbit " << orbit << " pld:" << pld;
+    } else if (digits.count(ir) == 1) {
+      if (digits[ir].CTPInputMask.count() == 0) {
+        digits[ir].setInputMask(pld);
+        LOG(debug) << bcid << " inputs bcid case 1 orbit " << orbit << " pld:" << pld;
+      } else {
+        if (mErrorIR < mErrorMax) {
+          LOG(error) << "Two CTP IRs with the same timestamp:" << ir.bc << " " << ir.orbit << " pld:" << pld << " dig:" << digits[ir];
+        }
+        ret = 4;
+        mErrorIR++;
+        mStickyError = true;
+      }
+    } else {
+      LOG(error) << "Two digits with the same timestamp:" << ir.bc << " " << ir.orbit;
+      ret = 8;
+    }
+  } else if (linkCRU == o2::ctp::GBTLinkIDClassRec) {
+    int32_t BCShiftCorrection = -o2::ctp::TriggerOffsetsParam::Instance().customOffset[o2::detectors::DetID::CTP];
+    int32_t offset = BCShiftCorrection + o2::ctp::TriggerOffsetsParam::Instance().LM_L0 + o2::ctp::TriggerOffsetsParam::Instance().L0_L1_classes - 1;
+    LOG(debug) << "tcr ir ori:" << ir;
+    if ((ir.orbit <= mTFOrbit) && ((int32_t)ir.bc < offset)) {
+      // LOG(warning) << "Loosing tclass:" << ir;
+      mTCRRejected++;
+      return 0;
+    }
+    ir -= offset;
+    LOG(debug) << "tcr ir corrected:" << ir;
+    digit.intRecord = ir;
+    if (digits.count(ir) == 0) {
+      digit.setClassMask(pld);
+      digits[ir] = digit;
+      LOG(debug) << bcid << " class bcid case 0 orbit " << orbit << " pld:" << pld;
+    } else if (digits.count(ir) == 1) {
+      if (digits[ir].CTPClassMask.count() == 0) {
+        digits[ir].setClassMask(pld);
+        LOG(debug) << bcid << " class bcid case 1 orbit " << orbit << " pld:" << pld;
+      } else {
+        if (mErrorTCR < mErrorMax) {
+          LOG(error) << "Two CTP Class masks for same timestamp";
+          mStickyError = true;
+        }
+        mErrorTCR++;
+        ret = 16;
+      }
+    } else {
+      LOG(error) << "Two digits with the same timestamp:" << ir.bc << " " << ir.orbit;
+      ret = 32;
+    }
+  } else {
+    LOG(error) << "Unxpected  CTP CRU link:" << linkCRU;
+  }
+  return ret;
+}
+//
+// Decodes one page
+// It is NOT assumed that CTP HBF has never more than one page.
+// 1 HBF/page <= 8000kB = 8*1024*8/120 = 546 GBT words = 546 IRs/page = 5.5 MHz
+int RawDataDecoder::decodeRaw(o2::framework::InputRecord& inputs, std::vector& filter, o2::pmr::vector& digits, std::vector& lumiPointsHBF1)
+{
+  int ret = 0;
+  static int nwrites = 0;
+  uint64_t countsMBT = 0;
+  uint64_t countsMBV = 0;
+  std::map digitsMap;
+  //
+  // using InputSpec = o2::framework::InputSpec;
+  // using ConcreteDataTypeMatcher = o2::framework::ConcreteDataTypeMatcher;
+  // using Lifetime = o2::framework::Lifetime;
+  o2::framework::DPLRawParser parser(inputs, filter);
+  uint32_t payloadCTP = 0;
+  gbtword80_t remnant = 0;
+  uint32_t size_gbt = 0;
+  mTFOrbit = 0;
+  uint32_t orbit0 = 0;
+  for (auto it = parser.begin(); it != parser.end(); ++it) {
+    const o2::header::RDHAny* rdh = nullptr;
+    try {
+      rdh = reinterpret_cast(it.raw());
+      mPadding = (o2::raw::RDHUtils::getDataFormat(rdh) == 0);
+    } catch (std::exception& e) {
+      LOG(error) << "Failed to extract RDH, abandoning TF sending dummy output, exception was: " << e.what();
+      // dummyOutput();
+      return 1;
+    }
+    // auto triggerOrbit = o2::raw::RDHUtils::getTriggerOrbit(rdh);
+    uint32_t stopBit = o2::raw::RDHUtils::getStop(rdh);
+    uint32_t packetCounter = o2::raw::RDHUtils::getPageCounter(rdh);
+    uint32_t version = o2::raw::RDHUtils::getVersion(rdh);
+    uint32_t rdhOrbit = o2::raw::RDHUtils::getHeartBeatOrbit(rdh);
+    uint32_t triggerType = o2::raw::RDHUtils::getTriggerType(rdh);
+    // std::cout << "diff orbits:" << triggerOrbit - rdhOrbit << std::endl;
+    bool tf = (triggerType & TF_TRIGGERTYPE_MASK) && (packetCounter == 0);
+    bool hb = (triggerType & HB_TRIGGERTYPE_MASK) && (packetCounter == 0);
+    if (tf) {
+      mTFOrbit = rdhOrbit;
+      // std::cout << "tforbit==================>" << mTFOrbit << " " << std::hex << mTFOrbit << std::endl;
+      mTFOrbits.push_back(mTFOrbit);
+    }
+    static bool prt = true;
+    if (prt) {
+      LOG(info) << "RDH version:" << version << " Padding:" << mPadding;
+      prt = false;
+    }
+    auto feeID = o2::raw::RDHUtils::getFEEID(rdh); // 0 = IR, 1 = TCR
+    auto linkCRU = (feeID & 0xf00) >> 8;
+    // LOG(info) << "CRU link:" << linkCRU;
+    if (linkCRU == o2::ctp::GBTLinkIDIntRec) {
+      payloadCTP = o2::ctp::NIntRecPayload;
+    } else if (linkCRU == o2::ctp::GBTLinkIDClassRec) {
+      payloadCTP = o2::ctp::NClassPayload;
+      if (!mDoDigits) { // Do not do TCR if only lumi
+        continue;
+      }
+    } else {
+      LOG(error) << "Unxpected  CTP CRU link:" << linkCRU;
+    }
+    LOG(debug) << "RDH FEEid: " << feeID << " CTP CRU link:" << linkCRU << " Orbit:" << rdhOrbit << " triggerType:" << triggerType;
+    // LOG(info) << "remnant :" << remnant.count();
+    gbtword80_t pldmask = 0;
+    for (uint32_t i = 0; i < payloadCTP; i++) {
+      pldmask[12 + i] = 1;
+    }
+    // std::cout << (orbit0 != rdhOrbit) << " comp " << (mTFOrbit==rdhOrbit) << std::endl;
+    // if(orbit0 != rdhOrbit) {
+    if (hb) {
+      if ((mDoLumi && payloadCTP == o2::ctp::NIntRecPayload) && !tf) { // create lumi per HB
+        lumiPointsHBF1.emplace_back(LumiInfo{rdhOrbit, 0, 0, countsMBT, countsMBV});
+        // std::cout << "hb:" << nhb << " countsMBT:" << countsMBT << std::endl;
+        countsMBT = 0;
+        countsMBV = 0;
+        // nhb++;
+      }
+      remnant = 0;
+      size_gbt = 0;
+      orbit0 = rdhOrbit;
+      // std::cout << "orbit0============>" << std::dec << orbit0 << " " << std::hex << orbit0 << std::endl;
+    }
+    // Create 80 bit words
+    gsl::span payload(it.data(), it.size());
+    gbtword80_t gbtWord80;
+    gbtWord80.set();
+    int wordCount = 0;
+    int wordSize = 10;
+    std::vector gbtwords80;
+    // mPadding = 0;
+    if (mPadding == 1) {
+      wordSize = 16;
+    }
+    // LOG(info) << ii << " payload size:" << payload.size();
+    /* if (payload.size()) {
+      //LOG(info) << "payload size:" << payload.size();
+      // LOG(info) << "RDH FEEid: " << feeID << " CTP CRU link:" << linkCRU << " Orbit:" << triggerOrbit << " stopbit:" << stopBit << " packet:" << packetCounter;
+      // LOGP(info, "RDH FEEid: {} CRU link: {}, Orbit: {}", feeID, linkCRU, triggerOrbit);
+      std::cout << std::hex << "RDH FEEid: " << feeID << " CTP CRU link:" << linkCRU << " Orbit:" << rdhOrbit << std::endl;
+    } */
+    gbtword80_t bcmask = std::bitset<80>("111111111111");
+    for (auto payloadWord : payload) {
+      int wc = wordCount % wordSize;
+      // LOG(info) << wordCount << ":" << wc << " payload:" << int(payloadWord);
+      if ((wc == 0) && (wordCount != 0)) {
+        if (gbtWord80.count() != 80) {
+          gbtwords80.push_back(gbtWord80);
+        }
+        gbtWord80.set();
+      }
+      if (wc < 10) {
+        for (int i = 0; i < 8; i++) {
+          gbtWord80[wc * 8 + i] = bool(int(payloadWord) & (1 << i));
+        }
+      }
+      wordCount++;
+    }
+    if ((gbtWord80.count() != 80) && (gbtWord80.count() > 0)) {
+      gbtwords80.push_back(gbtWord80);
+    }
+    // decode 80 bits payload
+    for (auto word : gbtwords80) {
+      std::vector diglets;
+      gbtword80_t gbtWord = word;
+      makeGBTWordInverse(diglets, gbtWord, remnant, size_gbt, payloadCTP);
+      for (auto diglet : diglets) {
+        if (mDoLumi && payloadCTP == o2::ctp::NIntRecPayload) {
+          gbtword80_t pld = (diglet >> 12) & mTVXMask;
+          if (pld.count() != 0) {
+            countsMBT++;
+          }
+          pld = (diglet >> 12) & mVBAMask;
+          if (pld.count() != 0) {
+            countsMBV++;
+          }
+        }
+        if (!mDoDigits) {
+          continue;
+        }
+        LOG(debug) << "diglet:" << diglet << " " << (diglet & bcmask).to_ullong();
+        ret = addCTPDigit(linkCRU, rdhOrbit, diglet, pldmask, digitsMap);
+      }
+    }
+    if ((remnant.count() > 0) && stopBit) {
+      if (mDoLumi && payloadCTP == o2::ctp::NIntRecPayload) {
+        gbtword80_t pld = (remnant >> 12) & mTVXMask;
+        if (pld.count() != 0) {
+          countsMBT++;
+        }
+        pld = (remnant >> 12) & mVBAMask;
+        if (pld.count() != 0) {
+          countsMBV++;
+        }
+      }
+      if (!mDoDigits) {
+        continue;
+      }
+      ret = addCTPDigit(linkCRU, rdhOrbit, remnant, pldmask, digitsMap);
+      LOG(debug) << "diglet:" << remnant << " " << (remnant & bcmask).to_ullong();
+      remnant = 0;
+    }
+  }
+  if (mDoLumi) {
+    lumiPointsHBF1.emplace_back(LumiInfo{orbit0, 0, 0, countsMBT, countsMBV});
+    // std::cout << "last lumi:" << nhb  << std::endl;
+  }
+  if (mDoDigits & mDecodeInps) {
+    uint64_t trgclassmask = 0xffffffffffffffff;
+    uint64_t trgclassmaskNOTRGDet = 0xffffffffffffffff;
+    if (mCTPConfig.getRunNumber() != 0) {
+      trgclassmask = mCTPConfig.getTriggerClassMaskWInputs(); // classes triggered by internal ctp generators not here
+      trgclassmaskNOTRGDet = mCTPConfig.getTriggerClassMaskWInputsNoTrgDets();
+      // mCTPConfig.printStream(std::cout);
+    }
+    // std::cout << "trgclassmask:" << std::hex << trgclassmask << std::dec << std::endl;
+    mLostDueToShiftInps += shiftInputs(digitsMap, digits, mTFOrbit);
+    if (mCheckConsistency) {
+      ret = checkReadoutConsistentncy(digits, trgclassmask, trgclassmaskNOTRGDet);
+    }
+  }
+  if (mDoDigits && !mDecodeInps) {
+    for (auto const& dig : digitsMap) {
+      digits.push_back(dig.second);
+    }
+  }
+  // ret = 1;
+  if (mStickyError) {
+    if (nwrites < mErrorMax) {
+      std::string file = "dumpCTP" + std::to_string(nwrites) + ".bin";
+      std::ofstream dumpctp(file.c_str(), std::ios::out | std::ios::binary);
+      if (!dumpctp.good()) {
+        LOGP(error, "Failed to open file {}", file);
+      } else {
+        LOGP(info, "CTP dump file open {}", file);
+        for (auto it = parser.begin(); it != parser.end(); ++it) {
+          char* dataout = (char*)(it.raw());
+          dumpctp.write(dataout, it.sizeTotal());
+        }
+        dumpctp.close();
+      }
+      nwrites++;
+    }
+    // LOG(error) << "CTP decoding IR errors:" << mErrorIR << " TCR errors:" << mErrorTCR;
+  }
+  return ret;
+}
+//
+int RawDataDecoder::decodeRawFatal(o2::framework::InputRecord& inputs, std::vector& filter)
+{
+  o2::framework::DPLRawParser parser(inputs, filter);
+  uint32_t payloadCTP = 0;
+  gbtword80_t remnant = 0;
+  uint32_t size_gbt = 0;
+  mTFOrbit = 0;
+  uint32_t orbit0 = 0;
+  std::array rates{};
+  std::array ratesC{};
+  for (auto it = parser.begin(); it != parser.end(); ++it) {
+    const o2::header::RDHAny* rdh = nullptr;
+    try {
+      rdh = reinterpret_cast(it.raw());
+      mPadding = (o2::raw::RDHUtils::getDataFormat(rdh) == 0);
+    } catch (std::exception& e) {
+      LOG(error) << "Failed to extract RDH, abandoning TF sending dummy output, exception was: " << e.what();
+      // dummyOutput();
+      return 1;
+    }
+    // auto triggerOrbit = o2::raw::RDHUtils::getTriggerOrbit(rdh);
+    uint32_t stopBit = o2::raw::RDHUtils::getStop(rdh);
+    uint32_t packetCounter = o2::raw::RDHUtils::getPageCounter(rdh);
+    uint32_t version = o2::raw::RDHUtils::getVersion(rdh);
+    uint32_t rdhOrbit = o2::raw::RDHUtils::getHeartBeatOrbit(rdh);
+    uint32_t triggerType = o2::raw::RDHUtils::getTriggerType(rdh);
+    // std::cout << "diff orbits:" << triggerOrbit - rdhOrbit << std::endl;
+    bool tf = (triggerType & TF_TRIGGERTYPE_MASK) && (packetCounter == 0);
+    bool hb = (triggerType & HB_TRIGGERTYPE_MASK) && (packetCounter == 0);
+    if (tf) {
+      mTFOrbit = rdhOrbit;
+      // std::cout << "tforbit==================>" << mTFOrbit << " " << std::hex << mTFOrbit << std::endl;
+      mTFOrbits.push_back(mTFOrbit);
+    }
+    static bool prt = true;
+    if (prt) {
+      LOG(info) << "RDH version:" << version << " Padding:" << mPadding;
+      prt = false;
+    }
+    auto feeID = o2::raw::RDHUtils::getFEEID(rdh); // 0 = IR, 1 = TCR
+    auto linkCRU = (feeID & 0xf00) >> 8;
+    // LOG(info) << "CRU link:" << linkCRU;
+    if (linkCRU == o2::ctp::GBTLinkIDIntRec) {
+      payloadCTP = o2::ctp::NIntRecPayload;
+    } else if (linkCRU == o2::ctp::GBTLinkIDClassRec) {
+      payloadCTP = o2::ctp::NClassPayload;
+    } else {
+      LOG(error) << "Unxpected  CTP CRU link:" << linkCRU;
+    }
+    LOG(debug) << "RDH FEEid: " << feeID << " CTP CRU link:" << linkCRU << " Orbit:" << rdhOrbit << " triggerType:" << triggerType;
+    // LOG(info) << "remnant :" << remnant.count();
+    gbtword80_t pldmask = 0;
+    for (uint32_t i = 0; i < payloadCTP; i++) {
+      pldmask[12 + i] = 1;
+    }
+    // std::cout << (orbit0 != rdhOrbit) << " comp " << (mTFOrbit==rdhOrbit) << std::endl;
+    // if(orbit0 != rdhOrbit) {
+    if (hb) {
+      remnant = 0;
+      size_gbt = 0;
+      orbit0 = rdhOrbit;
+      // std::cout << "orbit0============>" << std::dec << orbit0 << " " << std::hex << orbit0 << std::endl;
+    }
+    // Create 80 bit words
+    gsl::span payload(it.data(), it.size());
+    gbtword80_t gbtWord80;
+    gbtWord80.set();
+    int wordCount = 0;
+    int wordSize = 10;
+    std::vector gbtwords80;
+    // mPadding = 0;
+    if (mPadding == 1) {
+      wordSize = 16;
+    }
+    // LOG(info) << ii << " payload size:" << payload.size();
+    gbtword80_t bcmask = std::bitset<80>("111111111111");
+    for (auto payloadWord : payload) {
+      int wc = wordCount % wordSize;
+      // LOG(info) << wordCount << ":" << wc << " payload:" << int(payloadWord);
+      if ((wc == 0) && (wordCount != 0)) {
+        if (gbtWord80.count() != 80) {
+          gbtwords80.push_back(gbtWord80);
+        }
+        gbtWord80.set();
+      }
+      if (wc < 10) {
+        for (int i = 0; i < 8; i++) {
+          gbtWord80[wc * 8 + i] = bool(int(payloadWord) & (1 << i));
+        }
+      }
+      wordCount++;
+    }
+    if ((gbtWord80.count() != 80) && (gbtWord80.count() > 0)) {
+      gbtwords80.push_back(gbtWord80);
+    }
+    // decode 80 bits payload
+    for (auto word : gbtwords80) {
+      std::vector diglets;
+      gbtword80_t gbtWord = word;
+      makeGBTWordInverse(diglets, gbtWord, remnant, size_gbt, payloadCTP);
+      for (auto diglet : diglets) {
+        int nbits = payloadCTP - 12;
+        for (int i = 0; i < nbits; i++) {
+          gbtword80_t mask = 1ull << i;
+          gbtword80_t pld = (diglet >> 12) & mask;
+          // LOG(info) << "diglet:" << diglet << " pld:" << pld;
+          if (pld.count() != 0) {
+            if (linkCRU == o2::ctp::GBTLinkIDIntRec) {
+              rates[i]++;
+            } else {
+              ratesC[i]++;
+            }
+          }
+        }
+        // LOG(debug) << "diglet:" << diglet << " " << (diglet & bcmask).to_ullong();
+      }
+    }
+    if ((remnant.count() > 0) && stopBit) {
+      int nbits = payloadCTP - 12;
+      for (int i = 0; i < nbits; i++) {
+        gbtword80_t mask = 1ull << i;
+        gbtword80_t pld = (remnant >> 12) & mask;
+        // LOG(info) << "diglet:" << remnant << " pld:" << pld;
+        if (pld.count() != 0) {
+          if (linkCRU == o2::ctp::GBTLinkIDIntRec) {
+            rates[i]++;
+          } else {
+            ratesC[i]++;
+          }
+        }
+      }
+      remnant = 0;
+    }
+  }
+  // print max rates
+  std::map ratesmap;
+  std::map ratesmapC;
+  for (int i = 0; i < o2::ctp::CTP_NCLASSES; i++) {
+    if (rates[i]) {
+      ratesmap[rates[i]] = i;
+    }
+    if (ratesC[i]) {
+      ratesmapC[ratesC[i]] = i;
+    }
+  }
+  auto nhb = o2::base::GRPGeomHelper::getNHBFPerTF();
+  std::string message = "Ringing inputs [MHz]:";
+  for (auto const& r : boost::adaptors::reverse(ratesmap)) {
+    // LOG(error) << r.second;
+    message += " " + o2::ctp::CTPInputsConfiguration::getInputNameFromIndex(r.second + 1) + ":" + std::to_string(r.first / 32. / o2::constants::lhc::LHCOrbitMUS);
+  }
+  std::string messageC = "Ringing classes [MHz]:";
+  for (auto const& r : boost::adaptors::reverse(ratesmapC)) {
+    messageC += " class" + std::to_string(r.second) + ":" + std::to_string(r.first / 32. / o2::constants::lhc::LHCOrbitMUS);
+  }
+  LOG(error) << messageC;
+  LOG(fatal) << message;
+  return 0;
+}
+//
+int RawDataDecoder::decodeRaw(o2::framework::InputRecord& inputs, std::vector& filter, std::vector& digits, std::vector& lumiPointsHBF1)
+{
+  o2::pmr::vector pmrdigits;
+  int ret = decodeRaw(inputs, filter, pmrdigits, lumiPointsHBF1);
+  for (auto const d : pmrdigits) {
+    digits.push_back(d);
+  }
+  return ret;
+}
+//
+// Not to be called with level LM
+// Keeping shift in params if needed to be generalised
+int RawDataDecoder::shiftNew(const o2::InteractionRecord& irin, uint32_t TFOrbit, std::bitset<48>& inpmask, int64_t shift, int level, std::map& digmap)
+{
+  //
+  if (irin.orbit > TFOrbit || irin.bc >= shift) {
+    auto lxmask = L0MASKInputs;
+    if (level == 1) {
+      lxmask = L1MASKInputs;
+    }
+    auto ir = irin - shift; // add L0 to prev digit
+    if (digmap.count(ir)) {
+      if ((digmap[ir].CTPInputMask & lxmask).count()) {
+        LOG(error) << " Overwriting LX ? X:" << level;
+      }
+      digmap[ir].CTPInputMask = digmap[ir].CTPInputMask | (inpmask & lxmask);
+    } else {
+      CTPDigit digit = {ir, inpmask & lxmask, 0};
+      digmap[ir] = digit;
+    }
+  } else {
+    // LOG(info) << "LOST:" << irin << " shift:" << shift;
+    return 1;
+    ;
+  }
+  return 0;
+}
+//
+
+int RawDataDecoder::shiftInputs(std::map& digitsMap, o2::pmr::vector& digits, uint32_t TFOrbit, uint64_t trgclassmask)
+{
+  // int nClasswoInp = 0; // counting classes without input which should never happen
+  int lost = 0;
+  std::map digitsMapShifted;
+  auto L0shift = o2::ctp::TriggerOffsetsParam::Instance().LM_L0;
+  auto L1shift = L0shift + o2::ctp::TriggerOffsetsParam::Instance().L0_L1;
+  for (auto const& dig : digitsMap) {
+    auto inpmask = dig.second.CTPInputMask;
+    auto inpmaskLM = inpmask & LMMASKInputs;
+    auto inpmaskL0 = inpmask & L0MASKInputs;
+    auto inpmaskL1 = inpmask & L1MASKInputs;
+    int lm = inpmaskLM.count() > 0;
+    int l0 = inpmaskL0.count() > 0;
+    int l1 = inpmaskL1.count() > 0;
+    int lut = lm + (l0 << 1) + (l1 << 2);
+    // std::cout << "L0mask:" << L0MASKInputs << std::endl;
+    // std::cout << "L0:" << inpmaskL0 << std::endl;
+    // std::cout << "L1:" << inpmaskL1 << std::endl;
+    if (lut == 0 || lut == 1) { // no inps or LM
+      digitsMapShifted[dig.first] = dig.second;
+    } else if (lut == 2) { // L0
+      lost += shiftNew(dig.first, TFOrbit, inpmask, L0shift, 0, digitsMapShifted);
+      if (dig.second.CTPClassMask.count()) {
+        // LOG(error) << "Adding class mask without input ?";
+        //  This is not needed as it can happen; Full checj done below - see next LOG(error)
+        CTPDigit digi = {dig.first, 0, dig.second.CTPClassMask};
+        digitsMapShifted[dig.first] = digi;
+      }
+    } else if (lut == 4) { // L1
+      lost += shiftNew(dig.first, TFOrbit, inpmask, L1shift, 1, digitsMapShifted);
+      if (dig.second.CTPClassMask.count()) {
+        CTPDigit digi = {dig.first, 0, dig.second.CTPClassMask};
+        digitsMapShifted[dig.first] = digi;
+      }
+    } else if (lut == 6) { // L0 and L1
+      lost += shiftNew(dig.first, TFOrbit, inpmask, L0shift, 0, digitsMapShifted);
+      lost += shiftNew(dig.first, TFOrbit, inpmask, L1shift, 1, digitsMapShifted);
+      if (dig.second.CTPClassMask.count()) {
+        CTPDigit digi = {dig.first, 0, dig.second.CTPClassMask};
+        digitsMapShifted[dig.first] = digi;
+      }
+    } else if (lut == 3) { // LM and L0
+      lost += shiftNew(dig.first, TFOrbit, inpmask, L0shift, 0, digitsMapShifted);
+      CTPDigit digi = {dig.first, inpmask & (~L0MASKInputs), dig.second.CTPClassMask};
+      // if LM level do not need to add class as LM is not shifted;
+      digitsMapShifted[dig.first] = digi;
+    } else if (lut == 5) { // LM and L1
+      lost += shiftNew(dig.first, TFOrbit, inpmask, L1shift, 1, digitsMapShifted);
+      CTPDigit digi = {dig.first, inpmask & (~L1MASKInputs), dig.second.CTPClassMask};
+      digitsMapShifted[dig.first] = digi;
+    } else if (lut == 7) { // LM and L0 and L1
+      lost += shiftNew(dig.first, TFOrbit, inpmask, L0shift, 0, digitsMapShifted);
+      lost += shiftNew(dig.first, TFOrbit, inpmask, L1shift, 1, digitsMapShifted);
+      CTPDigit digi = {dig.first, inpmaskLM, dig.second.CTPClassMask};
+      digitsMapShifted[dig.first] = digi;
+    } else {
+      LOG(fatal) << "lut = " << lut;
+    }
+  }
+  for (auto const& dig : digitsMapShifted) {
+    digits.push_back(dig.second);
+  }
+  return lost;
+}
+//
+int RawDataDecoder::checkReadoutConsistentncy(o2::pmr::vector& digits, uint64_t trgclassmask, uint64_t trgclassmaskNoTrgDet)
+{
+  LOG(debug) << "Checking readout";
+  int ret = 0;
+  static int nerror = 0;
+  for (auto const& digit : digits) {
+    // if class mask => inps
+    for (int i = 0; i < digit.CTPClassMask.size(); i++) {
+      bool trgcls = trgclassmask & (1ull << i);
+      if (digit.CTPClassMask[i] & trgcls) {
+        const CTPClass* cls = mCTPConfig.getCTPClassFromHWIndex(i);
+        if (cls == nullptr) {
+          if (nerror < mErrorMax) {
+            LOG(error) << "Class mask index not found in CTP config:" << i;
+            nerror++;
+          }
+          ret = 128;
+          continue;
+        }
+        mClassCountersA[i]++;
+        if (cls->descriptor == nullptr) {
+          continue;
+        }
+        uint64_t clsinpmask = cls->descriptor->getInputsMask();
+        uint64_t diginpmask = digit.CTPInputMask.to_ullong();
+        if (!((clsinpmask & diginpmask) == clsinpmask)) {
+          if (nerror < mErrorMax) {
+            LOG(error) << "Cls=>Inps: CTP class:" << cls->name << " inpmask:" << clsinpmask << " not compatible with inputs mask:" << diginpmask;
+            nerror++;
+          }
+          mClassErrorsA[i]++;
+          ret = 128;
+        }
+      }
+    }
+    // if inps => class mask
+    for (auto const& cls : mCTPConfig.getCTPClasses()) {
+      // cls.printStream(std::cout);
+      if (cls.descriptor == nullptr) {
+        continue;
+      }
+      uint64_t clsinpmask = cls.descriptor->getInputsMask(); // class definition
+      uint64_t diginpmask = digit.CTPInputMask.to_ullong();
+      uint64_t digclsmask = digit.CTPClassMask.to_ullong();
+      if ((clsinpmask & diginpmask) == clsinpmask) {
+        if (cls.classMask & trgclassmask) {
+          mClassCountersB[cls.getIndex()]++;
+          if ((cls.classMask & digclsmask) == 0) {
+            int32_t BCShiftCorrection = -o2::ctp::TriggerOffsetsParam::Instance().customOffset[o2::detectors::DetID::CTP];
+            int32_t offset = BCShiftCorrection + o2::ctp::TriggerOffsetsParam::Instance().LM_L0 + o2::ctp::TriggerOffsetsParam::Instance().L0_L1_classes - 1;
+            offset = o2::constants::lhc::LHCMaxBunches - offset;
+            if (digit.intRecord.bc < offset) {
+              if ((nerror < mErrorMax) && (cls.classMask & ~trgclassmaskNoTrgDet)) {
+                LOG(info) << "Inp=>Cls: CTP class:" << cls.name << " inpmask:" << clsinpmask << " cls mask:" << cls.classMask << " not found in digit:" << digit;
+                nerror++;
+              }
+              mClassErrorsB[cls.getIndex()]++;
+              ret = 256;
+            } else {
+              mLostDueToShiftCC++;
+            }
+          }
+        }
+      }
+    }
+  }
+  return ret;
+}
+//
+int RawDataDecoder::setLumiInp(int lumiinp, std::string inp)
+{
+  // check if valid input
+  int index = o2::ctp::CTPInputsConfiguration::getInputIndexFromName(inp);
+  if (index == 0xff) {
+    LOG(fatal) << "CTP raw decoder: input index not found:" << inp;
+    return 0xff;
+  }
+  if (lumiinp == 1) {
+    mTVXMask.reset();
+    mTVXMask[index - 1] = true;
+  } else {
+    mVBAMask.reset();
+    mVBAMask[index - 1] = true;
+  }
+  return index;
+}
+//
+int RawDataDecoder::init()
+{
+  return 0;
+}
diff --git a/Detectors/CTP/simulation/include/CTPSimulation/Digitizer.h b/Detectors/CTP/simulation/include/CTPSimulation/Digitizer.h
index 68202c8d22996..7d29f2144c2b3 100644
--- a/Detectors/CTP/simulation/include/CTPSimulation/Digitizer.h
+++ b/Detectors/CTP/simulation/include/CTPSimulation/Digitizer.h
@@ -34,7 +34,10 @@ class Digitizer
   void setCCDBServer(const std::string& server) { mCCDBServer = server; }
   std::vector process(const gsl::span detinputs);
   void calculateClassMask(const std::bitset ctpinpmask, std::bitset& classmask);
+  void setCTPConfiguration(o2::ctp::CTPConfiguration* config);
+  o2::ctp::CTPConfiguration* getDefaultCTPConfiguration();
   void init();
+
  private:
   // CTP configuration
   std::string mCCDBServer = o2::base::NameConf::getCCDBServer();
@@ -43,4 +46,4 @@ class Digitizer
 };
 } // namespace ctp
 } // namespace o2
-#endif //ALICEO2_CTP_DIGITIZER_H
+#endif // ALICEO2_CTP_DIGITIZER_H
diff --git a/Detectors/CTP/simulation/include/CTPSimulation/Digits2Raw.h b/Detectors/CTP/simulation/include/CTPSimulation/Digits2Raw.h
index 0bdba7736825c..9448c5202b89c 100644
--- a/Detectors/CTP/simulation/include/CTPSimulation/Digits2Raw.h
+++ b/Detectors/CTP/simulation/include/CTPSimulation/Digits2Raw.h
@@ -41,6 +41,7 @@ class Digits2Raw
   void setBoardId(uint32_t boardid) { mBoardId = boardid; }
   void setZeroSuppressedIntRec(bool value) { mZeroSuppressedIntRec = value; }
   void setZeroSuppressedClassRec(bool value) { mZeroSuppressedClassRec = value; }
+  void setPadding(bool value) { mPadding = value; }
   bool getFilePerLink() const { return mOutputPerLink; }
   uint64_t getFEEIDIR() const { return uint64_t(mBoardId + (o2::ctp::GBTLinkIDIntRec << 8)); }
   uint64_t getFEEIDTC() const { return uint64_t(mBoardId + (o2::ctp::GBTLinkIDClassRec << 8)); }
@@ -71,8 +72,9 @@ class Digits2Raw
   //const uint32_t mGBTLinkTC = 1; // Trigger Class Record CTP GBT link
   //const uint32_t mGBTLinkMisc = 2; // HBrecord, Counters, ...
   uint32_t mBoardId = 33;
-  bool mZeroSuppressedIntRec = false;
+  bool mZeroSuppressedIntRec = true;
   bool mZeroSuppressedClassRec = true;
+  bool mPadding = true;
   //constexpr uint32_t CTPCRULinkIDMisc = 2;
   std::string mCTPRawDataFileName = "CTP_alio2-cr1-flp163_cru1111_0";
 };
diff --git a/Detectors/CTP/simulation/src/Digitizer.cxx b/Detectors/CTP/simulation/src/Digitizer.cxx
index 2980537e201bb..b1d4ef40b7b0e 100644
--- a/Detectors/CTP/simulation/src/Digitizer.cxx
+++ b/Detectors/CTP/simulation/src/Digitizer.cxx
@@ -15,79 +15,127 @@
 #include "CTPSimulation/Digitizer.h"
 #include "TRandom.h"
 #include 
-#include "FairLogger.h"
+#include 
 #include 
 
+#include 
+
 using namespace o2::ctp;
 
 ClassImp(Digitizer);
 // Trigger detector config needed here.
 std::vector Digitizer::process(const gsl::span detinputs)
 {
+  if (!mCTPConfiguration) {
+    setCTPConfiguration(getDefaultCTPConfiguration());
+  }
   std::map> det2ctpinp = mCTPConfiguration->getDet2InputMap();
   // To be taken from config database ?
   std::map detInputName2Mask =
-    //{{"MFV0MB", 1}, {"MFV0MBInner", 2}, {"MFV0MBOuter", 4}, {"MFV0HM", 8}, {"MFT0A", 1}, {"MFT0C", 2}, {"MFT0Vertex", 4}, {"MFT0Cent", 8}, {"MFT0SemiCent", 0x10}};
-    {{"MVBA", 1}, {"MVIR", 0x10}, {"MVOR", 2}, {"MVNC", 4}, {"MVCH", 8}, {"MT0A", 1}, {"MT0C", 2}, {"MTVX", 0x10}, {"MTCE", 8}, {"MTSC", 0x4}, {"0U0A", 1}, {"0U0C", 2}, {"0UVX", 0x10}, {"0UCE", 8}, {"0USC", 0x4}};
+    {{"MVBA", 1}, {"MVOR", 2}, {"MVNC", 4}, {"MVCH", 8}, {"MVIR", 0x10}, {"MT0A", 1}, {"MT0C", 2}, {"MTSC", 4}, {"MTCE", 8}, {"MTVX", 0x10}, {"0U0A", 1}, {"0U0C", 2}, {"0USC", 4}, {"0UCE", 8}, {"0UVX", 0x10}, {"EMBA", 0x1}, {"0EMC", 0x2}, {"0DMC", 0x4}};
+
+  // pre-sorting detector inputs per interaction record
   std::map> predigits;
   for (auto const& inp : detinputs) {
     predigits[inp.intRecord].push_back(&inp);
   }
+
   std::vector digits;
   for (auto const& hits : predigits) {
-    CTPDigit data;
-    data.intRecord = hits.first;
     std::bitset inpmaskcoll = 0;
+    auto currentIR = hits.first;
     for (auto const inp : hits.second) {
       switch (inp->detector) {
         case o2::detectors::DetID::FT0: {
           // see dummy database above
           for (auto const& ctpinp : det2ctpinp[o2::detectors::DetID::FT0]) {
             uint64_t mask = (inp->inputsMask).to_ullong() & detInputName2Mask[ctpinp.name];
-            inpmaskcoll |= std::bitset(ctpinp.inputMask);
+            if (mask) {
+              inpmaskcoll |= std::bitset(ctpinp.inputMask);
+            }
           }
           break;
         }
         case o2::detectors::DetID::FV0: {
           for (auto const& ctpinp : det2ctpinp[o2::detectors::DetID::FV0]) {
             uint64_t mask = (inp->inputsMask).to_ullong() & detInputName2Mask[ctpinp.name];
-            inpmaskcoll |= std::bitset(ctpinp.inputMask);
+            if (mask) {
+              inpmaskcoll |= std::bitset(ctpinp.inputMask);
+            }
           }
           break;
         }
         case o2::detectors::DetID::FDD: {
           for (auto const& ctpinp : det2ctpinp[o2::detectors::DetID::FDD]) {
             uint64_t mask = (inp->inputsMask).to_ullong() & detInputName2Mask[ctpinp.name];
-            inpmaskcoll |= std::bitset(ctpinp.inputMask);
+            if (mask) {
+              inpmaskcoll |= std::bitset(ctpinp.inputMask);
+            }
           }
+          break;
         }
         case o2::detectors::DetID::EMC: {
+          // uint64_t inpmaskdebug = 1;
+          uint64_t inpmaskdebug = (inp->inputsMask).to_ullong();
+          if (inpmaskdebug & detInputName2Mask["EMBA"]) {
+            // MB-accept must be treated separately, as it is not a CTP input
+            std::bitset emcMBaccept;
+            emcMBaccept.set(CTP_NINPUTS - 1, 1);
+            inpmaskcoll |= emcMBaccept;
+          } // else { // needs to be done always, remove else
           for (auto const& ctpinp : det2ctpinp[o2::detectors::DetID::EMC]) {
-            uint64_t mask = (inp->inputsMask).to_ullong() & detInputName2Mask[ctpinp.name];
-            inpmaskcoll |= std::bitset(ctpinp.inputMask);
+            uint64_t mask = inpmaskdebug & detInputName2Mask[ctpinp.name];
+            // uint64_t mask = (inp->inputsMask).to_ullong() & detInputName2Mask[ctpinp.name];
+            if (mask) {
+              inpmaskcoll |= std::bitset(ctpinp.inputMask);
+            }
           }
+          // }
+          // LOG(info) << "EMC input mask:" << inpmaskcoll << " with IR = " << currentIR.bc << ", orbit = " << currentIR.orbit;
+          break;
         }
         case o2::detectors::DetID::PHS: {
           for (auto const& ctpinp : det2ctpinp[o2::detectors::DetID::PHS]) {
             uint64_t mask = (inp->inputsMask).to_ullong() & detInputName2Mask[ctpinp.name];
-            inpmaskcoll |= std::bitset(ctpinp.inputMask);
+            if (mask) {
+              inpmaskcoll |= std::bitset(ctpinp.inputMask);
+            }
           }
+          break;
         }
         case o2::detectors::DetID::ZDC: {
           for (auto const& ctpinp : det2ctpinp[o2::detectors::DetID::ZDC]) {
             uint64_t mask = (inp->inputsMask).to_ullong() & detInputName2Mask[ctpinp.name];
-            inpmaskcoll |= std::bitset(ctpinp.inputMask);
+            if (mask) {
+              inpmaskcoll |= std::bitset(ctpinp.inputMask);
+            }
           }
+          break;
         }
         default:
           // Error
           LOG(error) << "CTP Digitizer: unknown detector:" << inp->detector;
           break;
       }
+      // inpmaskcoll.reset();  // debug
+      // inpmaskcoll[47] = 1;  // debug
+    } // end loop over trigger input for this interaction
+    if (inpmaskcoll.to_ullong()) {
+      // we put the trigger only when non-trivial
+      std::bitset<64> classmask;
+      calculateClassMask(inpmaskcoll, classmask);
+      if (classmask.to_ulong() == 0) {
+        // No class accepted
+        continue;
+      }
+      CTPDigit data;
+      data.intRecord = hits.first;
+      data.CTPInputMask = inpmaskcoll;
+      data.CTPClassMask = classmask;
+      digits.emplace_back(data);
+      LOG(info) << "Trigger-Event " << data.intRecord.bc << " " << data.intRecord.orbit << " Input mask:" << inpmaskcoll << " with IR = " << data.intRecord.bc << ", orbit = " << data.intRecord.orbit;
+      // LOG(info) << "Trigger-Event " << data.intRecord.bc << " " << data.intRecord.orbit << " Class mask:" << data.CTPClassMask << " with IR = " << data.intRecord.bc << ", orbit = " << data.intRecord.orbit;
     }
-    data.CTPInputMask = inpmaskcoll;
-    calculateClassMask(inpmaskcoll, data.CTPClassMask);
-    digits.emplace_back(data);
   }
   return std::move(digits);
 }
@@ -95,12 +143,48 @@ void Digitizer::calculateClassMask(const std::bitset ctpinpmask, st
 {
   classmask = 0;
   for (auto const& tcl : mCTPConfiguration->getCTPClasses()) {
-    if (tcl.descriptor->getInputsMask() & ctpinpmask.to_ullong()) {
-      classmask |= (1 << tcl.classMask);
+    auto clustername = boost::algorithm::to_lower_copy(tcl.cluster->name); // make case-insensitive for string comparison
+    if (clustername == "emc") {
+      // check if Min Bias EMC class
+      bool tvxMBemc = tcl.name.find("C0TVX-B-NOPF-EMC") != std::string::npos; // 2023
+      tvxMBemc |= tcl.name.find("C0TVX-A-NOPF-EMC") != std::string::npos;
+      tvxMBemc |= tcl.name.find("C0TVX-C-NOPF-EMC") != std::string::npos;
+      tvxMBemc |= tcl.name.find("C0TVX-E-NOPF-EMC") != std::string::npos;
+      if (clustername == "emc") {
+        tvxMBemc |= tcl.name.find("minbias_TVX_L0") != std::string::npos; // 2022
+      }
+      // require real physics input in any case
+      if (tvxMBemc) {
+        // if the class is a min. bias class accept it only if the MB-accept bit is set in addition
+        // (fake trigger input)
+        if (ctpinpmask[CTP_NINPUTS - 1]) {
+          classmask |= tcl.classMask;
+          LOG(info) << "adding MBA:" << tcl.name;
+        }
+      } else if ((ctpinpmask.to_ullong() & tcl.descriptor->getInputsMask()) == tcl.descriptor->getInputsMask()) {
+        // EMCAL rare triggers - physical trigger input
+        // class identification can be handled like in the case of the other
+        // classes as EMCAL trigger input is required
+        LOG(info) << "adding EMCal rare trigger:" << tcl.name;
+        classmask |= tcl.classMask;
+      }
+    } else {
+      if (tcl.descriptor && ((ctpinpmask.to_ullong() & tcl.descriptor->getInputsMask()) == tcl.descriptor->getInputsMask())) {
+        classmask |= tcl.classMask;
+      }
     }
   }
+  LOG(info) << "input mask:" << ctpinpmask;
+  LOG(info) << "class mask:" << classmask;
 }
-void Digitizer::init()
+
+void Digitizer::setCTPConfiguration(o2::ctp::CTPConfiguration* config)
+{
+  mCTPConfiguration = config;
+  LOG(info) << *mCTPConfiguration;
+}
+
+o2::ctp::CTPConfiguration* Digitizer::getDefaultCTPConfiguration()
 {
   // CTP Configuration
   if (mCCDBServer.empty()) {
@@ -110,8 +194,14 @@ void Digitizer::init()
   }
   auto& mgr = o2::ccdb::BasicCCDBManager::instance();
   mgr.setURL(mCCDBServer);
-  map metadata = {};
+  std::map metadata = {};
   long timestamp = 1546300800000;
-  mCTPConfiguration = mgr.getSpecific(o2::ctp::CCDBPathCTPConfig, timestamp, metadata);
+
+  auto config = mgr.getSpecific(o2::ctp::CCDBPathCTPConfig, timestamp, metadata);
   LOG(info) << " @@@ CTP Digitizer:: CCDB connected " << std::endl;
+  return config;
+}
+
+void Digitizer::init()
+{
 }
diff --git a/Detectors/CTP/simulation/src/Digits2Raw.cxx b/Detectors/CTP/simulation/src/Digits2Raw.cxx
index ab680caf1db09..920e890bf1d75 100644
--- a/Detectors/CTP/simulation/src/Digits2Raw.cxx
+++ b/Detectors/CTP/simulation/src/Digits2Raw.cxx
@@ -13,7 +13,7 @@
 /// \author Roman Lietava
 
 #include "CTPSimulation/Digits2Raw.h"
-#include "FairLogger.h"
+#include 
 #include "CommonUtils/StringUtils.h"
 
 using namespace o2::ctp;
@@ -27,7 +27,6 @@ using namespace o2::ctp;
 
 void Digits2Raw::init()
 {
-
   //
   // Register links
   //
@@ -36,6 +35,8 @@ void Digits2Raw::init()
     outd += '/';
   }
   LOG(info) << "Raw outpud dir:" << mOutDir;
+  //
+  LOG(info) << "Raw Data padding:" << mPadding;
   // Interaction Record
   int ilink = 0;
   uint64_t feeID = getFEEIDIR();
@@ -56,7 +57,7 @@ void Digits2Raw::processDigits(const std::string& fileDigitsName)
     LOG(fatal) << "Failed to open input digits file " << fileDigitsName;
     return;
   }
-  LOG(info) << "Processing digits to raw";
+  LOG(info) << "Processing digits to raw file:" << fileDigitsName;
   TTree* digiTree = (TTree*)digiFile->Get("o2sim");
   if (!digiTree) {
     LOG(fatal) << "Failed to get digits tree";
@@ -78,7 +79,7 @@ void Digits2Raw::processDigits(const std::string& fileDigitsName)
   for (int ient = 0; ient < digiTree->GetEntries(); ient++) {
     digiTree->GetEntry(ient);
     int nbc = CTPDigits.size();
-    LOG(info) << "Entry " << ient << " : " << nbc << " BCs stored";
+    LOG(debug) << "Entry " << ient << " : " << nbc << " BCs stored";
     std::vector hbfIR;
     std::vector hbfTC;
     for (auto const& ctpdig : CTPDigits) {
@@ -93,15 +94,14 @@ void Digits2Raw::processDigits(const std::string& fileDigitsName)
         gbtword80_t gbtdigIR;
         gbtword80_t gbtdigTC;
         digit2GBTdigit(gbtdigIR, gbtdigTC, ctpdig);
-        LOG(debug) << "ir:" << gbtdigIR;
+        LOG(debug) << "ir:" << gbtdigIR << " " << (gbtdigIR.to_ullong() & 0xfff);
         LOG(debug) << "tr:" << gbtdigTC;
         hbfIR.push_back(gbtdigIR);
         hbfTC.push_back(gbtdigTC);
       } else {
         std::vector buffer;
-        LOG(info) << "Packing orbit:" << orbit0;
+        LOG(info) << "Packing orbit:" << orbit0 << " hbfIR:" << hbfIR.size() << " hbfTC:" << hbfTC.size();
         intRec.orbit = orbit0;
-        LOG(info) << "hbfIR:" << hbfIR.size() << " hbfTC:" << hbfTC.size();
         if (mZeroSuppressedIntRec == true) {
           buffer = digits2HBTPayload(hbfIR, NIntRecPayload);
         } else {
@@ -129,10 +129,11 @@ void Digits2Raw::processDigits(const std::string& fileDigitsName)
         hbfIR.push_back(gbtdigIR);
         hbfTC.push_back(gbtdigTC);
       }
+      intRec = ctpdig.intRecord;
     }
     // Last orbit in record
     std::vector buffer;
-    LOG(info) << "Packing orbit:" << orbit0;
+    LOG(info) << "Packing orbit last:" << orbit0;
     intRec.orbit = orbit0;
     if (mZeroSuppressedIntRec == true) {
       buffer = digits2HBTPayload(hbfIR, NIntRecPayload);
@@ -141,12 +142,12 @@ void Digits2Raw::processDigits(const std::string& fileDigitsName)
       buffer = digits2HBTPayload(hbfIRnonZS, NIntRecPayload);
     }
     // add data for IR
-    LOG(debug) << "IR buffer size:" << buffer.size() << ":";
+    LOG(debug) << "IR buffer size:" << buffer.size() << " orbit:" << intRec.orbit;
     mWriter.addData(getFEEIDIR(), mCruID, GBTLinkIDIntRec, mEndPointID, intRec, buffer);
     // add data for Trigger Class Record
     buffer.clear();
     buffer = digits2HBTPayload(hbfTC, NClassPayload);
-    LOG(debug) << "TC buffer size:" << buffer.size() << ":";
+    LOG(debug) << "TC buffer size:" << buffer.size() << " orbit:" << intRec.orbit;
     mWriter.addData(getFEEIDTC(), mCruID, GBTLinkIDClassRec, mEndPointID, intRec, buffer);
     //
     //orbit0 = ctpdig.intRecord.orbit;
@@ -174,49 +175,58 @@ void Digits2Raw::emptyHBFMethod(const header::RDHAny* rdh, std::vector& to
 std::vector Digits2Raw::digits2HBTPayload(const gsl::span digits, uint32_t Npld) const
 {
   std::vector toAdd;
+  int countBytes = 0;
   uint32_t size_gbt = 0;
   gbtword80_t gbtword;
   gbtword80_t gbtsend;
   bool valid;
   for (auto const& dig : digits) {
     valid = makeGBTWord(dig, gbtword, size_gbt, Npld, gbtsend);
+    LOG(debug) << Npld << " digit:" << dig << " " << (dig.to_ulong() & 0xfff) << " ";
+    LOG(debug) << "gbt  :" << gbtsend << " valid:" << valid;
     if (valid == true) {
       for (uint32_t i = 0; i < NGBT; i += 8) {
         uint32_t w = 0;
         for (uint32_t j = 0; j < 8; j++) {
           w += (1 << j) * gbtsend[i + j];
         }
+        countBytes++;
         char c = w;
         toAdd.push_back(c);
       }
-      // Pad zeros up to 128 bits
-      uint32_t NZeros = (o2::raw::RDHUtils::GBTWord * 8 - NGBT) / 8;
-      for (uint32_t i = 0; i < NZeros; i++) {
-        char c = 0;
-        toAdd.push_back(c);
+      if (mPadding) {
+        // Pad zeros up to 128 bits
+        uint32_t NZeros = (o2::raw::RDHUtils::GBTWord128 * 8 - NGBT) / 8;
+        for (uint32_t i = 0; i < NZeros; i++) {
+          char c = 0;
+          toAdd.push_back(c);
+        }
       }
     }
   }
   // add what is left: maybe never left anything - tbc
-  //LOG(info) << size_gbt << " size valid " << valid;
-  //LOG(info) << "gbtword:" << gbtword;
-  //LOG(info) << "gbtsend:" << gbtsend;
+  LOG(debug) << size_gbt << " size valid " << valid;
+  LOG(debug) << "gbtword:" << gbtword;
+  LOG(debug) << "gbtsend:" << gbtsend;
   if (size_gbt > 0) {
-    LOG(debug) << "Adding left over.";
+    // LOG(info) << "Adding left over.";
     gbtword80_t gbtsend = gbtword;
     for (uint32_t i = 0; i < NGBT; i += 8) {
       uint32_t w = 0;
       for (uint32_t j = 0; j < 8; j++) {
         w += (1 << j) * gbtsend[i + j];
       }
+      countBytes++;
       char c = w;
       toAdd.push_back(c);
     }
     // Pad zeros up to 128 bits
-    uint32_t NZeros = (o2::raw::RDHUtils::GBTWord * 8 - NGBT) / 8;
-    for (uint32_t i = 0; i < NZeros; i++) {
-      char c = 0;
-      toAdd.push_back(c);
+    if (mPadding) {
+      uint32_t NZeros = (o2::raw::RDHUtils::GBTWord128 * 8 - NGBT) / 8;
+      for (uint32_t i = 0; i < NZeros; i++) {
+        char c = 0;
+        toAdd.push_back(c);
+      }
     }
   }
   return std::move(toAdd);
@@ -238,11 +248,6 @@ bool Digits2Raw::makeGBTWord(const gbtword80_t& pld, gbtword80_t& gbtword, uint3
     size_gbt = size_gbt + Npld - NGBT;
     valid = true;
   }
-  //printDigit("pld:", pld);
-  //printDigit("gbtword:", gbtword);
-  //std::cout << valid << " ";
-  //printDigit("gbtsend:", gbtsend);
-  //std::cout << gbtsend << std::endl;
   return valid;
 }
 int Digits2Raw::digit2GBTdigit(gbtword80_t& gbtdigitIR, gbtword80_t& gbtdigitTR, const CTPDigit& digit)
diff --git a/Detectors/CTP/simulation/src/digi2raw.cxx b/Detectors/CTP/simulation/src/digi2raw.cxx
index eceb58106f2e8..8aa4a3e12f0c1 100644
--- a/Detectors/CTP/simulation/src/digi2raw.cxx
+++ b/Detectors/CTP/simulation/src/digi2raw.cxx
@@ -27,7 +27,7 @@
 namespace bpo = boost::program_options;
 
 void digi2raw(const std::string& inpName, const std::string& outDir, int verbosity, const std::string& fileForLink, uint32_t rdhV = 4, bool noEmptyHBF = false,
-              bool zsIR = true, bool zsClass = true, int superPageSizeInB = 1024 * 1024);
+              bool zsIR = true, bool zsClass = true, bool enablePadding = true, int cruPageAlignment = 16, int superPageSizeInB = 1024 * 1024);
 
 int main(int argc, char** argv)
 {
@@ -44,13 +44,15 @@ int main(int argc, char** argv)
     add_option("verbosity,v", bpo::value()->default_value(0), "verbosity level");
     //    add_option("input-file,i", bpo::value()->default_value(o2::base::NameConf::getDigitsFileName(o2::detectors::DetID::CTP)),"input CTP digits file"); // why not used?
     add_option("input-file,i", bpo::value()->default_value("ctpdigits.root"), "input CTP digits file");
-    add_option("file-for,f", bpo::value()->default_value("all"), "single file per: all,link,cru");
+    add_option("file-for,f", bpo::value()->default_value("all"), "single file per: all,link,cruendpoint");
     add_option("output-dir,o", bpo::value()->default_value("./"), "output directory for raw data");
     uint32_t defRDH = o2::raw::RDHUtils::getVersion();
     add_option("rdh-version,r", bpo::value()->default_value(defRDH), "RDH version to use");
     add_option("no-empty-hbf,e", bpo::value()->default_value(false)->implicit_value(true), "do not create empty HBF pages (except for HBF starting TF)");
     add_option("no-zs-ir", bpo::value()->default_value(false)->implicit_value(true), "do not zero-suppress interaction records");
     add_option("no-zs-class", bpo::value()->default_value(false)->implicit_value(true), "do not zero-suppress trigger class records");
+    add_option("enable-padding", bpo::value()->default_value(false)->implicit_value(true), "pad raw gbt data to 128 bits: 80 bits payload+48 bits 0");
+    add_option("cru-page-alignment,a", bpo::value()->default_value(16), "CRU page alignment");
     add_option("hbfutils-config,u", bpo::value()->default_value(std::string(o2::base::NameConf::DIGITIZATIONCONFIGFILE)), "config file for HBFUtils (or none)");
     add_option("configKeyValues", bpo::value()->default_value(""), "comma-separated configKeyValues");
 
@@ -85,14 +87,16 @@ int main(int argc, char** argv)
            vm["rdh-version"].as(),
            vm["no-empty-hbf"].as(),
            !vm["no-zs-ir"].as(),
-           !vm["no-zs-class"].as());
+           !vm["no-zs-class"].as(),
+           vm["enable-padding"].as(),
+           vm["cru-page-alignment"].as());
 
   o2::raw::HBFUtils::Instance().print();
 
   return 0;
 }
 
-void digi2raw(const std::string& inpName, const std::string& outDir, int verbosity, const std::string& fileForLink, uint32_t rdhV, bool noEmptyHBF, bool zsIR, bool zsClass, int superPageSizeInB)
+void digi2raw(const std::string& inpName, const std::string& outDir, int verbosity, const std::string& fileForLink, uint32_t rdhV, bool noEmptyHBF, bool zsIR, bool zsClass, bool enablePadding, int cruPageAlignment, int superPageSizeInB)
 {
   TStopwatch swTot;
   swTot.Start();
@@ -106,22 +110,34 @@ void digi2raw(const std::string& inpName, const std::string& outDir, int verbosi
   wr.setSuperPageSize(superPageSizeInB);
   wr.useRDHVersion(rdhV);
   wr.setDontFillEmptyHBF(noEmptyHBF);
-
-  std::string outDirName(outDir);
-  if (outDirName.back() != '/') {
-    outDirName += '/';
+  if (rdhV < 7 && !enablePadding) {
+    enablePadding = true;
+    LOG(info) << "padding is always ON for RDH version " << rdhV;
   }
+  std::string outDirName(outDir);
   // if needed, create output directory
   if (!std::filesystem::exists(outDirName)) {
-    if (!std::filesystem::create_directories(outDirName)) {
-      LOG(fatal) << "could not create output directory " << outDirName;
+    std::error_code ec;
+    if (!std::filesystem::create_directories(outDirName, ec)) {
+      LOG(fatal) << "could not create output directory " << outDirName << ": " << ec.message();
     } else {
       LOG(info) << "created output directory " << outDirName;
     }
   }
+  if (outDirName.back() != '/') {
+    outDirName += '/';
+  }
   m2r.setOutDir(outDirName);
   m2r.setZeroSuppressedIntRec(zsIR);
   m2r.setZeroSuppressedClassRec(zsClass);
+  m2r.getWriter().useRDHDataFormat(enablePadding ? 0 : 2);
+  m2r.setPadding(enablePadding);
+  if (!enablePadding) { // CRU page alignment padding is used only if no GBT word padding is used
+    // m2r.getWriter().setAlignmentSize(o2::ctp::CRUPageAlignment);
+    LOG(info) << "CRU Page Alignment:" << cruPageAlignment;
+    m2r.getWriter().setAlignmentSize(cruPageAlignment);
+    m2r.getWriter().setAlignmentPaddingFiller(0xff);
+  }
   m2r.init();
   m2r.processDigits(inpName);
   wr.writeConfFile(wr.getOrigin().str, "RAWDATA", o2::utils::Str::concat_string(outDirName, wr.getOrigin().str, "raw.cfg"));
diff --git a/Detectors/CTP/workflow/CMakeLists.txt b/Detectors/CTP/workflow/CMakeLists.txt
index bfa962311c910..32d87b1cf2167 100644
--- a/Detectors/CTP/workflow/CMakeLists.txt
+++ b/Detectors/CTP/workflow/CMakeLists.txt
@@ -10,8 +10,7 @@
 # or submit itself to any jurisdiction.
 
 o2_add_library(CTPWorkflow
-               SOURCES src/RecoWorkflow.cxx
-                       src/RawToDigitConverterSpec.cxx
+               SOURCES src/RawDecoderSpec.cxx
                        src/EntropyEncoderSpec.cxx
                        src/EntropyDecoderSpec.cxx
                PUBLIC_LINK_LIBRARIES O2::Framework
diff --git a/Detectors/CTP/workflow/include/CTPWorkflow/EntropyDecoderSpec.h b/Detectors/CTP/workflow/include/CTPWorkflow/EntropyDecoderSpec.h
index 4596fe12cb31d..dda45c9f11a34 100644
--- a/Detectors/CTP/workflow/include/CTPWorkflow/EntropyDecoderSpec.h
+++ b/Detectors/CTP/workflow/include/CTPWorkflow/EntropyDecoderSpec.h
@@ -28,12 +28,13 @@ namespace ctp
 class EntropyDecoderSpec : public o2::framework::Task
 {
  public:
-  EntropyDecoderSpec(int verbosity);
+  EntropyDecoderSpec(int verbosity, const std::string& ctfdictOpt = "none");
   ~EntropyDecoderSpec() override = default;
   void run(o2::framework::ProcessingContext& pc) final;
   void init(o2::framework::InitContext& ic) final;
   void endOfStream(o2::framework::EndOfStreamContext& ec) final;
   void finaliseCCDB(o2::framework::ConcreteDataMatcher& matcher, void* obj) final;
+  void updateTimeDependentParams(framework::ProcessingContext& pc);
 
  private:
   o2::ctp::CTFCoder mCTFCoder;
@@ -41,7 +42,7 @@ class EntropyDecoderSpec : public o2::framework::Task
 };
 
 /// create a processor spec
-framework::DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec);
+framework::DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec, const std::string& ctfdictOpt);
 
 } // namespace ctp
 } // namespace o2
diff --git a/Detectors/CTP/workflow/include/CTPWorkflow/EntropyEncoderSpec.h b/Detectors/CTP/workflow/include/CTPWorkflow/EntropyEncoderSpec.h
index 0c5dca528db76..a63119264e071 100644
--- a/Detectors/CTP/workflow/include/CTPWorkflow/EntropyEncoderSpec.h
+++ b/Detectors/CTP/workflow/include/CTPWorkflow/EntropyEncoderSpec.h
@@ -28,7 +28,7 @@ namespace ctp
 class EntropyEncoderSpec : public o2::framework::Task
 {
  public:
-  EntropyEncoderSpec();
+  EntropyEncoderSpec(bool selIR, bool noLumi, const std::string& ctfdictOpt = "none");
   ~EntropyEncoderSpec() override = default;
   void run(o2::framework::ProcessingContext& pc) final;
   void init(o2::framework::InitContext& ic) final;
@@ -37,11 +37,13 @@ class EntropyEncoderSpec : public o2::framework::Task
 
  private:
   o2::ctp::CTFCoder mCTFCoder;
+  bool mSelIR = false;
+  bool mNoLumi = false;
   TStopwatch mTimer;
 };
 
 /// create a processor spec
-framework::DataProcessorSpec getEntropyEncoderSpec();
+framework::DataProcessorSpec getEntropyEncoderSpec(bool selIR = false, bool noLumiInput = false, const std::string& ctfdictOpt = "none");
 
 } // namespace ctp
 } // namespace o2
diff --git a/Detectors/CTP/workflow/include/CTPWorkflow/RawDecoderSpec.h b/Detectors/CTP/workflow/include/CTPWorkflow/RawDecoderSpec.h
new file mode 100644
index 0000000000000..3198e5c33e219
--- /dev/null
+++ b/Detectors/CTP/workflow/include/CTPWorkflow/RawDecoderSpec.h
@@ -0,0 +1,100 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef O2_CTP_RAWDECODER_H
+#define O2_CTP_RAWDECODER_H
+
+#include 
+#include 
+#include "Framework/DataProcessorSpec.h"
+#include "Framework/Task.h"
+#include "Framework/WorkflowSpec.h"
+#include "DataFormatsCTP/Digits.h"
+#include "DataFormatsCTP/LumiInfo.h"
+#include "CTPReconstruction/RawDataDecoder.h"
+
+namespace o2
+{
+namespace ctp
+{
+namespace reco_workflow
+{
+
+/// \class RawDecoderSpec
+/// \brief Coverter task for Raw data to CTP digits
+/// \author Roman Lietava from CPV example
+///
+class RawDecoderSpec : public framework::Task
+{
+ public:
+  /// \brief Constructor
+  /// \param propagateMC If true the MCTruthContainer is propagated to the output
+  RawDecoderSpec(bool digits, bool lumi) : mDoDigits(digits), mDoLumi(lumi) {}
+  /// \brief Destructor
+  ~RawDecoderSpec() override = default;
+  /// \brief Initializing the RawDecoderSpec
+  /// \param ctx Init context
+  void init(framework::InitContext& ctx) final;
+  void endOfStream(o2::framework::EndOfStreamContext& ec) final;
+  /// \brief Run conversion of raw data to cells
+  /// \param ctx Processing context
+  ///
+  /// The following branches are linked:
+  /// Input RawData: {"ROUT", "RAWDATA", 0, Lifetime::Timeframe}
+  /// Output HW errors: {"CTP", "RAWHWERRORS", 0, Lifetime::Timeframe} -later
+  void run(framework::ProcessingContext& ctx) final;
+  void updateTimeDependentParams(framework::ProcessingContext& pc);
+
+ protected:
+ private:
+  // for digits
+  bool mDoDigits = true;
+  o2::pmr::vector mOutputDigits;
+  int mMaxInputSize = 0;
+  bool mMaxInputSizeFatal = 0;
+  // for lumi
+  bool mDoLumi = true;
+  //
+  LumiInfo mOutputLumiInfo;
+  bool mVerbose = false;
+  uint64_t mCountsT = 0;
+  uint64_t mCountsV = 0;
+  uint32_t mNTFToIntegrate = 1;
+  uint32_t mNHBIntegratedT = 0;
+  uint32_t mNHBIntegratedV = 0;
+  bool mDecodeinputs = 0;
+  std::deque mHistoryT;
+  std::deque mHistoryV;
+  RawDataDecoder mDecoder;
+  // Errors
+  int mLostDueToShiftInps = 0;
+  int mErrorIR = 0;
+  int mErrorTCR = 0;
+  int mIRRejected = 0;
+  int mTCRRejected = 0;
+  std::array mClsEA{};
+  std::array mClsEB{}; // from inputs
+  std::array mClsA{};
+  std::array mClsB{}; // from inputs
+  bool mCheckConsistency = false;
+};
+
+/// \brief Creating DataProcessorSpec for the CTP
+///
+o2::framework::DataProcessorSpec getRawDecoderSpec(bool askSTFDist, bool digits, bool lumi);
+
+} // namespace reco_workflow
+
+} // namespace ctp
+
+} // namespace o2
+
+#endif
diff --git a/Detectors/CTP/workflow/include/CTPWorkflow/RawToDigitConverterSpec.h b/Detectors/CTP/workflow/include/CTPWorkflow/RawToDigitConverterSpec.h
deleted file mode 100644
index 22fdb0a979d3b..0000000000000
--- a/Detectors/CTP/workflow/include/CTPWorkflow/RawToDigitConverterSpec.h
+++ /dev/null
@@ -1,67 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-#include 
-
-#include "Framework/DataProcessorSpec.h"
-#include "Framework/Task.h"
-#include "DataFormatsCTP/Digits.h"
-
-namespace o2
-{
-
-namespace ctp
-{
-
-namespace reco_workflow
-{
-
-/// \class RawToDigitConverterSpec
-/// \brief Coverter task for Raw data to CTP digits
-/// \author Roman Lietava from CPV example
-///
-class RawToDigitConverterSpec : public framework::Task
-{
- public:
-  /// \brief Constructor
-  /// \param propagateMC If true the MCTruthContainer is propagated to the output
-  RawToDigitConverterSpec() = default;
-
-  /// \brief Destructor
-  ~RawToDigitConverterSpec() override = default;
-
-  /// \brief Initializing the RawToDigitConverterSpec
-  /// \param ctx Init context
-  void init(framework::InitContext& ctx) final;
-
-  /// \brief Run conversion of raw data to cells
-  /// \param ctx Processing context
-  ///
-  /// The following branches are linked:
-  /// Input RawData: {"ROUT", "RAWDATA", 0, Lifetime::Timeframe}
-  /// Output HW errors: {"CTP", "RAWHWERRORS", 0, Lifetime::Timeframe} -later
-  void run(framework::ProcessingContext& ctx) final;
-  void makeGBTWordInverse(std::vector& diglets, gbtword80_t& GBTWord, gbtword80_t& remnant, uint32_t& size_gbt, uint32_t Npld) const;
-
- protected:
- private:
-  std::vector mOutputDigits;
-};
-
-/// \brief Creating DataProcessorSpec for the CTP
-///
-o2::framework::DataProcessorSpec getRawToDigitConverterSpec(bool askSTFDist);
-
-} // namespace reco_workflow
-
-} // namespace ctp
-
-} // namespace o2
diff --git a/Detectors/CTP/workflow/include/CTPWorkflow/RecoWorkflow.h b/Detectors/CTP/workflow/include/CTPWorkflow/RecoWorkflow.h
deleted file mode 100644
index e243a4b810495..0000000000000
--- a/Detectors/CTP/workflow/include/CTPWorkflow/RecoWorkflow.h
+++ /dev/null
@@ -1,36 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-#ifndef O2_CTP_RECOWORKFLOW_H
-#define O2_CTP_RECOWORKFLOW_H
-
-#include "Framework/WorkflowSpec.h"
-#include 
-#include 
-
-namespace o2
-{
-
-namespace ctp
-{
-
-namespace reco_workflow
-{
-
-/// create the workflow for CTP reconstruction
-framework::WorkflowSpec getWorkflow(bool noLostTF);
-
-} // namespace reco_workflow
-
-} // namespace ctp
-
-} // namespace o2
-#endif
diff --git a/Detectors/CTP/workflow/src/EntropyDecoderSpec.cxx b/Detectors/CTP/workflow/src/EntropyDecoderSpec.cxx
index bcd00016758a6..0fa8fb0004e4c 100644
--- a/Detectors/CTP/workflow/src/EntropyDecoderSpec.cxx
+++ b/Detectors/CTP/workflow/src/EntropyDecoderSpec.cxx
@@ -24,11 +24,13 @@ namespace o2
 {
 namespace ctp
 {
-
-EntropyDecoderSpec::EntropyDecoderSpec(int verbosity) : mCTFCoder(o2::ctf::CTFCoderBase::OpType::Decoder)
+EntropyDecoderSpec::EntropyDecoderSpec(int verbosity, const std::string& ctfdictOpt) : mCTFCoder(o2::ctf::CTFCoderBase::OpType::Decoder, ctfdictOpt)
 {
   mTimer.Stop();
   mTimer.Reset();
+  mCTFCoder.setVerbosity(verbosity);
+  mCTFCoder.setSupportBCShifts(true);
+  mCTFCoder.setDictBinding("ctfdict_CTP");
 }
 
 void EntropyDecoderSpec::finaliseCCDB(o2::framework::ConcreteDataMatcher& matcher, void* obj)
@@ -41,6 +43,9 @@ void EntropyDecoderSpec::finaliseCCDB(o2::framework::ConcreteDataMatcher& matche
 void EntropyDecoderSpec::init(o2::framework::InitContext& ic)
 {
   mCTFCoder.init(ic);
+  bool decodeinps = !ic.options().get("ignore-ctpinputs-decoding-ctf");
+  mCTFCoder.setDecodeInps(decodeinps);
+  LOG(info) << "Decode inputs:" << decodeinps;
 }
 
 void EntropyDecoderSpec::run(ProcessingContext& pc)
@@ -49,15 +54,15 @@ void EntropyDecoderSpec::run(ProcessingContext& pc)
   mTimer.Start(false);
   o2::ctf::CTFIOSize iosize;
 
-  mCTFCoder.updateTimeDependentParams(pc);
-  auto buff = pc.inputs().get>("ctf");
-
+  updateTimeDependentParams(pc);
+  auto buff = pc.inputs().get>("ctf_CTP");
   auto& digits = pc.outputs().make>(OutputRef{"digits"});
+  auto& lumi = pc.outputs().make(OutputRef{"CTPLumi"});
 
   // since the buff is const, we cannot use EncodedBlocks::relocate directly, instead we wrap its data to another flat object
   if (buff.size()) {
     const auto ctfImage = o2::ctp::CTF::getImage(buff.data());
-    iosize = mCTFCoder.decode(ctfImage, digits);
+    iosize = mCTFCoder.decode(ctfImage, digits, lumi);
   }
   pc.outputs().snapshot({"ctfrep", 0}, iosize);
   mTimer.Stop();
@@ -69,24 +74,43 @@ void EntropyDecoderSpec::endOfStream(EndOfStreamContext& ec)
   LOGF(info, "CTP Entropy Decoding total timing: Cpu: %.3e Real: %.3e s in %d slots",
        mTimer.CpuTime(), mTimer.RealTime(), mTimer.Counter() - 1);
 }
+void EntropyDecoderSpec::updateTimeDependentParams(framework::ProcessingContext& pc)
+{
+  mCTFCoder.updateTimeDependentParams(pc, true);
+  if (pc.services().get().globalRunNumberChanged) {
+    const auto ctpcfg = pc.inputs().get("ctpconfig");
+    if (mCTFCoder.getDecodeInps()) {
+      const auto ctpcfg = pc.inputs().get("ctpconfig");
+      if (ctpcfg != nullptr) {
+        mCTFCoder.setCTPConfig(*ctpcfg);
+        LOG(info) << "ctpconfig for run done:" << mCTFCoder.getCTPConfig().getRunNumber();
+      }
+    }
+  }
+}
 
-DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec)
+DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec, const std::string& ctfdictOpt)
 {
   std::vector outputs{
     OutputSpec{{"digits"}, "CTP", "DIGITS", 0, Lifetime::Timeframe},
+    OutputSpec{{"CTPLumi"}, "CTP", "LUMI", 0, Lifetime::Timeframe},
     OutputSpec{{"ctfrep"}, "CTP", "CTFDECREP", 0, Lifetime::Timeframe}};
 
   std::vector inputs;
-  inputs.emplace_back("ctf", "CTP", "CTFDATA", sspec, Lifetime::Timeframe);
-  inputs.emplace_back("ctfdict", "CTP", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("CTP/Calib/CTFDictionary"));
+  inputs.emplace_back("ctf_CTP", "CTP", "CTFDATA", sspec, Lifetime::Timeframe);
 
+  if (ctfdictOpt.empty() || ctfdictOpt == "ccdb") {
+    inputs.emplace_back("ctfdict_CTP", "CTP", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("CTP/Calib/CTFDictionaryTree"));
+  }
+  inputs.emplace_back("trigoffset", "CTP", "Trig_Offset", 0, Lifetime::Condition, ccdbParamSpec("CTP/Config/TriggerOffsets"));
+  inputs.emplace_back("ctpconfig", "CTP", "CTPCONFIG", 0, Lifetime::Condition, ccdbParamSpec("CTP/Config/Config", 1));
   return DataProcessorSpec{
     "ctp-entropy-decoder",
     inputs,
     outputs,
-    AlgorithmSpec{adaptFromTask(verbosity)},
-    Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}}};
+    AlgorithmSpec{adaptFromTask(verbosity, ctfdictOpt)},
+    Options{{"ignore-ctpinputs-decoding-ctf", VariantType::Bool, false, {"Inputs alignment: false - CTF decoder - has to be compatible with reco: allowed options: 10,01,00"}},
+            {"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}};
 }
-
 } // namespace ctp
 } // namespace o2
diff --git a/Detectors/CTP/workflow/src/EntropyEncoderSpec.cxx b/Detectors/CTP/workflow/src/EntropyEncoderSpec.cxx
index 81fc7655f9187..902fe22dadcc9 100644
--- a/Detectors/CTP/workflow/src/EntropyEncoderSpec.cxx
+++ b/Detectors/CTP/workflow/src/EntropyEncoderSpec.cxx
@@ -25,8 +25,7 @@ namespace o2
 {
 namespace ctp
 {
-
-EntropyEncoderSpec::EntropyEncoderSpec() : mCTFCoder(o2::ctf::CTFCoderBase::OpType::Encoder)
+EntropyEncoderSpec::EntropyEncoderSpec(bool selIR, bool nolumi, const std::string& ctfdictOpt) : mCTFCoder(o2::ctf::CTFCoderBase::OpType::Encoder, ctfdictOpt), mSelIR(selIR), mNoLumi(nolumi)
 {
   mTimer.Stop();
   mTimer.Reset();
@@ -48,10 +47,24 @@ void EntropyEncoderSpec::run(ProcessingContext& pc)
 {
   auto cput = mTimer.CpuTime();
   mTimer.Start(false);
-  mCTFCoder.updateTimeDependentParams(pc);
+  mCTFCoder.updateTimeDependentParams(pc, true);
   auto digits = pc.inputs().get>("digits");
-  auto& buffer = pc.outputs().make>(Output{"CTP", "CTFDATA", 0, Lifetime::Timeframe});
-  auto iosize = mCTFCoder.encode(buffer, digits);
+  static LumiInfo lumiPrev;
+  const int maxDumRep = 5;
+  static int dumRep = 0;
+  LumiInfo lumi{};
+  if (!mNoLumi) {
+    if (pc.inputs().get>("CTPLumi").size() == sizeof(LumiInfo)) {
+      lumiPrev = lumi = pc.inputs().get("CTPLumi");
+    } else {
+      if (dumRep < maxDumRep && lumiPrev.nHBFCounted == 0 && lumiPrev.nHBFCountedFV0 == 0) {
+        LOGP(alarm, "Previous TF lumi used to substitute dummy input is empty, warning {} of {}", ++dumRep, maxDumRep);
+      }
+      lumi = lumiPrev;
+    }
+  }
+  auto& buffer = pc.outputs().make>(Output{"CTP", "CTFDATA", 0});
+  auto iosize = mCTFCoder.encode(buffer, digits, lumi);
   pc.outputs().snapshot({"ctfrep", 0}, iosize);
   mTimer.Stop();
   LOG(info) << iosize.asString() << " in " << mTimer.CpuTime() - cput << " s";
@@ -63,21 +76,29 @@ void EntropyEncoderSpec::endOfStream(EndOfStreamContext& ec)
        mTimer.CpuTime(), mTimer.RealTime(), mTimer.Counter() - 1);
 }
 
-DataProcessorSpec getEntropyEncoderSpec()
+DataProcessorSpec getEntropyEncoderSpec(bool selIR, bool nolumi, const std::string& ctfdictOpt)
 {
   std::vector inputs;
   inputs.emplace_back("digits", "CTP", "DIGITS", 0, Lifetime::Timeframe);
-  inputs.emplace_back("ctfdict", "CTP", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("CTP/Calib/CTFDictionary"));
+  if (!nolumi) {
+    inputs.emplace_back("CTPLumi", "CTP", "LUMI", 0, Lifetime::Timeframe);
+  }
 
+  if (ctfdictOpt.empty() || ctfdictOpt == "ccdb") {
+    inputs.emplace_back("ctfdict", "CTP", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("CTP/Calib/CTFDictionaryTree"));
+  }
+  if (selIR) {
+    inputs.emplace_back("selIRFrames", "CTF", "SELIRFRAMES", 0, Lifetime::Timeframe);
+  }
   return DataProcessorSpec{
     "ctp-entropy-encoder",
     inputs,
-    Outputs{{"CTP", "CTFDATA", 0, Lifetime::Timeframe},
-            {{"ctfrep"}, "CTP", "CTFENCREP", 0, Lifetime::Timeframe}},
-    AlgorithmSpec{adaptFromTask()},
-    Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}},
-            {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}}}};
+    Outputs{{"CTP", "CTFDATA", 0, Lifetime::Timeframe}, {{"ctfrep"}, "CTP", "CTFENCREP", 0, Lifetime::Timeframe}},
+    AlgorithmSpec{adaptFromTask(selIR, nolumi, ctfdictOpt)},
+    Options{{"irframe-margin-bwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame lower boundary when selection is requested"}},
+            {"irframe-margin-fwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame upper boundary when selection is requested"}},
+            {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}},
+            {"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}};
 }
-
 } // namespace ctp
 } // namespace o2
diff --git a/Detectors/CTP/workflow/src/RawDecoderSpec.cxx b/Detectors/CTP/workflow/src/RawDecoderSpec.cxx
new file mode 100644
index 0000000000000..041e6cb472ebb
--- /dev/null
+++ b/Detectors/CTP/workflow/src/RawDecoderSpec.cxx
@@ -0,0 +1,272 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include 
+#include 
+#include "Framework/InputRecordWalker.h"
+#include "Framework/DataRefUtils.h"
+#include "Framework/ConfigParamRegistry.h"
+#include "DetectorsRaw/RDHUtils.h"
+#include "CTPWorkflow/RawDecoderSpec.h"
+#include "CommonUtils/VerbosityConfig.h"
+#include "Framework/InputRecord.h"
+#include "DataFormatsCTP/TriggerOffsetsParam.h"
+#include "Framework/CCDBParamSpec.h"
+#include "DataFormatsCTP/Configuration.h"
+
+using namespace o2::ctp::reco_workflow;
+
+void RawDecoderSpec::init(framework::InitContext& ctx)
+{
+  mCheckConsistency = ctx.options().get("check-consistency");
+  mDecoder.setCheckConsistency(mCheckConsistency);
+  mDecodeinputs = ctx.options().get("ctpinputs-decoding");
+  mDecoder.setDecodeInps(mDecodeinputs);
+  mNTFToIntegrate = ctx.options().get("ntf-to-average");
+  mVerbose = ctx.options().get("use-verbose-mode");
+  int maxerrors = ctx.options().get("print-errors-num");
+  mDecoder.setVerbose(mVerbose);
+  mDecoder.setDoLumi(mDoLumi);
+  mDecoder.setDoDigits(mDoDigits);
+  mDecoder.setMAXErrors(maxerrors);
+  std::string lumiinp1 = ctx.options().get("lumi-inp1");
+  std::string lumiinp2 = ctx.options().get("lumi-inp2");
+  int inp1 = mDecoder.setLumiInp(1, lumiinp1);
+  int inp2 = mDecoder.setLumiInp(2, lumiinp2);
+  mOutputLumiInfo.inp1 = inp1;
+  mOutputLumiInfo.inp2 = inp2;
+  mMaxInputSize = ctx.options().get("max-input-size");
+  mMaxInputSizeFatal = ctx.options().get("max-input-size-fatal");
+  LOG(info) << "CTP reco init done. Inputs decoding here:" << mDecodeinputs << " DoLumi:" << mDoLumi << " DoDigits:" << mDoDigits << " NTF:" << mNTFToIntegrate << " Lumi inputs:" << lumiinp1 << ":" << inp1 << " " << lumiinp2 << ":" << inp2 << " Max errors:" << maxerrors << " Max input size:" << mMaxInputSize << " MaxInputSizeFatal:" << mMaxInputSizeFatal << " CheckConsistency:" << mCheckConsistency;
+  // mOutputLumiInfo.printInputs();
+}
+void RawDecoderSpec::endOfStream(framework::EndOfStreamContext& ec)
+{
+  auto& TFOrbits = mDecoder.getTFOrbits();
+  std::sort(TFOrbits.begin(), TFOrbits.end());
+  size_t l = TFOrbits.size();
+  uint32_t o0 = 0;
+  if (l) {
+    o0 = TFOrbits[0];
+  }
+  int nmiss = 0;
+  int nprt = 0;
+  std::cout << "Missing orbits:";
+  for (int i = 1; i < l; i++) {
+    if ((TFOrbits[i] - o0) > 0x20) {
+      if (nprt < 20) {
+        std::cout << " " << o0 << "-" << TFOrbits[i];
+      }
+      nmiss += (TFOrbits[i] - o0) / 0x20;
+      nprt++;
+    }
+    o0 = TFOrbits[i];
+  }
+  std::cout << std::endl;
+  LOG(info) << "Number of non continous TF:" << nmiss << std::endl;
+  LOG(info) << "Lost in shiftInputs:" << mLostDueToShiftInps;
+  LOG(info) << "Lost in addDigit Inputs:" << mIRRejected << " Classes:" << mTCRRejected;
+  if (mErrorIR || mErrorTCR) {
+    LOG(error) << "# of IR errors:" << mErrorIR << " TCR errors:" << mErrorTCR << std::endl;
+  }
+  if (mCheckConsistency) {
+    LOG(info) << "Lost due to the shift Consistency Checker:" << mDecoder.getLostDueToShiftCls();
+    auto ctpcfg = mDecoder.getCTPConfig();
+    for (int i = 0; i < o2::ctp::CTP_NCLASSES; i++) {
+      std::string name = ctpcfg.getClassNameFromIndex(i);
+      if (mClsEA[i]) {
+        LOG(error) << " Class without inputs:";
+      }
+      LOG(important) << "CLASS:" << name << ":" << i << " Cls=>Inp:" << mClsA[i] << " Inp=>Cls:" << mClsB[i] << "  ErrorsCls=>Inps:" << mClsEA[i] << "  MissingInps=>Cls:" << mClsEB[i];
+    }
+  }
+}
+void RawDecoderSpec::run(framework::ProcessingContext& ctx)
+{
+  updateTimeDependentParams(ctx);
+  mOutputDigits.clear();
+  std::map digits;
+  using InputSpec = o2::framework::InputSpec;
+  using ConcreteDataTypeMatcher = o2::framework::ConcreteDataTypeMatcher;
+  using Lifetime = o2::framework::Lifetime;
+  // setUpDummyLink
+  auto& inputs = ctx.inputs();
+  auto dummyOutput = [&ctx, this]() {
+    if (this->mDoDigits) {
+      ctx.outputs().snapshot(o2::framework::Output{"CTP", "DIGITS", 0}, this->mOutputDigits);
+    }
+    if (this->mDoLumi) {
+      ctx.outputs().snapshot(o2::framework::Output{"CTP", "LUMI", 0}, this->mOutputLumiInfo);
+    }
+  };
+  // if we see requested data type input with 0xDEADBEEF subspec and 0 payload this means that the "delayed message"
+  // mechanism created it in absence of real data from upstream. Processor should send empty output to not block the workflow
+  {
+    static size_t contDeadBeef = 0; // number of times 0xDEADBEEF was seen continuously
+    std::vector dummy{InputSpec{"dummy", o2::framework::ConcreteDataMatcher{"CTP", "RAWDATA", 0xDEADBEEF}}};
+    for (const auto& ref : o2::framework::InputRecordWalker(inputs, dummy)) {
+      const auto dh = o2::framework::DataRefUtils::getHeader(ref);
+      auto payloadSize = o2::framework::DataRefUtils::getPayloadSize(ref);
+      if (payloadSize == 0) {
+        auto maxWarn = o2::conf::VerbosityConfig::Instance().maxWarnDeadBeef;
+        if (++contDeadBeef <= maxWarn) {
+          LOGP(alarm, "Found input [{}/{}/{:#x}] TF#{} 1st_orbit:{} Payload {} : assuming no payload for all links in this TF{}",
+               dh->dataOrigin.str, dh->dataDescription.str, dh->subSpecification, dh->tfCounter, dh->firstTForbit, payloadSize,
+               contDeadBeef == maxWarn ? fmt::format(". {} such inputs in row received, stopping reporting", contDeadBeef) : "");
+        }
+        dummyOutput();
+        return;
+      }
+    }
+    contDeadBeef = 0; // if good data, reset the counter
+  }
+  //
+  std::vector lumiPointsHBF1;
+  std::vector filter{InputSpec{"filter", ConcreteDataTypeMatcher{"CTP", "RAWDATA"}, Lifetime::Timeframe}};
+  bool fatal_flag = 0;
+  if (mMaxInputSize > 0) {
+    size_t payloadSize = 0;
+    for (const auto& ref : o2::framework::InputRecordWalker(inputs, filter)) {
+      const auto dh = o2::framework::DataRefUtils::getHeader(ref);
+      payloadSize += o2::framework::DataRefUtils::getPayloadSize(ref);
+    }
+    if (payloadSize > (size_t)mMaxInputSize) {
+      if (mMaxInputSizeFatal) {
+        fatal_flag = 1;
+        LOG(error) << "Input data size bigger than threshold: " << mMaxInputSize << " < " << payloadSize << " decoding TF and exiting.";
+        // LOG(fatal) << "Input data size:" << payloadSize; - fatal issued in decoder
+      } else {
+        LOG(error) << "Input data size:" << payloadSize << " sending dummy output";
+        dummyOutput();
+        return;
+      }
+    }
+  }
+  int ret = 0;
+  if (fatal_flag) {
+    ret = mDecoder.decodeRawFatal(inputs, filter);
+  } else {
+    ret = mDecoder.decodeRaw(inputs, filter, mOutputDigits, lumiPointsHBF1);
+  }
+  if (ret == 1) {
+    dummyOutput();
+    return;
+  }
+  if (mDoDigits) {
+    LOG(info) << "[CTPRawToDigitConverter - run] Writing " << mOutputDigits.size() << " digits. IR rejected:" << mDecoder.getIRRejected() << " TCR rejected:" << mDecoder.getTCRRejected();
+    ctx.outputs().snapshot(o2::framework::Output{"CTP", "DIGITS", 0}, mOutputDigits);
+    mLostDueToShiftInps += mDecoder.getLostDueToShiftInp();
+    mErrorIR += mDecoder.getErrorIR();
+    mErrorTCR += mDecoder.getErrorTCR();
+    mIRRejected += mDecoder.getIRRejected();
+    mTCRRejected += mDecoder.getTCRRejected();
+    auto clsEA = mDecoder.getClassErrorsA();
+    auto clsEB = mDecoder.getClassErrorsB();
+    auto cntCA = mDecoder.getClassCountersA();
+    auto cntCB = mDecoder.getClassCountersB();
+    for (int i = 0; i < o2::ctp::CTP_NCLASSES; i++) {
+      mClsEA[i] += clsEA[i];
+      mClsEB[i] += clsEB[i];
+      mClsA[i] += cntCA[i];
+      mClsB[i] += cntCB[i];
+    }
+  }
+  if (mDoLumi) {
+    uint32_t tfCountsT = 0;
+    uint32_t tfCountsV = 0;
+    for (auto const& lp : lumiPointsHBF1) {
+      tfCountsT += lp.counts;
+      tfCountsV += lp.countsFV0;
+    }
+    // LOG(info) << "Lumi rate:" << tfCounts/(128.*88e-6);
+    // FT0
+    mHistoryT.push_back(tfCountsT);
+    mCountsT += tfCountsT;
+    if (mHistoryT.size() <= mNTFToIntegrate) {
+      mNHBIntegratedT += lumiPointsHBF1.size();
+    } else {
+      mCountsT -= mHistoryT.front();
+      mHistoryT.pop_front();
+    }
+    // FV0
+    mHistoryV.push_back(tfCountsV);
+    mCountsV += tfCountsV;
+    if (mHistoryV.size() <= mNTFToIntegrate) {
+      mNHBIntegratedV += lumiPointsHBF1.size();
+    } else {
+      mCountsV -= mHistoryV.front();
+      mHistoryV.pop_front();
+    }
+    //
+    if (mNHBIntegratedT || mNHBIntegratedV) {
+      mOutputLumiInfo.orbit = lumiPointsHBF1[0].orbit;
+    }
+    mOutputLumiInfo.counts = mCountsT;
+
+    mOutputLumiInfo.countsFV0 = mCountsV;
+    mOutputLumiInfo.nHBFCounted = mNHBIntegratedT;
+    mOutputLumiInfo.nHBFCountedFV0 = mNHBIntegratedV;
+    if (mVerbose) {
+      mOutputLumiInfo.printInputs();
+      LOGP(info, "Orbit {}: {}/{} counts inp1/inp2 in {}/{} HBFs -> lumi_inp1 = {:.3e}+-{:.3e} lumi_inp2 = {:.3e}+-{:.3e}", mOutputLumiInfo.orbit, mCountsT, mCountsV, mNHBIntegratedT, mNHBIntegratedV, mOutputLumiInfo.getLumi(), mOutputLumiInfo.getLumiError(), mOutputLumiInfo.getLumiFV0(), mOutputLumiInfo.getLumiFV0Error());
+    }
+    ctx.outputs().snapshot(o2::framework::Output{"CTP", "LUMI", 0}, mOutputLumiInfo);
+  }
+}
+o2::framework::DataProcessorSpec o2::ctp::reco_workflow::getRawDecoderSpec(bool askDISTSTF, bool digits, bool lumi)
+{
+  if (!digits && !lumi) {
+    throw std::runtime_error("all outputs were disabled");
+  }
+  std::vector inputs;
+  inputs.emplace_back("TF", o2::framework::ConcreteDataTypeMatcher{"CTP", "RAWDATA"}, o2::framework::Lifetime::Timeframe);
+  if (askDISTSTF) {
+    inputs.emplace_back("stdDist", "FLP", "DISTSUBTIMEFRAME", 0, o2::framework::Lifetime::Timeframe);
+  }
+
+  std::vector outputs;
+  inputs.emplace_back("ctpconfig", "CTP", "CTPCONFIG", 0, o2::framework::Lifetime::Condition, o2::framework::ccdbParamSpec("CTP/Config/Config", 1));
+  inputs.emplace_back("trigoffset", "CTP", "Trig_Offset", 0, o2::framework::Lifetime::Condition, o2::framework::ccdbParamSpec("CTP/Config/TriggerOffsets"));
+  if (digits) {
+    outputs.emplace_back("CTP", "DIGITS", 0, o2::framework::Lifetime::Timeframe);
+  }
+  if (lumi) {
+    outputs.emplace_back("CTP", "LUMI", 0, o2::framework::Lifetime::Timeframe);
+  }
+  return o2::framework::DataProcessorSpec{
+    "ctp-raw-decoder",
+    inputs,
+    outputs,
+    o2::framework::AlgorithmSpec{o2::framework::adaptFromTask(digits, lumi)},
+    o2::framework::Options{
+      {"ntf-to-average", o2::framework::VariantType::Int, 90, {"Time interval for averaging luminosity in units of TF"}},
+      {"print-errors-num", o2::framework::VariantType::Int, 3, {"Max number of errors to print"}},
+      {"lumi-inp1", o2::framework::VariantType::String, "TVX", {"The first input used for online lumi. Name in capital."}},
+      {"lumi-inp2", o2::framework::VariantType::String, "VBA", {"The second input used for online lumi. Name in capital."}},
+      {"use-verbose-mode", o2::framework::VariantType::Bool, false, {"Verbose logging"}},
+      {"max-input-size", o2::framework::VariantType::Int, 0, {"Do not process input if bigger than max size, 0 - do not check"}},
+      {"max-input-size-fatal", o2::framework::VariantType::Bool, false, {"If true issue fatal error otherwise error only"}},
+      {"check-consistency", o2::framework::VariantType::Bool, false, {"If true checks digits consistency using ctp config"}},
+      {"ctpinputs-decoding", o2::framework::VariantType::Bool, false, {"Inputs alignment: true - raw decoder - has to be compatible with CTF decoder: allowed options: 10,01,00"}}}};
+}
+void RawDecoderSpec::updateTimeDependentParams(framework::ProcessingContext& pc)
+{
+  if (pc.services().get().globalRunNumberChanged) {
+    pc.inputs().get("trigoffset");
+    const auto& trigOffsParam = o2::ctp::TriggerOffsetsParam::Instance();
+    LOG(info) << "updateing TroggerOffsetsParam: inputs L0_L1:" << trigOffsParam.L0_L1 << " classes L0_L1:" << trigOffsParam.L0_L1_classes;
+    const auto ctpcfg = pc.inputs().get("ctpconfig");
+    if (ctpcfg != nullptr) {
+      mDecoder.setCTPConfig(*ctpcfg);
+      LOG(info) << "ctpconfig for run done:" << mDecoder.getCTPConfig().getRunNumber();
+    }
+  }
+}
diff --git a/Detectors/CTP/workflow/src/RawToDigitConverterSpec.cxx b/Detectors/CTP/workflow/src/RawToDigitConverterSpec.cxx
deleted file mode 100644
index 9392029a15f4f..0000000000000
--- a/Detectors/CTP/workflow/src/RawToDigitConverterSpec.cxx
+++ /dev/null
@@ -1,222 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-#include 
-#include "FairLogger.h"
-#include "CommonDataFormat/InteractionRecord.h"
-#include "Framework/InputRecordWalker.h"
-#include "Framework/DataRefUtils.h"
-#include "Framework/WorkflowSpec.h"
-#include "DetectorsRaw/RDHUtils.h"
-#include "DPLUtils/DPLRawParser.h"
-#include "CTPWorkflow/RawToDigitConverterSpec.h"
-#include "CommonUtils/VerbosityConfig.h"
-
-using namespace o2::ctp::reco_workflow;
-
-void RawToDigitConverterSpec::init(framework::InitContext& ctx)
-{
-}
-
-void RawToDigitConverterSpec::run(framework::ProcessingContext& ctx)
-{
-  mOutputDigits.clear();
-  std::map digits;
-  const gbtword80_t bcidmask = 0xfff;
-  gbtword80_t pldmask;
-  using InputSpec = o2::framework::InputSpec;
-  using ConcreteDataTypeMatcher = o2::framework::ConcreteDataTypeMatcher;
-  using Lifetime = o2::framework::Lifetime;
-  //mOutputHWErrors.clear();
-  std::vector filter{InputSpec{"filter", ConcreteDataTypeMatcher{"CTP", "RAWDATA"}, Lifetime::Timeframe}};
-  o2::framework::DPLRawParser parser(ctx.inputs(), filter);
-  //setUpDummyLink
-  auto& inputs = ctx.inputs();
-  // if we see requested data type input with 0xDEADBEEF subspec and 0 payload this means that the "delayed message"
-  // mechanism created it in absence of real data from upstream. Processor should send empty output to not block the workflow
-  {
-    static size_t contDeadBeef = 0; // number of times 0xDEADBEEF was seen continuously
-    std::vector dummy{InputSpec{"dummy", o2::framework::ConcreteDataMatcher{"CTP", "RAWDATA", 0xDEADBEEF}}};
-    for (const auto& ref : o2::framework::InputRecordWalker(inputs, dummy)) {
-      const auto dh = o2::framework::DataRefUtils::getHeader(ref);
-      auto payloadSize = o2::framework::DataRefUtils::getPayloadSize(ref);
-      if (payloadSize == 0) {
-        auto maxWarn = o2::conf::VerbosityConfig::Instance().maxWarnDeadBeef;
-        if (++contDeadBeef <= maxWarn) {
-          LOGP(alarm, "Found input [{}/{}/{:#x}] TF#{} 1st_orbit:{} Payload {} : assuming no payload for all links in this TF{}",
-               dh->dataOrigin.str, dh->dataDescription.str, dh->subSpecification, dh->tfCounter, dh->firstTForbit, payloadSize,
-               contDeadBeef == maxWarn ? fmt::format(". {} such inputs in row received, stopping reporting", contDeadBeef) : "");
-        }
-        ctx.outputs().snapshot(o2::framework::Output{"CTP", "DIGITS", 0, o2::framework::Lifetime::Timeframe}, mOutputDigits);
-        return;
-      }
-    }
-    contDeadBeef = 0; // if good data, reset the counter
-  }
-  //
-  uint32_t payloadCTP;
-  uint32_t orbit0 = 0;
-  bool first = true;
-  gbtword80_t remnant = 0;
-  uint32_t size_gbt = 0;
-  for (auto it = parser.begin(); it != parser.end(); ++it) {
-    auto rdh = it.get_if();
-    auto triggerOrbit = o2::raw::RDHUtils::getTriggerOrbit(rdh);
-    if (first) {
-      orbit0 = triggerOrbit;
-      first = false;
-    }
-    auto feeID = o2::raw::RDHUtils::getFEEID(rdh); // 0 = IR, 1 = TCR
-    auto linkCRU = (feeID & 0xf00) >> 8;
-    if (linkCRU == o2::ctp::GBTLinkIDIntRec) {
-      payloadCTP = o2::ctp::NIntRecPayload;
-    } else if (linkCRU == o2::ctp::GBTLinkIDClassRec) {
-      payloadCTP = o2::ctp::NClassPayload;
-    } else {
-      LOG(error) << "Unxpected  CTP CRU link:" << linkCRU;
-    }
-    LOG(debug) << "RDH FEEid: " << feeID << " CTP CRU link:" << linkCRU << " Orbit:" << triggerOrbit;
-    pldmask = 0;
-    for (uint32_t i = 0; i < payloadCTP; i++) {
-      pldmask[12 + i] = 1;
-    }
-    //LOG(info) << "pldmask:" << pldmask;
-    // TF in 128 bits words
-    gsl::span payload(it.data(), it.size());
-    gbtword80_t gbtWord = 0;
-    int wordCount = 0;
-    std::vector diglets;
-    if (orbit0 != triggerOrbit) {
-      remnant = 0;
-      size_gbt = 0;
-      orbit0 = triggerOrbit;
-    }
-    for (auto payloadWord : payload) {
-      //LOG(info) << wordCount << " payload:" <<  int(payloadWord);
-      if (wordCount == 15) {
-        wordCount = 0;
-      } else if (wordCount > 9) {
-        wordCount++;
-      } else if (wordCount == 9) {
-        for (int i = 0; i < 8; i++) {
-          gbtWord[wordCount * 8 + i] = bool(int(payloadWord) & (1 << i));
-        }
-        wordCount++;
-        diglets.clear();
-        //LOG(info) << " gbtword:" << gbtWord;
-        makeGBTWordInverse(diglets, gbtWord, remnant, size_gbt, payloadCTP);
-        // save digit in buffer recs
-        for (auto diglet : diglets) {
-          //LOG(info) << " diglet:" << diglet;
-          //LOG(info) << " pldmas:" << pldmask;
-          gbtword80_t pld = (diglet & pldmask);
-          if (pld.count() == 0) {
-            continue;
-          }
-          //LOG(info) << "    pld:" << pld;
-          pld >>= 12;
-          CTPDigit digit;
-          uint32_t bcid = (diglet & bcidmask).to_ulong();
-          o2::InteractionRecord ir;
-          ir.orbit = triggerOrbit;
-          ir.bc = bcid;
-          digit.intRecord = ir;
-          if (linkCRU == o2::ctp::GBTLinkIDIntRec) {
-            LOG(debug) << "InputMaskCount:" << digits[ir].CTPInputMask.count();
-            if (digits.count(ir) == 0) {
-              digit.setInputMask(pld);
-              digits[ir] = digit;
-              LOG(debug) << bcid << " inputs case 0 bcid orbit " << triggerOrbit << " pld:" << pld;
-            } else if (digits.count(ir) == 1) {
-              if (digits[ir].CTPInputMask.count() == 0) {
-                digits[ir].setInputMask(pld);
-                LOG(debug) << bcid << " inputs bcid vase 1 orbit " << triggerOrbit << " pld:" << pld;
-              } else {
-                LOG(error) << "Two CTP IRs with the same timestamp.";
-              }
-            } else {
-              LOG(error) << "Two digits with the same rimestamp.";
-            }
-          } else if (linkCRU == o2::ctp::GBTLinkIDClassRec) {
-            if (digits.count(ir) == 0) {
-              digit.setClassMask(pld);
-              digits[ir] = digit;
-              LOG(debug) << bcid << " class bcid case 0 orbit " << triggerOrbit << " pld:" << pld;
-            } else if (digits.count(ir) == 1) {
-              if (digits[ir].CTPClassMask.count() == 0) {
-                digits[ir].setClassMask(pld);
-                LOG(debug) << bcid << " class bcid case 1 orbit " << triggerOrbit << " pld:" << pld;
-              } else {
-                LOG(error) << "Two CTP Class masks for same timestamp";
-              }
-            } else {
-            }
-          } else {
-            LOG(error) << "Unxpected  CTP CRU link:" << linkCRU;
-          }
-        }
-        gbtWord = 0;
-      } else {
-        //std::cout << "wordCount:" << wordCount << std::endl;
-        for (int i = 0; i < 8; i++) {
-          gbtWord[wordCount * 8 + i] = bool(int(payloadWord) & (1 << i));
-          //gbtWord[(9-wordCount) * 8 + i] = bool(int(payloadWord) & (1 << i));
-        }
-        wordCount++;
-      }
-    }
-  }
-  for (auto const digmap : digits) {
-    mOutputDigits.push_back(digmap.second);
-  }
-
-  LOG(info) << "[CTPRawToDigitConverter - run] Writing " << mOutputDigits.size() << " digits ...";
-  ctx.outputs().snapshot(o2::framework::Output{"CTP", "DIGITS", 0, o2::framework::Lifetime::Timeframe}, mOutputDigits);
-  //ctx.outputs().snapshot(o2::framework::Output{"CPV", "RAWHWERRORS", 0, o2::framework::Lifetime::Timeframe}, mOutputHWErrors);
-}
-// Inverse of Digits2Raw::makeGBTWord
-void RawToDigitConverterSpec::makeGBTWordInverse(std::vector& diglets, gbtword80_t& GBTWord, gbtword80_t& remnant, uint32_t& size_gbt, uint32_t Npld) const
-{
-  gbtword80_t diglet = remnant;
-  uint32_t i = 0;
-  while (i < (NGBT - Npld)) {
-    std::bitset masksize = 0;
-    for (uint32_t j = 0; j < (Npld - size_gbt); j++) {
-      masksize[j] = 1;
-    }
-    diglet |= (GBTWord & masksize) << (size_gbt);
-    diglets.push_back(diglet);
-    diglet = 0;
-    i += Npld - size_gbt;
-    GBTWord = GBTWord >> (Npld - size_gbt);
-    size_gbt = 0;
-  }
-  size_gbt = NGBT - i;
-  remnant = GBTWord;
-}
-o2::framework::DataProcessorSpec o2::ctp::reco_workflow::getRawToDigitConverterSpec(bool askDISTSTF)
-{
-  std::vector inputs;
-  inputs.emplace_back("TF", o2::framework::ConcreteDataTypeMatcher{"CTP", "RAWDATA"}, o2::framework::Lifetime::Optional);
-  if (askDISTSTF) {
-    inputs.emplace_back("stdDist", "FLP", "DISTSUBTIMEFRAME", 0, o2::framework::Lifetime::Timeframe);
-  }
-
-  std::vector outputs;
-  outputs.emplace_back("CTP", "DIGITS", 0, o2::framework::Lifetime::Timeframe);
-
-  return o2::framework::DataProcessorSpec{
-    "CTP-RawStreamDecoder",
-    inputs,
-    outputs,
-    o2::framework::AlgorithmSpec{o2::framework::adaptFromTask()},
-    o2::framework::Options{{"result-file", o2::framework::VariantType::String, "/tmp/hmpCTPDecodeResults", {"Base name of the decoding results files."}}}};
-}
diff --git a/Detectors/CTP/workflow/src/RecoWorkflow.cxx b/Detectors/CTP/workflow/src/RecoWorkflow.cxx
deleted file mode 100644
index 2b4a91211584a..0000000000000
--- a/Detectors/CTP/workflow/src/RecoWorkflow.cxx
+++ /dev/null
@@ -1,43 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-#include 
-#include 
-#include 
-
-#include "FairLogger.h"
-
-#include "DataFormatsCTP/Digits.h"
-#include "CTPWorkflow/RecoWorkflow.h"
-#include "CTPWorkflow/RawToDigitConverterSpec.h"
-#include "Framework/DataSpecUtils.h"
-
-namespace o2
-{
-
-namespace ctp
-{
-
-namespace reco_workflow
-{
-
-o2::framework::WorkflowSpec getWorkflow(bool noLostTF)
-{
-  o2::framework::WorkflowSpec specs;
-  specs.emplace_back(o2::ctp::reco_workflow::getRawToDigitConverterSpec(noLostTF));
-  return std::move(specs);
-}
-
-} // namespace reco_workflow
-
-} // namespace ctp
-
-} // namespace o2
diff --git a/Detectors/CTP/workflow/src/ctp-raw-decoder.cxx b/Detectors/CTP/workflow/src/ctp-raw-decoder.cxx
index 41c293dbfab25..31b9647972c79 100644
--- a/Detectors/CTP/workflow/src/ctp-raw-decoder.cxx
+++ b/Detectors/CTP/workflow/src/ctp-raw-decoder.cxx
@@ -14,8 +14,6 @@
 /// @brief  Basic DPL workflow for CTP reconstruction starting from digits
 #include "Framework/WorkflowSpec.h"
 #include "Framework/ConfigParamSpec.h"
-#include "CTPWorkflow/RecoWorkflow.h"
-#include "Algorithm/RangeTokenizer.h"
 #include "CommonUtils/ConfigurableParam.h"
 #include "Framework/CallbacksPolicy.h"
 
@@ -29,29 +27,30 @@ void customize(std::vector& workflowOptions)
 {
   std::vector options{
     {"ignore-dist-stf", o2::framework::VariantType::Bool, false, {"do not subscribe to FLP/DISTSUBTIMEFRAME/0 message (no lost TF recovery)"}},
+    {"no-lumi", o2::framework::VariantType::Bool, false, {"do not produce luminosity output"}},
+    {"no-digits", o2::framework::VariantType::Bool, false, {"do not produce digits output"}},
+    {"disable-root-output", o2::framework::VariantType::Bool, false, {"disable root-files output writer"}},
     {"configKeyValues", o2::framework::VariantType::String, "", {"Semicolon separated key=value strings ..."}}};
   std::swap(workflowOptions, options);
 }
 
 #include "Framework/runDataProcessing.h" // the main driver
+#include "CTPWorkflow/RawDecoderSpec.h"
+#include "CTPWorkflowIO/DigitWriterSpec.h"
 
 /// The workflow executable for the stand alone CTP reconstruction workflow
-/// The basic workflow for CTP reconstruction is defined in RecoWorkflow.cxx
-/// and contains the following default processors
-/// - digit reader
-///
-/// The default workflow can be customized by specifying input and output types
-/// e.g. digits, raw
-///
-/// MC info is processed by default, disabled by using command line option `--disable-mc`
-///
+/// - digit and lumi reader
 /// This function hooks up the the workflow specifications into the DPL driver.
 o2::framework::WorkflowSpec defineDataProcessing(o2::framework::ConfigContext const& cfgc)
 {
-  //
-  // Update the (declared) parameters if changed from the command line
+  o2::framework::WorkflowSpec specs;
   o2::conf::ConfigurableParam::updateFromString(cfgc.options().get("configKeyValues"));
 
-  auto wf = o2::ctp::reco_workflow::getWorkflow(!cfgc.options().get("ignore-dist-stf"));
-  return std::move(wf);
+  specs.emplace_back(o2::ctp::reco_workflow::getRawDecoderSpec(!cfgc.options().get("ignore-dist-stf"),
+                                                               !cfgc.options().get("no-digits"),
+                                                               !cfgc.options().get("no-lumi")));
+  if (!cfgc.options().get("disable-root-output")) {
+    specs.emplace_back(o2::ctp::getDigitWriterSpec(!cfgc.options().get("no-lumi")));
+  }
+  return specs;
 }
diff --git a/Detectors/CTP/workflow/src/entropy-encoder-workflow.cxx b/Detectors/CTP/workflow/src/entropy-encoder-workflow.cxx
index 0d25c85462d2b..c2b324a4b3bfa 100644
--- a/Detectors/CTP/workflow/src/entropy-encoder-workflow.cxx
+++ b/Detectors/CTP/workflow/src/entropy-encoder-workflow.cxx
@@ -21,7 +21,11 @@ using namespace o2::framework;
 void customize(std::vector& workflowOptions)
 {
   // option allowing to set parameters
-  std::vector options{ConfigParamSpec{"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}}};
+  std::vector options{
+    ConfigParamSpec{"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}},
+    ConfigParamSpec{"ctf-dict", VariantType::String, "none", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}},
+    ConfigParamSpec{"no-lumi-input", VariantType::Bool, false, {"Lumi info not available"}},
+    ConfigParamSpec{"select-ir-frames", VariantType::Bool, false, {"Subscribe and filter according to external IR Frames"}}};
 
   std::swap(workflowOptions, options);
 }
@@ -35,6 +39,6 @@ WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
   WorkflowSpec wf;
   // Update the (declared) parameters if changed from the command line
   o2::conf::ConfigurableParam::updateFromString(cfgc.options().get("configKeyValues"));
-  wf.emplace_back(o2::ctp::getEntropyEncoderSpec());
+  wf.emplace_back(o2::ctp::getEntropyEncoderSpec(cfgc.options().get("select-ir-frames"), cfgc.options().get("no-lumi-input"), cfgc.options().get("ctf-dict")));
   return wf;
 }
diff --git a/Detectors/CTP/workflowIO/CMakeLists.txt b/Detectors/CTP/workflowIO/CMakeLists.txt
index 94d9a5f4abe91..e1db635515cac 100644
--- a/Detectors/CTP/workflowIO/CMakeLists.txt
+++ b/Detectors/CTP/workflowIO/CMakeLists.txt
@@ -1,18 +1,29 @@
-# Copyright CERN and copyright holders of ALICE O2. This software is distributed
-# under the terms of the GNU General Public License v3 (GPL Version 3), copied
-# verbatim in the file "COPYING".
+# Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+# See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+# All rights not expressly granted are reserved.
 #
-# See http://alice-o2.web.cern.ch/license for full licensing information.
+# This software is distributed under the terms of the GNU General Public
+# License v3 (GPL Version 3), copied verbatim in the file "COPYING".
 #
 # In applying this license CERN does not waive the privileges and immunities
-# granted to it by virtue of its status as an Intergovernmental Organization or
-# submit itself to any jurisdiction.
+# granted to it by virtue of its status as an Intergovernmental Organization
+# or submit itself to any jurisdiction.
 
 o2_add_library(CTPWorkflowIO
-               SOURCES src/DigitReaderSpec.cxx
-                       src/DigitWriterSpec.cxx
-               PUBLIC_LINK_LIBRARIES O2::Framework
-                                     O2::DataFormatsCTP
-                                     O2::DPLUtils
-                                     O2::DetectorsRaw
-                                     O2::Algorithm)
+        SOURCES src/DigitReaderSpec.cxx
+        src/DigitWriterSpec.cxx
+        PUBLIC_LINK_LIBRARIES O2::Framework
+        O2::DataFormatsCTP
+        O2::DPLUtils
+        O2::DetectorsRaw
+        O2::Algorithm)
+
+o2_add_executable(digit-writer
+        COMPONENT_NAME ctp
+        SOURCES src/digits-writer-workflow.cxx
+        PUBLIC_LINK_LIBRARIES O2::CTPWorkflowIO)
+
+o2_add_executable(digit-reader
+        COMPONENT_NAME ctp
+        SOURCES src/digits-reader-workflow.cxx
+        PUBLIC_LINK_LIBRARIES O2::CTPWorkflowIO)
\ No newline at end of file
diff --git a/Detectors/CTP/workflowIO/include/CTPWorkflowIO/DigitReaderSpec.h b/Detectors/CTP/workflowIO/include/CTPWorkflowIO/DigitReaderSpec.h
index d5e990b216849..bb7d3559d2e82 100644
--- a/Detectors/CTP/workflowIO/include/CTPWorkflowIO/DigitReaderSpec.h
+++ b/Detectors/CTP/workflowIO/include/CTPWorkflowIO/DigitReaderSpec.h
@@ -20,7 +20,7 @@ namespace o2
 namespace ctp
 {
 
-framework::DataProcessorSpec getDigitsReaderSpec(bool propagateMC = true);
+framework::DataProcessorSpec getDigitsReaderSpec(bool propagateMC = true, const std::string& defFile = "ctpdigits.root");
 
 } // namespace ctp
 } // end namespace o2
diff --git a/Detectors/CTP/workflowIO/include/CTPWorkflowIO/DigitWriterSpec.h b/Detectors/CTP/workflowIO/include/CTPWorkflowIO/DigitWriterSpec.h
index d03b6a95ff72a..f5f0fa195eff8 100644
--- a/Detectors/CTP/workflowIO/include/CTPWorkflowIO/DigitWriterSpec.h
+++ b/Detectors/CTP/workflowIO/include/CTPWorkflowIO/DigitWriterSpec.h
@@ -16,9 +16,6 @@
 #define O2_CTPDIGITWRITERSPEC_H
 
 #include "Framework/DataProcessorSpec.h"
-#include "DPLUtils/MakeRootTreeWriterSpec.h"
-#include "Framework/InputSpec.h"
-#include "DataFormatsCTP/Digits.h"
 
 using namespace o2::framework;
 namespace o2
diff --git a/Detectors/CTP/workflowIO/src/DigitReaderSpec.cxx b/Detectors/CTP/workflowIO/src/DigitReaderSpec.cxx
index 1357b6b4c9203..81e6f53f42dcc 100644
--- a/Detectors/CTP/workflowIO/src/DigitReaderSpec.cxx
+++ b/Detectors/CTP/workflowIO/src/DigitReaderSpec.cxx
@@ -14,9 +14,13 @@
 #include "TFile.h"
 #include "TTree.h"
 #include "DataFormatsCTP/Digits.h"
+#include "DataFormatsCTP/LumiInfo.h"
 #include "Headers/DataHeader.h"
 #include "DetectorsCommonDataFormats/DetID.h"
+#include "SimulationDataFormat/MCCompLabel.h"
+#include "SimulationDataFormat/ConstMCTruthContainer.h"
 #include "CommonUtils/NameConf.h"
+#include "CommonUtils/IRFrameSelector.h"
 #include "Framework/DataProcessorSpec.h"
 #include "Framework/Task.h"
 #include "Framework/ControlService.h"
@@ -44,18 +48,21 @@ class DigitReader : public Task
   void connectTree(const std::string& filename);
 
   std::vector mDigits, *mDigitsPtr = &mDigits;
+  o2::ctp::LumiInfo mLumi, *mLumiPtr = &mLumi;
   std::unique_ptr mFile;
   std::unique_ptr mTree;
 
   bool mUseMC = false; // use MC truth
+  bool mUseIRFrames = false; // selected IRFrames mode
   std::string mDigTreeName = "o2sim";
   std::string mDigitBranchName = "CTPDigits";
+  std::string mLumiBranchName = "CTPLumi";
 };
 
 DigitReader::DigitReader(bool useMC)
 {
   if (useMC) {
-    LOG(info) << "CTP does not support MC truth at the moment";
+    LOG(info) << "CTP : truth = data as CTP inputs are already digital";
   }
 }
 
@@ -63,20 +70,70 @@ void DigitReader::init(InitContext& ic)
 {
   auto filename = o2::utils::Str::concat_string(o2::utils::Str::rectifyDirectory(ic.options().get("input-dir")),
                                                 ic.options().get("ctp-digit-infile"));
+  if (ic.options().hasOption("ignore-irframes") && !ic.options().get("ignore-irframes")) {
+    mUseIRFrames = true;
+  }
   connectTree(filename);
 }
 
 void DigitReader::run(ProcessingContext& pc)
 {
-  auto ent = mTree->GetReadEntry() + 1;
-  assert(ent < mTree->GetEntries()); // this should not happen
-
-  mTree->GetEntry(ent);
-  LOG(info) << "DigitReader pushes " << mDigits.size() << " digits at entry " << ent;
-  pc.outputs().snapshot(Output{"CTP", "DIGITS", 0, Lifetime::Timeframe}, mDigits);
-  if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
-    pc.services().get().endOfStream();
-    pc.services().get().readyToQuit(QuitRequest::Me);
+  gsl::span irFrames{};
+  // LOG(info) << "Using IRs:" << mUseIRFrames;
+  if (mUseIRFrames) {
+    irFrames = pc.inputs().get>("driverInfo");
+  }
+  auto ent = mTree->GetReadEntry();
+  if (!mUseIRFrames) {
+    ent++;
+    assert(ent < mTree->GetEntries()); // this should not happen
+    mTree->GetEntry(ent);
+    LOG(info) << "DigitReader pushes " << mDigits.size() << " digits at entry " << ent;
+    pc.outputs().snapshot(Output{"CTP", "DIGITS", 0}, mDigits);
+    pc.outputs().snapshot(Output{"CTP", "LUMI", 0}, mLumi);
+    if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
+      pc.services().get().endOfStream();
+      pc.services().get().readyToQuit(QuitRequest::Me);
+    }
+  } else {
+    std::vector digitSel;
+    if (irFrames.size()) { // we assume the IRFrames are in the increasing order
+      if (ent < 0) {
+        ent++;
+      }
+      o2::utils::IRFrameSelector irfSel;
+      // MC  digits are already aligned
+      irfSel.setSelectedIRFrames(irFrames, 0, 0, 0, true);
+      const auto irMin = irfSel.getIRFrames().front().getMin(); // use processed IRframes for rough comparisons (possible shift!)
+      const auto irMax = irfSel.getIRFrames().back().getMax();
+      LOGP(info, "Selecting IRFrame {}-{}", irMin.asString(), irMax.asString());
+      while (ent < mTree->GetEntries()) {
+        if (ent > mTree->GetReadEntry()) {
+          mTree->GetEntry(ent);
+        }
+        if (mDigits.front().intRecord <= irMax && mDigits.back().intRecord >= irMin) { // THere is overlap
+          for (int i = 0; i < (int)mDigits.size(); i++) {
+            const auto& dig = mDigits[i];
+            // if(irfSel.check(dig.intRecord)) { // adding selected digit
+            if (dig.intRecord >= irMin && dig.intRecord <= irMax) {
+              digitSel.push_back(dig);
+              LOG(info) << "adding:" << dig.intRecord << " ent:" << ent;
+            }
+          }
+        }
+        if (mDigits.back().intRecord < irMax) { // need to check the next entry
+          ent++;
+          continue;
+        }
+        break; // push collected data
+      }
+    }
+    pc.outputs().snapshot(Output{"CTP", "DIGITS", 0}, digitSel);
+    pc.outputs().snapshot(Output{"CTP", "LUMI", 0}, mLumi); // add full lumi for this TF
+    if (!irFrames.size() || irFrames.back().isLast()) {
+      pc.services().get().endOfStream();
+      pc.services().get().readyToQuit(QuitRequest::Me);
+    }
   }
 }
 
@@ -87,19 +144,30 @@ void DigitReader::connectTree(const std::string& filename)
   assert(mFile && !mFile->IsZombie());
   mTree.reset((TTree*)mFile->Get(mDigTreeName.c_str()));
   assert(mTree);
+  if (mTree->GetBranch(mDigitBranchName.c_str())) {
+    mTree->SetBranchAddress(mDigitBranchName.c_str(), &mDigitsPtr);
+  } else {
+    LOGP(warn, "Digits branch {} is absent", mDigitBranchName);
+  }
+  if (mTree->GetBranch(mLumiBranchName.c_str())) {
+    mTree->SetBranchAddress(mLumiBranchName.c_str(), &mLumiPtr);
+  } else {
+    LOGP(warn, "Lumi branch {} is absent", mLumiBranchName);
+  }
   mTree->SetBranchAddress(mDigitBranchName.c_str(), &mDigitsPtr);
   LOG(info) << "Loaded tree from " << filename << " with " << mTree->GetEntries() << " entries";
 }
 
-DataProcessorSpec getDigitsReaderSpec(bool useMC)
+DataProcessorSpec getDigitsReaderSpec(bool useMC, const std::string& defFile)
 {
   return DataProcessorSpec{
     "ctp-digit-reader",
     Inputs{},
-    Outputs{{"CTP", "DIGITS", 0, Lifetime::Timeframe}},
+    Outputs{{"CTP", "DIGITS", 0, Lifetime::Timeframe},
+            {"CTP", "LUMI", 0, o2::framework::Lifetime::Timeframe}},
     AlgorithmSpec{adaptFromTask(useMC)},
     Options{
-      {"ctp-digit-infile", VariantType::String, "ctpdigits.root", {"Name of the input digit file"}},
+      {"ctp-digit-infile", VariantType::String, defFile, {"Name of the input digit file"}},
       {"input-dir", VariantType::String, "none", {"Input directory"}}}};
 }
 
diff --git a/Detectors/CTP/workflowIO/src/DigitWriterSpec.cxx b/Detectors/CTP/workflowIO/src/DigitWriterSpec.cxx
index 072ffa387679e..a65b94d16f43c 100644
--- a/Detectors/CTP/workflowIO/src/DigitWriterSpec.cxx
+++ b/Detectors/CTP/workflowIO/src/DigitWriterSpec.cxx
@@ -13,6 +13,10 @@
 /// \author Roman Lietava
 
 #include "CTPWorkflowIO/DigitWriterSpec.h"
+#include "DPLUtils/MakeRootTreeWriterSpec.h"
+#include "Framework/InputSpec.h"
+#include "DataFormatsCTP/Digits.h"
+#include "DataFormatsCTP/LumiInfo.h"
 
 namespace o2
 {
@@ -29,8 +33,14 @@ framework::DataProcessorSpec getDigitWriterSpec(bool raw)
   auto logger = [](std::vector const& vecDigits) {
     LOG(info) << "CTPDigitWriter pulled " << vecDigits.size() << " digits";
   };
-  return MakeRootTreeWriterSpec(raw ? "ctp-digit-writer-dec" : "ctp-digit-writer",
-                                raw ? "o2_ctpdigits.root" : "ctpdigits.root",
+  if (raw) {
+    return MakeRootTreeWriterSpec("ctp-digit-writer-dec", "ctpdigits.root",
+                                  MakeRootTreeWriterSpec::TreeAttributes{"o2sim", "Tree with CTP digits/Lumi"},
+                                  BranchDefinition>{InputSpec{"digit", "CTP", "DIGITS", 0}, "CTPDigits", logger},
+                                  BranchDefinition{InputSpec{"CTPLumi", "CTP", "LUMI", 0}, "CTPLumi"})();
+  }
+  // MC digits case, no lumi available
+  return MakeRootTreeWriterSpec("ctp-digit-writer", "ctpdigits.root",
                                 MakeRootTreeWriterSpec::TreeAttributes{"o2sim", "Tree with CTP digits"},
                                 BranchDefinition>{InputSpec{"digit", "CTP", "DIGITS", 0}, "CTPDigits", logger})();
 }
diff --git a/Detectors/CTP/workflowIO/src/digits-reader-workflow.cxx b/Detectors/CTP/workflowIO/src/digits-reader-workflow.cxx
new file mode 100644
index 0000000000000..fe9104fcf9978
--- /dev/null
+++ b/Detectors/CTP/workflowIO/src/digits-reader-workflow.cxx
@@ -0,0 +1,48 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "CommonUtils/ConfigurableParam.h"
+#include "Framework/ConfigParamSpec.h"
+#include "Framework/CompletionPolicyHelpers.h"
+#include "DetectorsRaw/HBFUtilsInitializer.h"
+
+using namespace o2::framework;
+
+void customize(std::vector& policies)
+{
+  o2::raw::HBFUtilsInitializer::addNewTimeSliceCallback(policies);
+}
+
+// we need to add workflow options before including Framework/runDataProcessing
+void customize(std::vector& workflowOptions)
+{
+  // option allowing to set parameters
+  std::vector options{
+    {"inputfile", o2::framework::VariantType::String, "ctpdigits.root", {"File with CTP digits"}},
+    {"disable-mc", o2::framework::VariantType::Bool, false, {"Do not reed MC truth branch"}},
+    {"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}}};
+  o2::raw::HBFUtilsInitializer::addConfigOption(options);
+  std::swap(workflowOptions, options);
+}
+
+#include "Framework/runDataProcessing.h"
+#include "Framework/WorkflowSpec.h"
+#include "CTPWorkflowIO/DigitReaderSpec.h"
+
+WorkflowSpec defineDataProcessing(ConfigContext const& configcontext)
+{
+  o2::conf::ConfigurableParam::updateFromString(configcontext.options().get("configKeyValues"));
+  WorkflowSpec specs;
+  specs.emplace_back(o2::ctp::getDigitsReaderSpec(!configcontext.options().get("disable-mc"), configcontext.options().get("inputfile")));
+  // configure dpl timer to inject correct firstTForbit: start from the 1st orbit of TF containing 1st sampled orbit
+  o2::raw::HBFUtilsInitializer hbfIni(configcontext, specs);
+  return std::move(specs);
+}
diff --git a/Detectors/CTP/workflowIO/src/digits-writer-workflow.cxx b/Detectors/CTP/workflowIO/src/digits-writer-workflow.cxx
new file mode 100644
index 0000000000000..c7bd7e846629d
--- /dev/null
+++ b/Detectors/CTP/workflowIO/src/digits-writer-workflow.cxx
@@ -0,0 +1,44 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "CommonUtils/ConfigurableParam.h"
+#include "Framework/ConfigParamSpec.h"
+#include "Framework/CompletionPolicyHelpers.h"
+
+using namespace o2::framework;
+
+// we need to add workflow options before including Framework/runDataProcessing
+void customize(std::vector& workflowOptions)
+{
+  // option allowing to set parameters
+  std::vector options{
+    {"no-lumi", o2::framework::VariantType::Bool, false, {"write only digits as from MC instead of digits/lumi of raw mode"}},
+    {"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}}};
+  std::swap(workflowOptions, options);
+}
+
+void customize(std::vector& policies)
+{
+  // ordered policies for the writers
+  policies.push_back(CompletionPolicyHelpers::consumeWhenAllOrdered(".*(?:CTP|ctp).*[W,w]riter.*"));
+}
+
+#include "Framework/runDataProcessing.h"
+#include "Framework/WorkflowSpec.h"
+#include "CTPWorkflowIO/DigitWriterSpec.h"
+
+WorkflowSpec defineDataProcessing(ConfigContext const& configcontext)
+{
+  o2::conf::ConfigurableParam::updateFromString(configcontext.options().get("configKeyValues"));
+  WorkflowSpec specs;
+  specs.emplace_back(o2::ctp::getDigitWriterSpec(!configcontext.options().get("no-lumi")));
+  return std::move(specs);
+}
diff --git a/Detectors/CTP/workflowScalers/CMakeLists.txt b/Detectors/CTP/workflowScalers/CMakeLists.txt
index 716da7e596482..f02a7f33e2abd 100644
--- a/Detectors/CTP/workflowScalers/CMakeLists.txt
+++ b/Detectors/CTP/workflowScalers/CMakeLists.txt
@@ -1,12 +1,44 @@
+# Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+# See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+# All rights not expressly granted are reserved.
+#
+# This software is distributed under the terms of the GNU General Public
+# License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+#
+# In applying this license CERN does not waive the privileges and immunities
+# granted to it by virtue of its status as an Intergovernmental Organization
+# or submit itself to any jurisdiction.
+o2_add_library(CTPWorkflowScalers
+               SOURCES src/ctpCCDBManager.cxx
+               SOURCES src/RunManager.cxx
+               PUBLIC_LINK_LIBRARIES O2::Framework
+                                     O2::DataFormatsCTP
+                                    AliceO2::BookkeepingApi)
+o2_target_root_dictionary(CTPWorkflowScalers HEADERS
+  include/CTPWorkflowScalers/ctpCCDBManager.h)
 o2_add_executable(
                   proxy
                   COMPONENT_NAME ctp
                   SOURCES src/ctp-proxy.cxx
                   PUBLIC_LINK_LIBRARIES O2::DCStestWorkflow
-                  O2::DataFormatsCTP)
+                  O2::CTPWorkflowScalers)
 o2_add_executable(
                   qc-proxy
                   COMPONENT_NAME ctp
                   SOURCES src/ctp-qc-proxy.cxx
                   PUBLIC_LINK_LIBRARIES O2::DCStestWorkflow
                   O2::DataFormatsCTP)
+o2_add_executable(
+                  ccdb-orbit
+                  COMPONENT_NAME ctp
+                  SOURCES src/ctp-ccdb-orbit.cxx
+                  PUBLIC_LINK_LIBRARIES O2::DataFormatsCTP
+                  Boost::program_options)
+o2_add_executable(
+                  bk-write
+                  COMPONENT_NAME ctp
+                  SOURCES src/ctp-bk-write.cxx
+                  PUBLIC_LINK_LIBRARIES O2::DataFormatsCTP
+                  O2::CTPWorkflowScalers
+                  AliceO2::BookkeepingApi
+                  Boost::program_options)
diff --git a/Detectors/CTP/workflowScalers/README.md b/Detectors/CTP/workflowScalers/README.md
index 4b109f779dc1d..df8df2c55cc44 100644
--- a/Detectors/CTP/workflowScalers/README.md
+++ b/Detectors/CTP/workflowScalers/README.md
@@ -1,3 +1,7 @@
+
+
 How to generate c++ proto files ?
 On the computer where project
 https://gitlab.cern.ch/aliceCTP3/ctp3-ipbus/-/tree/master
diff --git a/Detectors/CTP/workflowScalers/include/CTPWorkflowScalers/RunManager.h b/Detectors/CTP/workflowScalers/include/CTPWorkflowScalers/RunManager.h
new file mode 100644
index 0000000000000..6d2172e3da165
--- /dev/null
+++ b/Detectors/CTP/workflowScalers/include/CTPWorkflowScalers/RunManager.h
@@ -0,0 +1,79 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file RunManager.h
+/// \brief Managing runs for config and scalers
+/// \author Roman Lietava
+#ifndef _CTP_RUNMANAGER_H_
+#define _CTP_RUNMANAGER_H_
+#include "CTPWorkflowScalers/ctpCCDBManager.h"
+#include "DataFormatsCTP/Configuration.h"
+#include "BookkeepingApi/BkpClientFactory.h"
+#include "BookkeepingApi/BkpClient.h"
+
+using namespace o2::bkp::api;
+namespace o2
+{
+namespace ctp
+{
+typedef std::map> counters_t;
+typedef std::map> counters64_t;
+struct CTPActiveRun {
+  CTPActiveRun() = default;
+  long timeStart;
+  long timeStop;
+  CTPConfiguration cfg;
+  CTPRunScalers scalers;
+  void initBK();
+  int send2BK(std::unique_ptr& BKClient, size_t ts, bool start);
+  //
+  counters_t cnts0;     // first counters in run
+  counters_t cntslast0; // last minus one read counters needed for overflow correction
+  counters_t cntslast;  // last read counters
+  counters64_t overflows;
+  // QC
+  int qcwpcount = 0;
+};
+class CTPRunManager : public ctpCCDBManager
+{
+ public:
+  CTPRunManager() = default;
+  void init();
+  int loadRun(const std::string& cfg);
+  int setRunConfigBK(uint32_t runNumber, const std::string& cfg);
+  int stopRun(uint32_t irun, long timeStamp);
+  int addScalers(uint32_t irun, std::time_t time, bool start = 0);
+  int processMessage(std::string& topic, const std::string& message);
+  void printActiveRuns() const;
+  int loadScalerNames();
+  int getNRuns();
+  void setBKHost(std::string host) { mBKHost = host; };
+  void setQCWritePeriod(int period) { mQCWritePeriod = period; };
+  uint64_t checkOverflow(uint32_t lcnt0, uint32_t lcnt1, uint64_t lcntcor);
+  void printCounters();
+
+ private:
+  /// Database constants
+  std::string mBKHost = "";
+  std::array mActiveRuns;
+  std::array mActiveRunNumbers;
+  std::array mCounters;
+  std::map mScalerName2Position;
+  std::map mRunsLoaded;
+  std::unique_ptr mBKClient;
+  int mEOX = 0; // redundancy check
+  int mNew = 1; // 1 - no CCDB: used for QC
+  int mQCWritePeriod = 3; // Time in 10secs between two writes to QCCD
+  ClassDefNV(CTPRunManager, 8);
+};
+} // namespace ctp
+} // namespace o2
+#endif //_CTP_RUNMANAGER_H_
diff --git a/Detectors/CTP/workflowScalers/include/CTPWorkflowScalers/ctpCCDBManager.h b/Detectors/CTP/workflowScalers/include/CTPWorkflowScalers/ctpCCDBManager.h
new file mode 100644
index 0000000000000..df2aa79d18697
--- /dev/null
+++ b/Detectors/CTP/workflowScalers/include/CTPWorkflowScalers/ctpCCDBManager.h
@@ -0,0 +1,59 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file RunManager.h
+/// \brief Managing runs for config and scalers
+/// \author Roman Lietava
+#ifndef _CTP_CTPCCDB_H_
+#define _CTP_CTPCCDB_H_
+#include "DataFormatsCTP/Configuration.h"
+
+namespace o2
+{
+namespace ctp
+{
+class ctpCCDBManager
+{
+ public:
+  ctpCCDBManager() = default;
+  int saveRunScalersToCCDB(CTPRunScalers& scalers, long timeStart, long timeStop);
+  int saveRunScalersToQCDB(CTPRunScalers& scalers, long timeStart, long timeStop);
+  int saveRunConfigToCCDB(CTPConfiguration* cfg, long timeStart);
+  int saveSoxOrbit(uint32_t runNumber, uint32_t soxOrbit, long timeStart);
+  int saveOrbitReset(long timeStamp);
+  int saveCtpCfg(uint32_t runNumber, long timeStamp);
+  static CTPConfiguration getConfigFromCCDB(long timestamp, std::string run, bool& ok);
+  CTPConfiguration getConfigFromCCDB(long timestamp, std::string run);
+  CTPRunScalers getScalersFromCCDB(long timestamp, std::string run, bool& ok);
+  static CTPRunScalers getScalersFromCCDB(long timestamp, std::string, std::string path, bool& ok);
+  static void setCCDBHost(std::string host) { mCCDBHost = host; };
+  static void setQCDBHost(std::string host) { mQCDBHost = host; };
+  void setCtpCfgDir(std::string& ctpcfgdir) { mCtpCfgDir = ctpcfgdir; };
+
+ protected:
+  /// Database constants
+  // std::string mCCDBHost = "http://ccdb-test.cern.ch:8080";
+  // std::string mQCDBHost = "http://ali-qcdb.cern.ch:8083";
+  static std::string mCCDBHost;
+  static std::string mQCDBHost;
+  const std::string mCCDBPathCTPScalers = "CTP/Calib/Scalers";
+  // std::string mCCDBPathCTPConfig = "CTP/Config/Config";  - in Configuration.h
+  const std::string mQCDBPathCTPScalers = "qc/CTP/Scalers";
+  const std::string mCCDBPathSoxOrbit = "CTP/Calib/FirstRunOrbit";
+  const std::string mCCDBPathOrbitReset = "CTP/Calib/OrbitReset";
+  const std::string mCCDBPathCtpCfg = "CTP/Config/CtpCfg";
+  std::string mCtpCfgDir;
+
+  ClassDefNV(ctpCCDBManager, 2);
+};
+} // namespace ctp
+} // namespace o2
+#endif //_CTP_CTPCCDB_H_
diff --git a/Detectors/CTP/workflowScalers/py/createCnts.py b/Detectors/CTP/workflowScalers/py/createCnts.py
new file mode 100644
index 0000000000000..4773065cfca43
--- /dev/null
+++ b/Detectors/CTP/workflowScalers/py/createCnts.py
@@ -0,0 +1,99 @@
+import zmq
+import random
+import sys
+import time
+port = "500901"
+context = zmq.Context()
+socket = context.socket(zmq.PUB)
+socket.bind("tcp://*:%s" % port)
+time.sleep(1)
+#
+path = "/home/rl/countersLHCo/"
+#
+run = "527345"
+filename = path + "20221014.cc"
+#
+#run = "528543"
+#run = "528537"
+#filename = path + "20221102.cc"
+#
+#run = "527963"
+#filename = path + "20221024.cc"
+#
+#run = "527349"
+#filename = path + "20221015.cc"
+#
+filecfg = path + run + ".rcfg"
+#
+if len(sys.argv) == 2:
+  run =  sys.argv[1]
+  int(run)
+elif len(sys.argv) == 3:
+  port =  sys.argv[1]
+  int(port)
+  print("port:", port)
+print("run:", run)
+#
+
+# CTP Config
+def sendctpconfig(starttime):
+  print("starttime:",starttime)
+  fcfg = open(filecfg,"r")
+  lines = fcfg.readlines()
+  ctpcfg = starttime+" "
+  for line in lines:
+    ctpcfg += line
+  fcfg.close()
+  print(ctpcfg)
+  senddata("ctpconfig",ctpcfg)
+def senddata(header, messagedata):
+  global socket
+  data = messagedata
+  if len(data) > 20:
+    data = data[0:20]
+  print("Sending:",header, data)
+  data = str(messagedata).encode('UTF-8')
+  header = str(header).encode('UTF-8')
+  test = str("test").encode('UTF-8')
+  msg = [header, data,test]
+  socket.send_multipart(msg)
+  time.sleep(1)
+##########################
+#
+f = open(filename,"r")
+#
+n = 0
+start = time.time()
+runcnts = []
+runactive = 0
+while True:
+    line = f.readline()
+    if not line:
+      break
+    items = line.split(" ")
+    runfound = 0
+    for i in range(1,17):
+      if items[i] == run:
+        runcnts.append(line)
+        runfound = 1
+        print("1:",line[0:20])
+        break;
+    if (runfound == 1) and (runactive == 0):
+      runactive = 1
+    if (runfound == 0) and (runactive==1):
+      runcnts.append(line)
+      runactive = 0
+      print("0:",line[0:20])
+
+print("runcnts size:", len(runcnts))
+starttime = runcnts[0].split(" ",1)[0]
+#
+sendctpconfig(starttime)
+time.sleep(5)
+senddata("sox",runcnts[0])
+for line in runcnts[1:-1]:
+  senddata("ctpd",line)
+senddata("eox",runcnts[-1])
+
+
+
diff --git a/Detectors/CTP/workflowScalers/py/pub_server.py b/Detectors/CTP/workflowScalers/py/pub_server.py
index c1e3dc3b407b9..fb26d2e3929f1 100644
--- a/Detectors/CTP/workflowScalers/py/pub_server.py
+++ b/Detectors/CTP/workflowScalers/py/pub_server.py
@@ -3,7 +3,7 @@
 import sys
 import time
 
-port = "500901"
+port = "50090"
 if len(sys.argv) > 1:
     port =  sys.argv[1]
     int(port)
@@ -11,14 +11,29 @@
 context = zmq.Context()
 socket = context.socket(zmq.PUB)
 socket.bind("tcp://*:%s" % port)
+time.sleep(1)
+#
+topic = random.randrange(0,2**32)
+header = "CTP"
+messagedata = str(topic)+"1111111111111111st"
+tag = "tag"
+print("Sending:",header, messagedata)
+data = str(messagedata).encode()
+header = str(header).encode()
+tag = str(tag).encode()
+socket.send(memoryview(header),zmq.SNDMORE)
+socket.send(memoryview(data),zmq.SNDMORE)
+socket.send(memoryview(tag))
+time.sleep(1)
 while True:
     topic = random.randrange(0,2**32)
-    #messagedata = random.randrange(1,215) - 80
     header = "CTP"
-    messagedata = str(topic)+"1 2 3 4 5"
-    print("Sending:",header, messagedata)
+    #messagedata = str(topic)+"1 2 3 4 5+6 uuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuukkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk"
+    messagedata = "x "*1072
+    #print("Sending:",header, messagedata)
     data = str(messagedata).encode()
     header = str(header).encode()
     socket.send(memoryview(header),zmq.SNDMORE)
-    socket.send(memoryview(data))
+    socket.send(memoryview(data),zmq.SNDMORE)
+    socket.send(memoryview(tag))
     time.sleep(1)
diff --git a/Detectors/CTP/workflowScalers/py/scalersCTP_v2.txt b/Detectors/CTP/workflowScalers/py/scalersCTP_v2.txt
new file mode 100644
index 0000000000000..812b937f3f600
--- /dev/null
+++ b/Detectors/CTP/workflowScalers/py/scalersCTP_v2.txt
@@ -0,0 +1,1071 @@
+names: "runn0"
+names: "runn1"
+names: "runn2"
+names: "runn3"
+names: "runn4"
+names: "runn5"
+names: "runn6"
+names: "runn7"
+names: "runn8"
+names: "runn9"
+names: "runn10"
+names: "runn11"
+names: "runn12"
+names: "runn13"
+names: "runn14"
+names: "runn15"
+names: "ltg1_ORB"
+names: "ltg1_HB"
+names: "ltg1_HBr"
+names: "ltg1_HC"
+names: "ltg1_PH"
+names: "ltg1_PP"
+names: "ltg1_CAL"
+names: "ltg1_SOT"
+names: "ltg1_EOT"
+names: "ltg1_SOC"
+names: "ltg1_EOC"
+names: "ltg1_TF"
+names: "ltg1_FERST"
+names: "ltg1_RT"
+names: "ltg1_RS"
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: "ltg1_GAP1"
+names: "ltg1_GAP2"
+names: "ltg1_TPC_sync"
+names: "ltg1_TPC_rst"
+names: "ltg1_TOF"
+names: "ltg2_ORB"
+names: "ltg2_HB"
+names: "ltg2_HBr"
+names: "ltg2_HC"
+names: "ltg2_PH"
+names: "ltg2_PP"
+names: "ltg2_CAL"
+names: "ltg2_SOT"
+names: "ltg2_EOT"
+names: "ltg2_SOC"
+names: "ltg2_EOC"
+names: "ltg2_TF"
+names: "ltg2_FERST"
+names: "ltg2_RT"
+names: "ltg2_RS"
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: "ltg2_GAP1"
+names: "ltg2_GAP2"
+names: "ltg2_TPC_sync"
+names: "ltg2_TPC_rst"
+names: "ltg2_TOF"
+names: "ltg3_ORB"
+names: "ltg3_HB"
+names: "ltg3_HBr"
+names: "ltg3_HC"
+names: "ltg3_PH"
+names: "ltg3_PP"
+names: "ltg3_CAL"
+names: "ltg3_SOT"
+names: "ltg3_EOT"
+names: "ltg3_SOC"
+names: "ltg3_EOC"
+names: "ltg3_TF"
+names: "ltg3_FERST"
+names: "ltg3_RT"
+names: "ltg3_RS"
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: "ltg3_GAP1"
+names: "ltg3_GAP2"
+names: "ltg3_TPC_sync"
+names: "ltg3_TPC_rst"
+names: "ltg3_TOF"
+names: "ltg4_ORB"
+names: "ltg4_HB"
+names: "ltg4_HBr"
+names: "ltg4_HC"
+names: "ltg4_PH"
+names: "ltg4_PP"
+names: "ltg4_CAL"
+names: "ltg4_SOT"
+names: "ltg4_EOT"
+names: "ltg4_SOC"
+names: "ltg4_EOC"
+names: "ltg4_TF"
+names: "ltg4_FERST"
+names: "ltg4_RT"
+names: "ltg4_RS"
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: "ltg4_GAP1"
+names: "ltg4_GAP2"
+names: "ltg4_TPC_sync"
+names: "ltg4_TPC_rst"
+names: "ltg4_TOF"
+names: "ltg5_ORB"
+names: "ltg5_HB"
+names: "ltg5_HBr"
+names: "ltg5_HC"
+names: "ltg5_PH"
+names: "ltg5_PP"
+names: "ltg5_CAL"
+names: "ltg5_SOT"
+names: "ltg5_EOT"
+names: "ltg5_SOC"
+names: "ltg5_EOC"
+names: "ltg5_TF"
+names: "ltg5_FERST"
+names: "ltg5_RT"
+names: "ltg5_RS"
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: "ltg5_GAP1"
+names: "ltg5_GAP2"
+names: "ltg5_TPC_sync"
+names: "ltg5_TPC_rst"
+names: "ltg5_TOF"
+names: "ltg6_ORB"
+names: "ltg6_HB"
+names: "ltg6_HBr"
+names: "ltg6_HC"
+names: "ltg6_PH"
+names: "ltg6_PP"
+names: "ltg6_CAL"
+names: "ltg6_SOT"
+names: "ltg6_EOT"
+names: "ltg6_SOC"
+names: "ltg6_EOC"
+names: "ltg6_TF"
+names: "ltg6_FERST"
+names: "ltg6_RT"
+names: "ltg6_RS"
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: "ltg6_GAP1"
+names: "ltg6_GAP2"
+names: "ltg6_TPC_sync"
+names: "ltg6_TPC_rst"
+names: "ltg6_TOF"
+names: "ltg7_ORB"
+names: "ltg7_HB"
+names: "ltg7_HBr"
+names: "ltg7_HC"
+names: "ltg7_PH"
+names: "ltg7_PP"
+names: "ltg7_CAL"
+names: "ltg7_SOT"
+names: "ltg7_EOT"
+names: "ltg7_SOC"
+names: "ltg7_EOC"
+names: "ltg7_TF"
+names: "ltg7_FERST"
+names: "ltg7_RT"
+names: "ltg7_RS"
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: "ltg7_GAP1"
+names: "ltg7_GAP2"
+names: "ltg7_TPC_sync"
+names: "ltg7_TPC_rst"
+names: "ltg7_TOF"
+names: "ltg8_ORB"
+names: "ltg8_HB"
+names: "ltg8_HBr"
+names: "ltg8_HC"
+names: "ltg8_PH"
+names: "ltg8_PP"
+names: "ltg8_CAL"
+names: "ltg8_SOT"
+names: "ltg8_EOT"
+names: "ltg8_SOC"
+names: "ltg8_EOC"
+names: "ltg8_TF"
+names: "ltg8_FERST"
+names: "ltg8_RT"
+names: "ltg8_RS"
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: "ltg8_GAP1"
+names: "ltg8_GAP2"
+names: "ltg8_TPC_sync"
+names: "ltg8_TPC_rst"
+names: "ltg8_TOF"
+names: "ltg9_ORB"
+names: "ltg9_HB"
+names: "ltg9_HBr"
+names: "ltg9_HC"
+names: "ltg9_PH"
+names: "ltg9_PP"
+names: "ltg9_CAL"
+names: "ltg9_SOT"
+names: "ltg9_EOT"
+names: "ltg9_SOC"
+names: "ltg9_EOC"
+names: "ltg9_TF"
+names: "ltg9_FERST"
+names: "ltg9_RT"
+names: "ltg9_RS"
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: "ltg9_GAP1"
+names: "ltg9_GAP2"
+names: "ltg9_TPC_sync"
+names: "ltg9_TPC_rst"
+names: "ltg9_TOF"
+names: "ltg10_ORB"
+names: "ltg10_HB"
+names: "ltg10_HBr"
+names: "ltg10_HC"
+names: "ltg10_PH"
+names: "ltg10_PP"
+names: "ltg10_CAL"
+names: "ltg10_SOT"
+names: "ltg10_EOT"
+names: "ltg10_SOC"
+names: "ltg10_EOC"
+names: "ltg10_TF"
+names: "ltg10_FERST"
+names: "ltg10_RT"
+names: "ltg10_RS"
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: "ltg10_GAP1"
+names: "ltg10_GAP2"
+names: "ltg10_TPC_sync"
+names: "ltg10_TPC_rst"
+names: "ltg10_TOF"
+names: "ltg11_ORB"
+names: "ltg11_HB"
+names: "ltg11_HBr"
+names: "ltg11_HC"
+names: "ltg11_PH"
+names: "ltg11_PP"
+names: "ltg11_CAL"
+names: "ltg11_SOT"
+names: "ltg11_EOT"
+names: "ltg11_SOC"
+names: "ltg11_EOC"
+names: "ltg11_TF"
+names: "ltg11_FERST"
+names: "ltg11_RT"
+names: "ltg11_RS"
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: "ltg11_GAP1"
+names: "ltg11_GAP2"
+names: "ltg11_TPC_sync"
+names: "ltg11_TPC_rst"
+names: "ltg11_TOF"
+names: "ltg12_ORB"
+names: "ltg12_HB"
+names: "ltg12_HBr"
+names: "ltg12_HC"
+names: "ltg12_PH"
+names: "ltg12_PP"
+names: "ltg12_CAL"
+names: "ltg12_SOT"
+names: "ltg12_EOT"
+names: "ltg12_SOC"
+names: "ltg12_EOC"
+names: "ltg12_TF"
+names: "ltg12_FERST"
+names: "ltg12_RT"
+names: "ltg12_RS"
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: "ltg12_GAP1"
+names: "ltg12_GAP2"
+names: "ltg12_TPC_sync"
+names: "ltg12_TPC_rst"
+names: "ltg12_TOF"
+names: "ltg13_ORB"
+names: "ltg13_HB"
+names: "ltg13_HBr"
+names: "ltg13_HC"
+names: "ltg13_PH"
+names: "ltg13_PP"
+names: "ltg13_CAL"
+names: "ltg13_SOT"
+names: "ltg13_EOT"
+names: "ltg13_SOC"
+names: "ltg13_EOC"
+names: "ltg13_TF"
+names: "ltg13_FERST"
+names: "ltg13_RT"
+names: "ltg13_RS"
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: "ltg13_GAP1"
+names: "ltg13_GAP2"
+names: "ltg13_TPC_sync"
+names: "ltg13_TPC_rst"
+names: "ltg13_TOF"
+names: "ltg14_ORB"
+names: "ltg14_HB"
+names: "ltg14_HBr"
+names: "ltg14_HC"
+names: "ltg14_PH"
+names: "ltg14_PP"
+names: "ltg14_CAL"
+names: "ltg14_SOT"
+names: "ltg14_EOT"
+names: "ltg14_SOC"
+names: "ltg14_EOC"
+names: "ltg14_TF"
+names: "ltg14_FERST"
+names: "ltg14_RT"
+names: "ltg14_RS"
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: "ltg14_GAP1"
+names: "ltg14_GAP2"
+names: "ltg14_TPC_sync"
+names: "ltg14_TPC_rst"
+names: "ltg14_TOF"
+names: "ltg15_ORB"
+names: "ltg15_HB"
+names: "ltg15_HBr"
+names: "ltg15_HC"
+names: "ltg15_PH"
+names: "ltg15_PP"
+names: "ltg15_CAL"
+names: "ltg15_SOT"
+names: "ltg15_EOT"
+names: "ltg15_SOC"
+names: "ltg15_EOC"
+names: "ltg15_TF"
+names: "ltg15_FERST"
+names: "ltg15_RT"
+names: "ltg15_RS"
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: "ltg15_GAP1"
+names: "ltg15_GAP2"
+names: "ltg15_TPC_sync"
+names: "ltg15_TPC_rst"
+names: "ltg15_TOF"
+names: "ltg16_ORB"
+names: "ltg16_HB"
+names: "ltg16_HBr"
+names: "ltg16_HC"
+names: "ltg16_PH"
+names: "ltg16_PP"
+names: "ltg16_CAL"
+names: "ltg16_SOT"
+names: "ltg16_EOT"
+names: "ltg16_SOC"
+names: "ltg16_EOC"
+names: "ltg16_TF"
+names: "ltg16_FERST"
+names: "ltg16_RT"
+names: "ltg16_RS"
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: "ltg16_GAP1"
+names: "ltg16_GAP2"
+names: "ltg16_TPC_sync"
+names: "ltg16_TPC_rst"
+names: "ltg16_TOF"
+names: "ltg17_ORB"
+names: "ltg17_HB"
+names: "ltg17_HBr"
+names: "ltg17_HC"
+names: "ltg17_PH"
+names: "ltg17_PP"
+names: "ltg17_CAL"
+names: "ltg17_SOT"
+names: "ltg17_EOT"
+names: "ltg17_SOC"
+names: "ltg17_EOC"
+names: "ltg17_TF"
+names: "ltg17_FERST"
+names: "ltg17_RT"
+names: "ltg17_RS"
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: "ltg17_GAP1"
+names: "ltg17_GAP2"
+names: "ltg17_TPC_sync"
+names: "ltg17_TPC_rst"
+names: "ltg17_TOF"
+names: "ltg18_ORB"
+names: "ltg18_HB"
+names: "ltg18_HBr"
+names: "ltg18_HC"
+names: "ltg18_PH"
+names: "ltg18_PP"
+names: "ltg18_CAL"
+names: "ltg18_SOT"
+names: "ltg18_EOT"
+names: "ltg18_SOC"
+names: "ltg18_EOC"
+names: "ltg18_TF"
+names: "ltg18_FERST"
+names: "ltg18_RT"
+names: "ltg18_RS"
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: "ltg18_GAP1"
+names: "ltg18_GAP2"
+names: "ltg18_TPC_sync"
+names: "ltg18_TPC_rst"
+names: "ltg18_TOF"
+names: "bc40"
+names: "clk240"
+names: "extorb"
+names: "PLSRin"
+names: "FastLMin"
+names: "BUSYin"
+names: "SPAREin"
+names: "inp1"
+names: "inp2"
+names: "inp3"
+names: "inp4"
+names: "inp5"
+names: "inp6"
+names: "inp7"
+names: "inp8"
+names: "inp9"
+names: "inp10"
+names: "inp11"
+names: "inp12"
+names: "inp13"
+names: "inp14"
+names: "inp15"
+names: "inp16"
+names: "inp17"
+names: "inp18"
+names: "inp19"
+names: "inp20"
+names: "inp21"
+names: "inp22"
+names: "inp23"
+names: "inp24"
+names: "inp25"
+names: "inp26"
+names: "inp27"
+names: "inp28"
+names: "inp29"
+names: "inp30"
+names: "inp31"
+names: "inp32"
+names: "inp33"
+names: "inp34"
+names: "inp35"
+names: "inp36"
+names: "inp37"
+names: "inp38"
+names: "inp39"
+names: "inp40"
+names: "inp41"
+names: "inp42"
+names: "inp43"
+names: "inp44"
+names: "inp45"
+names: "inp46"
+names: "inp47"
+names: "inp48"
+names: "clamb1"
+names: "clamb2"
+names: "clamb3"
+names: "clamb4"
+names: "clamb5"
+names: "clamb6"
+names: "clamb7"
+names: "clamb8"
+names: "clamb9"
+names: "clamb10"
+names: "clamb11"
+names: "clamb12"
+names: "clamb13"
+names: "clamb14"
+names: "clamb15"
+names: "clamb16"
+names: "clamb17"
+names: "clamb18"
+names: "clamb19"
+names: "clamb20"
+names: "clamb21"
+names: "clamb22"
+names: "clamb23"
+names: "clamb24"
+names: "clamb25"
+names: "clamb26"
+names: "clamb27"
+names: "clamb28"
+names: "clamb29"
+names: "clamb30"
+names: "clamb31"
+names: "clamb32"
+names: "clamb33"
+names: "clamb34"
+names: "clamb35"
+names: "clamb36"
+names: "clamb37"
+names: "clamb38"
+names: "clamb39"
+names: "clamb40"
+names: "clamb41"
+names: "clamb42"
+names: "clamb43"
+names: "clamb44"
+names: "clamb45"
+names: "clamb46"
+names: "clamb47"
+names: "clamb48"
+names: "clamb49"
+names: "clamb50"
+names: "clamb51"
+names: "clamb52"
+names: "clamb53"
+names: "clamb54"
+names: "clamb55"
+names: "clamb56"
+names: "clamb57"
+names: "clamb58"
+names: "clamb59"
+names: "clamb60"
+names: "clamb61"
+names: "clamb62"
+names: "clamb63"
+names: "clamb64"
+names: "clama1"
+names: "clama2"
+names: "clama3"
+names: "clama4"
+names: "clama5"
+names: "clama6"
+names: "clama7"
+names: "clama8"
+names: "clama9"
+names: "clama10"
+names: "clama11"
+names: "clama12"
+names: "clama13"
+names: "clama14"
+names: "clama15"
+names: "clama16"
+names: "clama17"
+names: "clama18"
+names: "clama19"
+names: "clama20"
+names: "clama21"
+names: "clama22"
+names: "clama23"
+names: "clama24"
+names: "clama25"
+names: "clama26"
+names: "clama27"
+names: "clama28"
+names: "clama29"
+names: "clama30"
+names: "clama31"
+names: "clama32"
+names: "clama33"
+names: "clama34"
+names: "clama35"
+names: "clama36"
+names: "clama37"
+names: "clama38"
+names: "clama39"
+names: "clama40"
+names: "clama41"
+names: "clama42"
+names: "clama43"
+names: "clama44"
+names: "clama45"
+names: "clama46"
+names: "clama47"
+names: "clama48"
+names: "clama49"
+names: "clama50"
+names: "clama51"
+names: "clama52"
+names: "clama53"
+names: "clama54"
+names: "clama55"
+names: "clama56"
+names: "clama57"
+names: "clama58"
+names: "clama59"
+names: "clama60"
+names: "clama61"
+names: "clama62"
+names: "clama63"
+names: "clama64"
+names: "cla0b1"
+names: "cla0b2"
+names: "cla0b3"
+names: "cla0b4"
+names: "cla0b5"
+names: "cla0b6"
+names: "cla0b7"
+names: "cla0b8"
+names: "cla0b9"
+names: "cla0b10"
+names: "cla0b11"
+names: "cla0b12"
+names: "cla0b13"
+names: "cla0b14"
+names: "cla0b15"
+names: "cla0b16"
+names: "cla0b17"
+names: "cla0b18"
+names: "cla0b19"
+names: "cla0b20"
+names: "cla0b21"
+names: "cla0b22"
+names: "cla0b23"
+names: "cla0b24"
+names: "cla0b25"
+names: "cla0b26"
+names: "cla0b27"
+names: "cla0b28"
+names: "cla0b29"
+names: "cla0b30"
+names: "cla0b31"
+names: "cla0b32"
+names: "cla0b33"
+names: "cla0b34"
+names: "cla0b35"
+names: "cla0b36"
+names: "cla0b37"
+names: "cla0b38"
+names: "cla0b39"
+names: "cla0b40"
+names: "cla0b41"
+names: "cla0b42"
+names: "cla0b43"
+names: "cla0b44"
+names: "cla0b45"
+names: "cla0b46"
+names: "cla0b47"
+names: "cla0b48"
+names: "cla0b49"
+names: "cla0b50"
+names: "cla0b51"
+names: "cla0b52"
+names: "cla0b53"
+names: "cla0b54"
+names: "cla0b55"
+names: "cla0b56"
+names: "cla0b57"
+names: "cla0b58"
+names: "cla0b59"
+names: "cla0b60"
+names: "cla0b61"
+names: "cla0b62"
+names: "cla0b63"
+names: "cla0b64"
+names: "cla0a1"
+names: "cla0a2"
+names: "cla0a3"
+names: "cla0a4"
+names: "cla0a5"
+names: "cla0a6"
+names: "cla0a7"
+names: "cla0a8"
+names: "cla0a9"
+names: "cla0a10"
+names: "cla0a11"
+names: "cla0a12"
+names: "cla0a13"
+names: "cla0a14"
+names: "cla0a15"
+names: "cla0a16"
+names: "cla0a17"
+names: "cla0a18"
+names: "cla0a19"
+names: "cla0a20"
+names: "cla0a21"
+names: "cla0a22"
+names: "cla0a23"
+names: "cla0a24"
+names: "cla0a25"
+names: "cla0a26"
+names: "cla0a27"
+names: "cla0a28"
+names: "cla0a29"
+names: "cla0a30"
+names: "cla0a31"
+names: "cla0a32"
+names: "cla0a33"
+names: "cla0a34"
+names: "cla0a35"
+names: "cla0a36"
+names: "cla0a37"
+names: "cla0a38"
+names: "cla0a39"
+names: "cla0a40"
+names: "cla0a41"
+names: "cla0a42"
+names: "cla0a43"
+names: "cla0a44"
+names: "cla0a45"
+names: "cla0a46"
+names: "cla0a47"
+names: "cla0a48"
+names: "cla0a49"
+names: "cla0a50"
+names: "cla0a51"
+names: "cla0a52"
+names: "cla0a53"
+names: "cla0a54"
+names: "cla0a55"
+names: "cla0a56"
+names: "cla0a57"
+names: "cla0a58"
+names: "cla0a59"
+names: "cla0a60"
+names: "cla0a61"
+names: "cla0a62"
+names: "cla0a63"
+names: "cla0a64"
+names: "cla1b1"
+names: "cla1b2"
+names: "cla1b3"
+names: "cla1b4"
+names: "cla1b5"
+names: "cla1b6"
+names: "cla1b7"
+names: "cla1b8"
+names: "cla1b9"
+names: "cla1b10"
+names: "cla1b11"
+names: "cla1b12"
+names: "cla1b13"
+names: "cla1b14"
+names: "cla1b15"
+names: "cla1b16"
+names: "cla1b17"
+names: "cla1b18"
+names: "cla1b19"
+names: "cla1b20"
+names: "cla1b21"
+names: "cla1b22"
+names: "cla1b23"
+names: "cla1b24"
+names: "cla1b25"
+names: "cla1b26"
+names: "cla1b27"
+names: "cla1b28"
+names: "cla1b29"
+names: "cla1b30"
+names: "cla1b31"
+names: "cla1b32"
+names: "cla1b33"
+names: "cla1b34"
+names: "cla1b35"
+names: "cla1b36"
+names: "cla1b37"
+names: "cla1b38"
+names: "cla1b39"
+names: "cla1b40"
+names: "cla1b41"
+names: "cla1b42"
+names: "cla1b43"
+names: "cla1b44"
+names: "cla1b45"
+names: "cla1b46"
+names: "cla1b47"
+names: "cla1b48"
+names: "cla1b49"
+names: "cla1b50"
+names: "cla1b51"
+names: "cla1b52"
+names: "cla1b53"
+names: "cla1b54"
+names: "cla1b55"
+names: "cla1b56"
+names: "cla1b57"
+names: "cla1b58"
+names: "cla1b59"
+names: "cla1b60"
+names: "cla1b61"
+names: "cla1b62"
+names: "cla1b63"
+names: "cla1b64"
+names: "cla1a1"
+names: "cla1a2"
+names: "cla1a3"
+names: "cla1a4"
+names: "cla1a5"
+names: "cla1a6"
+names: "cla1a7"
+names: "cla1a8"
+names: "cla1a9"
+names: "cla1a10"
+names: "cla1a11"
+names: "cla1a12"
+names: "cla1a13"
+names: "cla1a14"
+names: "cla1a15"
+names: "cla1a16"
+names: "cla1a17"
+names: "cla1a18"
+names: "cla1a19"
+names: "cla1a20"
+names: "cla1a21"
+names: "cla1a22"
+names: "cla1a23"
+names: "cla1a24"
+names: "cla1a25"
+names: "cla1a26"
+names: "cla1a27"
+names: "cla1a28"
+names: "cla1a29"
+names: "cla1a30"
+names: "cla1a31"
+names: "cla1a32"
+names: "cla1a33"
+names: "cla1a34"
+names: "cla1a35"
+names: "cla1a36"
+names: "cla1a37"
+names: "cla1a38"
+names: "cla1a39"
+names: "cla1a40"
+names: "cla1a41"
+names: "cla1a42"
+names: "cla1a43"
+names: "cla1a44"
+names: "cla1a45"
+names: "cla1a46"
+names: "cla1a47"
+names: "cla1a48"
+names: "cla1a49"
+names: "cla1a50"
+names: "cla1a51"
+names: "cla1a52"
+names: "cla1a53"
+names: "cla1a54"
+names: "cla1a55"
+names: "cla1a56"
+names: "cla1a57"
+names: "cla1a58"
+names: "cla1a59"
+names: "cla1a60"
+names: "cla1a61"
+names: "cla1a62"
+names: "cla1a63"
+names: "cla1a64"
+names: "l0_trigger"
+names: "l1_trigger"
+names: "l2_trigger"
+names: "clum1"
+names: "clum2"
+names: "clum3"
+names: "clum4"
+names: "clum5"
+names: "clum6"
+names: "clu01"
+names: "clu02"
+names: "clu03"
+names: "clu04"
+names: "clu05"
+names: "clu06"
+names: "clu11"
+names: "clu12"
+names: "clu13"
+names: "clu14"
+names: "clu15"
+names: "clu16"
+names: "ltg1_busy"
+names: "ltg2_busy"
+names: "ltg3_busy"
+names: "ltg4_busy"
+names: "ltg5_busy"
+names: "ltg6_busy"
+names: "ltg7_busy"
+names: "ltg8_busy"
+names: "ltg9_busy"
+names: "ltg10_busy"
+names: "ltg11_busy"
+names: "ltg12_busy"
+names: "ltg13_busy"
+names: "ltg14_busy"
+names: "ltg15_busy"
+names: "ltg16_busy"
+names: "ltg17_busy"
+names: "ltg18_busy"
+names: "orbitid"
diff --git a/Detectors/CTP/workflowScalers/py/scalersCTP_v3.txt b/Detectors/CTP/workflowScalers/py/scalersCTP_v3.txt
new file mode 100644
index 0000000000000..ccad42af3f38d
--- /dev/null
+++ b/Detectors/CTP/workflowScalers/py/scalersCTP_v3.txt
@@ -0,0 +1,1085 @@
+names: "runn0"
+names: "runn1"
+names: "runn2"
+names: "runn3"
+names: "runn4"
+names: "runn5"
+names: "runn6"
+names: "runn7"
+names: "runn8"
+names: "runn9"
+names: "runn10"
+names: "runn11"
+names: "runn12"
+names: "runn13"
+names: "runn14"
+names: "runn15"
+names: "ltg1_ORB"
+names: "ltg1_HB"
+names: "ltg1_HBr"
+names: "ltg1_HC"
+names: "ltg1_PH"
+names: "ltg1_PP"
+names: "ltg1_CAL"
+names: "ltg1_SOT"
+names: "ltg1_EOT"
+names: "ltg1_SOC"
+names: "ltg1_EOC"
+names: "ltg1_TF"
+names: "ltg1_FERST"
+names: "ltg1_RT"
+names: "ltg1_RS"
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: "ltg1_GAP1"
+names: "ltg1_GAP2"
+names: "ltg1_TPC_sync"
+names: "ltg1_TPC_rst"
+names: "ltg1_TOF"
+names: "ltg2_ORB"
+names: "ltg2_HB"
+names: "ltg2_HBr"
+names: "ltg2_HC"
+names: "ltg2_PH"
+names: "ltg2_PP"
+names: "ltg2_CAL"
+names: "ltg2_SOT"
+names: "ltg2_EOT"
+names: "ltg2_SOC"
+names: "ltg2_EOC"
+names: "ltg2_TF"
+names: "ltg2_FERST"
+names: "ltg2_RT"
+names: "ltg2_RS"
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: "ltg2_GAP1"
+names: "ltg2_GAP2"
+names: "ltg2_TPC_sync"
+names: "ltg2_TPC_rst"
+names: "ltg2_TOF"
+names: "ltg3_ORB"
+names: "ltg3_HB"
+names: "ltg3_HBr"
+names: "ltg3_HC"
+names: "ltg3_PH"
+names: "ltg3_PP"
+names: "ltg3_CAL"
+names: "ltg3_SOT"
+names: "ltg3_EOT"
+names: "ltg3_SOC"
+names: "ltg3_EOC"
+names: "ltg3_TF"
+names: "ltg3_FERST"
+names: "ltg3_RT"
+names: "ltg3_RS"
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: "ltg3_GAP1"
+names: "ltg3_GAP2"
+names: "ltg3_TPC_sync"
+names: "ltg3_TPC_rst"
+names: "ltg3_TOF"
+names: "ltg4_ORB"
+names: "ltg4_HB"
+names: "ltg4_HBr"
+names: "ltg4_HC"
+names: "ltg4_PH"
+names: "ltg4_PP"
+names: "ltg4_CAL"
+names: "ltg4_SOT"
+names: "ltg4_EOT"
+names: "ltg4_SOC"
+names: "ltg4_EOC"
+names: "ltg4_TF"
+names: "ltg4_FERST"
+names: "ltg4_RT"
+names: "ltg4_RS"
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: "ltg4_GAP1"
+names: "ltg4_GAP2"
+names: "ltg4_TPC_sync"
+names: "ltg4_TPC_rst"
+names: "ltg4_TOF"
+names: "ltg5_ORB"
+names: "ltg5_HB"
+names: "ltg5_HBr"
+names: "ltg5_HC"
+names: "ltg5_PH"
+names: "ltg5_PP"
+names: "ltg5_CAL"
+names: "ltg5_SOT"
+names: "ltg5_EOT"
+names: "ltg5_SOC"
+names: "ltg5_EOC"
+names: "ltg5_TF"
+names: "ltg5_FERST"
+names: "ltg5_RT"
+names: "ltg5_RS"
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: "ltg5_GAP1"
+names: "ltg5_GAP2"
+names: "ltg5_TPC_sync"
+names: "ltg5_TPC_rst"
+names: "ltg5_TOF"
+names: "ltg6_ORB"
+names: "ltg6_HB"
+names: "ltg6_HBr"
+names: "ltg6_HC"
+names: "ltg6_PH"
+names: "ltg6_PP"
+names: "ltg6_CAL"
+names: "ltg6_SOT"
+names: "ltg6_EOT"
+names: "ltg6_SOC"
+names: "ltg6_EOC"
+names: "ltg6_TF"
+names: "ltg6_FERST"
+names: "ltg6_RT"
+names: "ltg6_RS"
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: "ltg6_GAP1"
+names: "ltg6_GAP2"
+names: "ltg6_TPC_sync"
+names: "ltg6_TPC_rst"
+names: "ltg6_TOF"
+names: "ltg7_ORB"
+names: "ltg7_HB"
+names: "ltg7_HBr"
+names: "ltg7_HC"
+names: "ltg7_PH"
+names: "ltg7_PP"
+names: "ltg7_CAL"
+names: "ltg7_SOT"
+names: "ltg7_EOT"
+names: "ltg7_SOC"
+names: "ltg7_EOC"
+names: "ltg7_TF"
+names: "ltg7_FERST"
+names: "ltg7_RT"
+names: "ltg7_RS"
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: "ltg7_GAP1"
+names: "ltg7_GAP2"
+names: "ltg7_TPC_sync"
+names: "ltg7_TPC_rst"
+names: "ltg7_TOF"
+names: "ltg8_ORB"
+names: "ltg8_HB"
+names: "ltg8_HBr"
+names: "ltg8_HC"
+names: "ltg8_PH"
+names: "ltg8_PP"
+names: "ltg8_CAL"
+names: "ltg8_SOT"
+names: "ltg8_EOT"
+names: "ltg8_SOC"
+names: "ltg8_EOC"
+names: "ltg8_TF"
+names: "ltg8_FERST"
+names: "ltg8_RT"
+names: "ltg8_RS"
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: "ltg8_GAP1"
+names: "ltg8_GAP2"
+names: "ltg8_TPC_sync"
+names: "ltg8_TPC_rst"
+names: "ltg8_TOF"
+names: "ltg9_ORB"
+names: "ltg9_HB"
+names: "ltg9_HBr"
+names: "ltg9_HC"
+names: "ltg9_PH"
+names: "ltg9_PP"
+names: "ltg9_CAL"
+names: "ltg9_SOT"
+names: "ltg9_EOT"
+names: "ltg9_SOC"
+names: "ltg9_EOC"
+names: "ltg9_TF"
+names: "ltg9_FERST"
+names: "ltg9_RT"
+names: "ltg9_RS"
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: "ltg9_GAP1"
+names: "ltg9_GAP2"
+names: "ltg9_TPC_sync"
+names: "ltg9_TPC_rst"
+names: "ltg9_TOF"
+names: "ltg10_ORB"
+names: "ltg10_HB"
+names: "ltg10_HBr"
+names: "ltg10_HC"
+names: "ltg10_PH"
+names: "ltg10_PP"
+names: "ltg10_CAL"
+names: "ltg10_SOT"
+names: "ltg10_EOT"
+names: "ltg10_SOC"
+names: "ltg10_EOC"
+names: "ltg10_TF"
+names: "ltg10_FERST"
+names: "ltg10_RT"
+names: "ltg10_RS"
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: "ltg10_GAP1"
+names: "ltg10_GAP2"
+names: "ltg10_TPC_sync"
+names: "ltg10_TPC_rst"
+names: "ltg10_TOF"
+names: "ltg11_ORB"
+names: "ltg11_HB"
+names: "ltg11_HBr"
+names: "ltg11_HC"
+names: "ltg11_PH"
+names: "ltg11_PP"
+names: "ltg11_CAL"
+names: "ltg11_SOT"
+names: "ltg11_EOT"
+names: "ltg11_SOC"
+names: "ltg11_EOC"
+names: "ltg11_TF"
+names: "ltg11_FERST"
+names: "ltg11_RT"
+names: "ltg11_RS"
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: "ltg11_GAP1"
+names: "ltg11_GAP2"
+names: "ltg11_TPC_sync"
+names: "ltg11_TPC_rst"
+names: "ltg11_TOF"
+names: "ltg12_ORB"
+names: "ltg12_HB"
+names: "ltg12_HBr"
+names: "ltg12_HC"
+names: "ltg12_PH"
+names: "ltg12_PP"
+names: "ltg12_CAL"
+names: "ltg12_SOT"
+names: "ltg12_EOT"
+names: "ltg12_SOC"
+names: "ltg12_EOC"
+names: "ltg12_TF"
+names: "ltg12_FERST"
+names: "ltg12_RT"
+names: "ltg12_RS"
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: "ltg12_GAP1"
+names: "ltg12_GAP2"
+names: "ltg12_TPC_sync"
+names: "ltg12_TPC_rst"
+names: "ltg12_TOF"
+names: "ltg13_ORB"
+names: "ltg13_HB"
+names: "ltg13_HBr"
+names: "ltg13_HC"
+names: "ltg13_PH"
+names: "ltg13_PP"
+names: "ltg13_CAL"
+names: "ltg13_SOT"
+names: "ltg13_EOT"
+names: "ltg13_SOC"
+names: "ltg13_EOC"
+names: "ltg13_TF"
+names: "ltg13_FERST"
+names: "ltg13_RT"
+names: "ltg13_RS"
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: "ltg13_GAP1"
+names: "ltg13_GAP2"
+names: "ltg13_TPC_sync"
+names: "ltg13_TPC_rst"
+names: "ltg13_TOF"
+names: "ltg14_ORB"
+names: "ltg14_HB"
+names: "ltg14_HBr"
+names: "ltg14_HC"
+names: "ltg14_PH"
+names: "ltg14_PP"
+names: "ltg14_CAL"
+names: "ltg14_SOT"
+names: "ltg14_EOT"
+names: "ltg14_SOC"
+names: "ltg14_EOC"
+names: "ltg14_TF"
+names: "ltg14_FERST"
+names: "ltg14_RT"
+names: "ltg14_RS"
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: "ltg14_GAP1"
+names: "ltg14_GAP2"
+names: "ltg14_TPC_sync"
+names: "ltg14_TPC_rst"
+names: "ltg14_TOF"
+names: "ltg15_ORB"
+names: "ltg15_HB"
+names: "ltg15_HBr"
+names: "ltg15_HC"
+names: "ltg15_PH"
+names: "ltg15_PP"
+names: "ltg15_CAL"
+names: "ltg15_SOT"
+names: "ltg15_EOT"
+names: "ltg15_SOC"
+names: "ltg15_EOC"
+names: "ltg15_TF"
+names: "ltg15_FERST"
+names: "ltg15_RT"
+names: "ltg15_RS"
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: "ltg15_GAP1"
+names: "ltg15_GAP2"
+names: "ltg15_TPC_sync"
+names: "ltg15_TPC_rst"
+names: "ltg15_TOF"
+names: "ltg16_ORB"
+names: "ltg16_HB"
+names: "ltg16_HBr"
+names: "ltg16_HC"
+names: "ltg16_PH"
+names: "ltg16_PP"
+names: "ltg16_CAL"
+names: "ltg16_SOT"
+names: "ltg16_EOT"
+names: "ltg16_SOC"
+names: "ltg16_EOC"
+names: "ltg16_TF"
+names: "ltg16_FERST"
+names: "ltg16_RT"
+names: "ltg16_RS"
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: "ltg16_GAP1"
+names: "ltg16_GAP2"
+names: "ltg16_TPC_sync"
+names: "ltg16_TPC_rst"
+names: "ltg16_TOF"
+names: "ltg17_ORB"
+names: "ltg17_HB"
+names: "ltg17_HBr"
+names: "ltg17_HC"
+names: "ltg17_PH"
+names: "ltg17_PP"
+names: "ltg17_CAL"
+names: "ltg17_SOT"
+names: "ltg17_EOT"
+names: "ltg17_SOC"
+names: "ltg17_EOC"
+names: "ltg17_TF"
+names: "ltg17_FERST"
+names: "ltg17_RT"
+names: "ltg17_RS"
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: "ltg17_GAP1"
+names: "ltg17_GAP2"
+names: "ltg17_TPC_sync"
+names: "ltg17_TPC_rst"
+names: "ltg17_TOF"
+names: "ltg18_ORB"
+names: "ltg18_HB"
+names: "ltg18_HBr"
+names: "ltg18_HC"
+names: "ltg18_PH"
+names: "ltg18_PP"
+names: "ltg18_CAL"
+names: "ltg18_SOT"
+names: "ltg18_EOT"
+names: "ltg18_SOC"
+names: "ltg18_EOC"
+names: "ltg18_TF"
+names: "ltg18_FERST"
+names: "ltg18_RT"
+names: "ltg18_RS"
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: ""
+names: "ltg18_GAP1"
+names: "ltg18_GAP2"
+names: "ltg18_TPC_sync"
+names: "ltg18_TPC_rst"
+names: "ltg18_TOF"
+names: "bc40"
+names: "clk240"
+names: "extorb"
+names: "PLSRin"
+names: "FastLMin"
+names: "BUSYin"
+names: "SPAREin"
+names: "inp1"
+names: "inp2"
+names: "inp3"
+names: "inp4"
+names: "inp5"
+names: "inp6"
+names: "inp7"
+names: "inp8"
+names: "inp9"
+names: "inp10"
+names: "inp11"
+names: "inp12"
+names: "inp13"
+names: "inp14"
+names: "inp15"
+names: "inp16"
+names: "inp17"
+names: "inp18"
+names: "inp19"
+names: "inp20"
+names: "inp21"
+names: "inp22"
+names: "inp23"
+names: "inp24"
+names: "inp25"
+names: "inp26"
+names: "inp27"
+names: "inp28"
+names: "inp29"
+names: "inp30"
+names: "inp31"
+names: "inp32"
+names: "inp33"
+names: "inp34"
+names: "inp35"
+names: "inp36"
+names: "inp37"
+names: "inp38"
+names: "inp39"
+names: "inp40"
+names: "inp41"
+names: "inp42"
+names: "inp43"
+names: "inp44"
+names: "inp45"
+names: "inp46"
+names: "inp47"
+names: "inp48"
+names: "clamb1"
+names: "clamb2"
+names: "clamb3"
+names: "clamb4"
+names: "clamb5"
+names: "clamb6"
+names: "clamb7"
+names: "clamb8"
+names: "clamb9"
+names: "clamb10"
+names: "clamb11"
+names: "clamb12"
+names: "clamb13"
+names: "clamb14"
+names: "clamb15"
+names: "clamb16"
+names: "clamb17"
+names: "clamb18"
+names: "clamb19"
+names: "clamb20"
+names: "clamb21"
+names: "clamb22"
+names: "clamb23"
+names: "clamb24"
+names: "clamb25"
+names: "clamb26"
+names: "clamb27"
+names: "clamb28"
+names: "clamb29"
+names: "clamb30"
+names: "clamb31"
+names: "clamb32"
+names: "clamb33"
+names: "clamb34"
+names: "clamb35"
+names: "clamb36"
+names: "clamb37"
+names: "clamb38"
+names: "clamb39"
+names: "clamb40"
+names: "clamb41"
+names: "clamb42"
+names: "clamb43"
+names: "clamb44"
+names: "clamb45"
+names: "clamb46"
+names: "clamb47"
+names: "clamb48"
+names: "clamb49"
+names: "clamb50"
+names: "clamb51"
+names: "clamb52"
+names: "clamb53"
+names: "clamb54"
+names: "clamb55"
+names: "clamb56"
+names: "clamb57"
+names: "clamb58"
+names: "clamb59"
+names: "clamb60"
+names: "clamb61"
+names: "clamb62"
+names: "clamb63"
+names: "clamb64"
+names: "clama1"
+names: "clama2"
+names: "clama3"
+names: "clama4"
+names: "clama5"
+names: "clama6"
+names: "clama7"
+names: "clama8"
+names: "clama9"
+names: "clama10"
+names: "clama11"
+names: "clama12"
+names: "clama13"
+names: "clama14"
+names: "clama15"
+names: "clama16"
+names: "clama17"
+names: "clama18"
+names: "clama19"
+names: "clama20"
+names: "clama21"
+names: "clama22"
+names: "clama23"
+names: "clama24"
+names: "clama25"
+names: "clama26"
+names: "clama27"
+names: "clama28"
+names: "clama29"
+names: "clama30"
+names: "clama31"
+names: "clama32"
+names: "clama33"
+names: "clama34"
+names: "clama35"
+names: "clama36"
+names: "clama37"
+names: "clama38"
+names: "clama39"
+names: "clama40"
+names: "clama41"
+names: "clama42"
+names: "clama43"
+names: "clama44"
+names: "clama45"
+names: "clama46"
+names: "clama47"
+names: "clama48"
+names: "clama49"
+names: "clama50"
+names: "clama51"
+names: "clama52"
+names: "clama53"
+names: "clama54"
+names: "clama55"
+names: "clama56"
+names: "clama57"
+names: "clama58"
+names: "clama59"
+names: "clama60"
+names: "clama61"
+names: "clama62"
+names: "clama63"
+names: "clama64"
+names: "cla0b1"
+names: "cla0b2"
+names: "cla0b3"
+names: "cla0b4"
+names: "cla0b5"
+names: "cla0b6"
+names: "cla0b7"
+names: "cla0b8"
+names: "cla0b9"
+names: "cla0b10"
+names: "cla0b11"
+names: "cla0b12"
+names: "cla0b13"
+names: "cla0b14"
+names: "cla0b15"
+names: "cla0b16"
+names: "cla0b17"
+names: "cla0b18"
+names: "cla0b19"
+names: "cla0b20"
+names: "cla0b21"
+names: "cla0b22"
+names: "cla0b23"
+names: "cla0b24"
+names: "cla0b25"
+names: "cla0b26"
+names: "cla0b27"
+names: "cla0b28"
+names: "cla0b29"
+names: "cla0b30"
+names: "cla0b31"
+names: "cla0b32"
+names: "cla0b33"
+names: "cla0b34"
+names: "cla0b35"
+names: "cla0b36"
+names: "cla0b37"
+names: "cla0b38"
+names: "cla0b39"
+names: "cla0b40"
+names: "cla0b41"
+names: "cla0b42"
+names: "cla0b43"
+names: "cla0b44"
+names: "cla0b45"
+names: "cla0b46"
+names: "cla0b47"
+names: "cla0b48"
+names: "cla0b49"
+names: "cla0b50"
+names: "cla0b51"
+names: "cla0b52"
+names: "cla0b53"
+names: "cla0b54"
+names: "cla0b55"
+names: "cla0b56"
+names: "cla0b57"
+names: "cla0b58"
+names: "cla0b59"
+names: "cla0b60"
+names: "cla0b61"
+names: "cla0b62"
+names: "cla0b63"
+names: "cla0b64"
+names: "cla0a1"
+names: "cla0a2"
+names: "cla0a3"
+names: "cla0a4"
+names: "cla0a5"
+names: "cla0a6"
+names: "cla0a7"
+names: "cla0a8"
+names: "cla0a9"
+names: "cla0a10"
+names: "cla0a11"
+names: "cla0a12"
+names: "cla0a13"
+names: "cla0a14"
+names: "cla0a15"
+names: "cla0a16"
+names: "cla0a17"
+names: "cla0a18"
+names: "cla0a19"
+names: "cla0a20"
+names: "cla0a21"
+names: "cla0a22"
+names: "cla0a23"
+names: "cla0a24"
+names: "cla0a25"
+names: "cla0a26"
+names: "cla0a27"
+names: "cla0a28"
+names: "cla0a29"
+names: "cla0a30"
+names: "cla0a31"
+names: "cla0a32"
+names: "cla0a33"
+names: "cla0a34"
+names: "cla0a35"
+names: "cla0a36"
+names: "cla0a37"
+names: "cla0a38"
+names: "cla0a39"
+names: "cla0a40"
+names: "cla0a41"
+names: "cla0a42"
+names: "cla0a43"
+names: "cla0a44"
+names: "cla0a45"
+names: "cla0a46"
+names: "cla0a47"
+names: "cla0a48"
+names: "cla0a49"
+names: "cla0a50"
+names: "cla0a51"
+names: "cla0a52"
+names: "cla0a53"
+names: "cla0a54"
+names: "cla0a55"
+names: "cla0a56"
+names: "cla0a57"
+names: "cla0a58"
+names: "cla0a59"
+names: "cla0a60"
+names: "cla0a61"
+names: "cla0a62"
+names: "cla0a63"
+names: "cla0a64"
+names: "cla1b1"
+names: "cla1b2"
+names: "cla1b3"
+names: "cla1b4"
+names: "cla1b5"
+names: "cla1b6"
+names: "cla1b7"
+names: "cla1b8"
+names: "cla1b9"
+names: "cla1b10"
+names: "cla1b11"
+names: "cla1b12"
+names: "cla1b13"
+names: "cla1b14"
+names: "cla1b15"
+names: "cla1b16"
+names: "cla1b17"
+names: "cla1b18"
+names: "cla1b19"
+names: "cla1b20"
+names: "cla1b21"
+names: "cla1b22"
+names: "cla1b23"
+names: "cla1b24"
+names: "cla1b25"
+names: "cla1b26"
+names: "cla1b27"
+names: "cla1b28"
+names: "cla1b29"
+names: "cla1b30"
+names: "cla1b31"
+names: "cla1b32"
+names: "cla1b33"
+names: "cla1b34"
+names: "cla1b35"
+names: "cla1b36"
+names: "cla1b37"
+names: "cla1b38"
+names: "cla1b39"
+names: "cla1b40"
+names: "cla1b41"
+names: "cla1b42"
+names: "cla1b43"
+names: "cla1b44"
+names: "cla1b45"
+names: "cla1b46"
+names: "cla1b47"
+names: "cla1b48"
+names: "cla1b49"
+names: "cla1b50"
+names: "cla1b51"
+names: "cla1b52"
+names: "cla1b53"
+names: "cla1b54"
+names: "cla1b55"
+names: "cla1b56"
+names: "cla1b57"
+names: "cla1b58"
+names: "cla1b59"
+names: "cla1b60"
+names: "cla1b61"
+names: "cla1b62"
+names: "cla1b63"
+names: "cla1b64"
+names: "cla1a1"
+names: "cla1a2"
+names: "cla1a3"
+names: "cla1a4"
+names: "cla1a5"
+names: "cla1a6"
+names: "cla1a7"
+names: "cla1a8"
+names: "cla1a9"
+names: "cla1a10"
+names: "cla1a11"
+names: "cla1a12"
+names: "cla1a13"
+names: "cla1a14"
+names: "cla1a15"
+names: "cla1a16"
+names: "cla1a17"
+names: "cla1a18"
+names: "cla1a19"
+names: "cla1a20"
+names: "cla1a21"
+names: "cla1a22"
+names: "cla1a23"
+names: "cla1a24"
+names: "cla1a25"
+names: "cla1a26"
+names: "cla1a27"
+names: "cla1a28"
+names: "cla1a29"
+names: "cla1a30"
+names: "cla1a31"
+names: "cla1a32"
+names: "cla1a33"
+names: "cla1a34"
+names: "cla1a35"
+names: "cla1a36"
+names: "cla1a37"
+names: "cla1a38"
+names: "cla1a39"
+names: "cla1a40"
+names: "cla1a41"
+names: "cla1a42"
+names: "cla1a43"
+names: "cla1a44"
+names: "cla1a45"
+names: "cla1a46"
+names: "cla1a47"
+names: "cla1a48"
+names: "cla1a49"
+names: "cla1a50"
+names: "cla1a51"
+names: "cla1a52"
+names: "cla1a53"
+names: "cla1a54"
+names: "cla1a55"
+names: "cla1a56"
+names: "cla1a57"
+names: "cla1a58"
+names: "cla1a59"
+names: "cla1a60"
+names: "cla1a61"
+names: "cla1a62"
+names: "cla1a63"
+names: "cla1a64"
+names: "l0_trigger"
+names: "l1_trigger"
+names: "l2_trigger"
+names: "clum1"
+names: "clum2"
+names: "clum3"
+names: "clum4"
+names: "clum5"
+names: "clum6"
+names: "clu01"
+names: "clu02"
+names: "clu03"
+names: "clu04"
+names: "clu05"
+names: "clu06"
+names: "clu11"
+names: "clu12"
+names: "clu13"
+names: "clu14"
+names: "clu15"
+names: "clu16"
+names: "ltg1_busy"
+names: "ltg2_busy"
+names: "ltg3_busy"
+names: "ltg4_busy"
+names: "ltg5_busy"
+names: "ltg6_busy"
+names: "ltg7_busy"
+names: "ltg8_busy"
+names: "ltg9_busy"
+names: "ltg10_busy"
+names: "ltg11_busy"
+names: "ltg12_busy"
+names: "ltg13_busy"
+names: "ltg14_busy"
+names: "ltg15_busy"
+names: "ltg16_busy"
+names: "ltg17_busy"
+names: "ltg18_busy"
+names: "orbitid"
+names: "clum7"
+names: "clum8"
+names: "clu07"
+names: "clu08"
+names: "clu17"
+names: "clu18"
+names: "clu_busy1"
+names: "clu_busy2"
+names: "clu_busy3"
+names: "clu_busy4"
+names: "clu_busy5"
+names: "clu_busy6"
+names: "clu_busy7"
+names: "clu_busy8"
diff --git a/Detectors/CTP/workflowScalers/py/sub_client.py b/Detectors/CTP/workflowScalers/py/sub_client.py
index 79bc5594f74f5..0c5d50bbf4807 100644
--- a/Detectors/CTP/workflowScalers/py/sub_client.py
+++ b/Detectors/CTP/workflowScalers/py/sub_client.py
@@ -20,15 +20,18 @@
 if len(sys.argv) > 2:
     socket.connect ("tcp://localhost:%s" % port1)
 # Subscribe to zipcode, default is NYC, 10001
-topicfilter = "CTP"
+topicfilter = ""
 socket.setsockopt_string(zmq.SUBSCRIBE, topicfilter)
 
 # Process 5 updates
 total_value = 0
 while(1):
     string = socket.recv_multipart()
-    #string = stringr.decode()
-    print("string:",string)
+    h = string[0].decode('UTF-8')
+    data = string[1]
+    if len(data) > 20:
+      data = data[0:20]
+    print("string:",h,data)
     #topic, messagedata = string.split()
     #total_value += int(messagedata)
     #print(topic, messagedata)
diff --git a/Detectors/CTP/workflowScalers/py/watchProcess.py b/Detectors/CTP/workflowScalers/py/watchProcess.py
new file mode 100755
index 0000000000000..5ac9a98833706
--- /dev/null
+++ b/Detectors/CTP/workflowScalers/py/watchProcess.py
@@ -0,0 +1,117 @@
+#!/usr/bin/python3
+import requests,re
+import os,sys,time,math
+import time
+from subprocess import Popen, PIPE
+def send2(text = "ahoj"):
+  headers = { 'Content-Type':'application/json',}
+  data = {
+    'text':text,
+    'username':'alicetrg',
+  }
+  response = requests.post('https://mattermost.web.cern.ch/hooks/75949oimoinr9b47uhp8c1oomh',headers=headers,json=data)
+  print(response)
+#
+def send():
+  """
+     does not work always return can not parse data
+  """
+  Hpar = '\\"Content-Type:application/json\\"'
+  texth = '\\"text\\"'
+  text = '\\"lalal\\"'
+  usernameh = '\\"username\\"'
+  username = '\\"alicetrg\\"'
+  dpar = '{}:{},{}:{}'.format(texth,text,usernameh,username)
+  dpar = "{"+dpar+"}"
+  dpar = '"'+dpar+'"'
+  #print(dpar)
+  cmd = 'curl -g -i -X POST -H {} -d {} https://mattermost.web.cern.ch/hooks/75949oimoinr9b47uhp8c1oomh'.format(Hpar,dpar)
+  #cmd = 'curl -i -X POST -H {} https://mattermost.web.cern.ch/hooks/75949oimoinr9b47uhp8c1oomh'.format(Hpar)
+  #cmd='curl -s -o /dev/null {} -F blob=@datafile.root {}/CTP/Calib/{}/{}/{}'.format(wpar, ccdb, actid, tval_s, tval_e)
+  process = Popen(cmd.split(), stdout = PIPE, stderr = PIPE)
+  stdo, stde = process.communicate()
+  print(cmd)
+  print(stdo)
+  print("====")
+  print(stde)
+def getLog(service):
+  #cmd = "journalctl --no-hostname --user-unit "+service
+  cmd = 'journalctl --no-hostname --user-unit --since "1 day ago" '+service
+  print(cmd,cmd.split())
+  process = Popen(cmd.split(), stdout = PIPE, stderr = PIPE)
+  stdo, stde= process.communicate()
+  stdo_str = stdo.decode("utf-8")
+  if stdo_str.find("No entries") != -1:
+    print("No entries for service:",service)
+    return None
+  #print(stdo)
+  return stdo_str
+NMAX = 3
+def parseLog(log):
+  nsent = 0
+  lines = log.split('\n')
+  print("# lines:",len(lines))
+  print(lines[len(lines)-2])
+  for line in lines:
+    if line.find("ERROR") != -1:
+     print(line)
+     if nsent < NMAX:
+      send2(line)
+      nsent += 1
+    if line.find("ALARM") != -1:
+     print(line)
+     if nsent < NMAX:
+      send2(line)
+      nsent += 1
+#
+nWarn = 0
+nAlarm = 0
+nError = 0
+def printNew(list,n,send = 0):
+  print(n,len(list))
+  sendnow = 0
+  if len(list) > n :
+    sendnow = 1
+    for i in list[n:]:
+      print(i)
+    n = len(list)
+  if sendnow and send:
+    print("sending to mm:",line)
+    send2(line)
+  return n
+def getLogFile():
+  global nWarn, nAlarm,nError
+  nWarnList = []
+  nAlarmList = []
+  nErrorList = []
+  MAX = 3
+  file = "/home/rl/WORK/ctpproxy110424.log"
+  f = open(file,"r")
+  nsent = 0
+  for line in f:
+    if line.find("ERROR") != -1:
+      print(line)
+    if line.find("ALARM") != -1:
+      #print(line)
+      nAlarmList.append(line)
+    #send2(line)
+    #nsent += 1
+    if line.find("WARN") != -1:
+      #print(line)
+      #items = line.split("\]\[")
+      #items = re.split('\[|\]',line)
+      nWarnList.append(line)
+  #print(nWarn)
+  nWarn = printNew(nWarnList,nWarn)
+  nAlarm = printNew(nAlarmList,nAlarm,1)
+  f.close()
+if __name__ =="__main__":
+  #send2("uj0");
+  while 1:
+    now = time.localtime()
+    current_time = time.strftime("%H:%M:%S",now)
+    print("===> Time =", current_time)
+    log = getLogFile()
+    time.sleep(15)
+
+  #parseLog(log)
diff --git a/Detectors/CTP/workflowScalers/src/CTPWorkflowScalersLinkDef.h b/Detectors/CTP/workflowScalers/src/CTPWorkflowScalersLinkDef.h
new file mode 100644
index 0000000000000..284e64a5222b1
--- /dev/null
+++ b/Detectors/CTP/workflowScalers/src/CTPWorkflowScalersLinkDef.h
@@ -0,0 +1,18 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#ifdef __CLING__
+
+#pragma link off all globals;
+#pragma link off all classes;
+#pragma link off all functions;
+
+#pragma link C++ class o2::ctp::ctpCCDBManager + ;
+#endif
diff --git a/Detectors/CTP/workflowScalers/src/RunManager.cxx b/Detectors/CTP/workflowScalers/src/RunManager.cxx
new file mode 100644
index 0000000000000..778677bec2ec9
--- /dev/null
+++ b/Detectors/CTP/workflowScalers/src/RunManager.cxx
@@ -0,0 +1,488 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file RunManager.cxx
+/// \author Roman Lietava
+
+#include "CTPWorkflowScalers/RunManager.h"
+#include 
+#include 
+#include 
+#include "CommonUtils/StringUtils.h"
+#include 
+
+using namespace o2::ctp;
+///
+/// Active run to keep cfg and saclers of active runs
+/// Also used for Bookkeeping counters managment;
+///
+void CTPActiveRun::initBK()
+{
+  std::vector clslist = cfg.getTriggerClassList();
+  for (auto const& cls : clslist) {
+    cntslast[cls] = {0, 0, 0, 0, 0, 0};
+    cntslast0[cls] = {0, 0, 0, 0, 0, 0};
+    overflows[cls] = {0, 0, 0, 0, 0, 0};
+  }
+}
+int CTPActiveRun::send2BK(std::unique_ptr& BKClient, size_t ts, bool start)
+{
+  int runNumber = cfg.getRunNumber();
+  // LOG(info) << "BK Filling run:" << runNumber;
+  // int runOri = runNumber;
+  // runNumber = 123;
+  if (start) {
+    for (auto const& cls : cntslast) {
+      for (int i = 0; i < 6; i++) {
+        cnts0[cls.first][i] = cls.second[i];
+      }
+    }
+  }
+  std::array cntsbk{0};
+  for (auto const& cls : cntslast) {
+    for (int i = 0; i < 6; i++) {
+      if (cls.second[i] < cntslast0[cls.first][i]) {
+        overflows[cls.first][i]++;
+      }
+      cntslast0[cls.first][i] = cls.second[i];
+      cntsbk[i] = (uint64_t)cls.second[i] + 0xffffffffull * overflows[cls.first][i] - (uint64_t)cnts0[cls.first][i];
+    }
+    std::string clsname = cfg.getClassNameFromHWIndex(cls.first);
+    // clsname = std::to_string(runOri) + "_" + clsname;
+    try {
+      BKClient->ctpTriggerCounters()->createOrUpdateForRun(runNumber, clsname, ts, cntsbk[0], cntsbk[1], cntsbk[2], cntsbk[3], cntsbk[4], cntsbk[5]);
+    } catch (std::runtime_error& error) {
+      std::cerr << "An error occurred: " << error.what() << std::endl;
+      return 1;
+    }
+    LOG(debug) << "Run BK:" << runNumber << " class:" << clsname << " cls.first" << cls.first << " ts:" << ts << "  cnts:" << cntsbk[0] << " " << cntsbk[1] << " " << cntsbk[2] << " " << cntsbk[3] << " " << cntsbk[4] << " " << cntsbk[5];
+  }
+  return 0;
+}
+///
+/// Run Manager to manage Config and Scalers
+///
+void CTPRunManager::init()
+{
+  for (uint32_t i = 0; i < NRUNS; i++) {
+    mActiveRuns[i] = nullptr;
+  }
+  loadScalerNames();
+  if (mBKHost != "none") {
+    mBKClient = BkpClientFactory::create(mBKHost);
+    LOG(info) << "BK Client created with:" << mBKHost;
+  } else {
+    LOG(info) << "BK not sent";
+  }
+  setQCDBHost(mQCDBHost);
+  LOG(info) << "QC host:" << mQCDBHost;
+  LOG(info) << "QCDB writing every:" << mQCWritePeriod << " 10 secs";
+  LOG(info) << "CCDB host:" << mCCDBHost;
+  LOG(info) << "CTP vNew cfg:" << mNew;
+  LOG(info) << "ctp.cfg dir:" << mCtpCfgDir;
+  LOG(info) << "CTPRunManager initialised.";
+}
+int CTPRunManager::loadRun(const std::string& cfg)
+{
+  LOG(info) << "Loading run: " << cfg;
+  auto now = std::chrono::system_clock::now();
+  long timeStamp = std::chrono::duration_cast(now.time_since_epoch()).count();
+  LOG(info) << "Timestamp real time:" << timeStamp;
+  size_t pos = cfg.find(" ");
+  std::string cfgmod = cfg;
+  if (pos == std::string::npos) {
+    LOG(warning) << "Space not found in CTP config";
+  } else {
+    std::string f = cfg.substr(0, pos);
+    if (f.find("run") == std::string::npos) {
+      double_t tt = std::stold(f);
+      timeStamp = (tt * 1000.);
+      LOG(info) << "Timestamp file:" << timeStamp;
+      cfgmod = cfg.substr(pos, cfg.size());
+      LOG(info) << "ctpconfig: using ctp time";
+    }
+  }
+  CTPActiveRun* activerun = new CTPActiveRun;
+  activerun->timeStart = timeStamp;
+  activerun->cfg.loadConfigurationRun3(cfgmod);
+  activerun->cfg.printStream(std::cout);
+  //
+  uint32_t runnumber = activerun->cfg.getRunNumber();
+  activerun->scalers.setRunNumber(runnumber);
+  activerun->scalers.setClassMask(activerun->cfg.getTriggerClassMask());
+  o2::detectors::DetID::mask_t detmask = activerun->cfg.getDetectorMask();
+  activerun->scalers.setDetectorMask(detmask);
+  //
+  mRunsLoaded[runnumber] = activerun;
+  saveRunConfigToCCDB(&activerun->cfg, timeStamp);
+  if (mCtpCfgDir != "none") {
+    saveCtpCfg(runnumber, timeStamp);
+  }
+  return 0;
+}
+int CTPRunManager::setRunConfigBK(uint32_t runNumber, const std::string& cfg)
+{
+  std::cout << "Printing run:" << runNumber << " cfg:" << cfg << std::endl;
+  if (mBKClient) {
+    try {
+      mBKClient->run()->setRawCtpTriggerConfiguration(runNumber, cfg);
+    } catch (std::runtime_error& error) {
+      std::cerr << "An error occurred: " << error.what() << std::endl;
+      return 1;
+    }
+    LOG(info) << "Run BK:" << runNumber << " CFG:" << cfg;
+  }
+  return 0;
+}
+int CTPRunManager::stopRun(uint32_t irun, long timeStamp)
+{
+  LOG(info) << "Stopping run index: " << irun;
+  if (mActiveRuns[irun] == nullptr) {
+    LOG(error) << "No config for run index:" << irun;
+    return 1;
+  }
+  // const auto now = std::chrono::system_clock::now();
+  // const long timeStamp = std::chrono::duration_cast(now.time_since_epoch()).count();
+  mActiveRuns[irun]->timeStop = timeStamp * 1000.;
+  saveRunScalersToCCDB(mActiveRuns[irun]->scalers, mActiveRuns[irun]->timeStart, mActiveRuns[irun]->timeStop);
+  saveRunScalersToQCDB(mActiveRuns[irun]->scalers, mActiveRuns[irun]->timeStart, mActiveRuns[irun]->timeStop);
+  delete mActiveRuns[irun];
+  mActiveRuns[irun] = nullptr;
+  return 0;
+}
+int CTPRunManager::addScalers(uint32_t irun, std::time_t time, bool start)
+{
+  if (mActiveRuns[irun] == nullptr) {
+    LOG(error) << "No config for run index:" << irun;
+    return 1;
+  }
+  std::string orb = "extorb";
+  std::string orbitid = "orbitid";
+  CTPScalerRecordRaw scalrec;
+  scalrec.epochTime = time;
+  std::vector clslist = mActiveRuns[irun]->cfg.getTriggerClassList();
+  for (auto const& cls : clslist) {
+    std::string cmb = "clamb" + std::to_string(cls + 1);
+    std::string cma = "clama" + std::to_string(cls + 1);
+    std::string c0b = "cla0b" + std::to_string(cls + 1);
+    std::string c0a = "cla0a" + std::to_string(cls + 1);
+    std::string c1b = "cla1b" + std::to_string(cls + 1);
+    std::string c1a = "cla1a" + std::to_string(cls + 1);
+    CTPScalerRaw scalraw;
+    scalraw.classIndex = (uint32_t)cls;
+    // std::cout << "cls:" << cls << " " << scalraw.classIndex << std::endl;
+    scalraw.lmBefore = mCounters[mScalerName2Position[cmb]];
+    scalraw.lmAfter = mCounters[mScalerName2Position[cma]];
+    scalraw.l0Before = mCounters[mScalerName2Position[c0b]];
+    scalraw.l0After = mCounters[mScalerName2Position[c0a]];
+    scalraw.l1Before = mCounters[mScalerName2Position[c1b]];
+    scalraw.l1After = mCounters[mScalerName2Position[c1a]];
+    // std::cout << "positions:" << cmb << " " <<  mScalerName2Position[cmb] << std::endl;
+    // std::cout << "positions:" << cma << " " <<  mScalerName2Position[cma] << std::endl;
+    scalrec.scalers.push_back(scalraw);
+    // BK scalers to be corrected for overflow
+    if (mBKClient) {
+      CTPActiveRun* ar = mActiveRuns[irun];
+      ar->cntslast[cls][0] = scalraw.lmBefore;
+      ar->cntslast[cls][1] = scalraw.lmAfter;
+      ar->cntslast[cls][2] = scalraw.l0Before;
+      ar->cntslast[cls][3] = scalraw.l0After;
+      ar->cntslast[cls][4] = scalraw.l1Before;
+      ar->cntslast[cls][5] = scalraw.l1After;
+    }
+  }
+  mActiveRuns[irun]->send2BK(mBKClient, time, start);
+  //
+  uint32_t NINPS = 48;
+  int offset = 599;
+  for (uint32_t i = 0; i < NINPS; i++) {
+    uint32_t inpcount = mCounters[offset + i];
+    scalrec.scalersInps.push_back(inpcount);
+    // LOG(info) << "Scaler for input:" << CTPRunScalers::scalerNames[offset+i] << ":" << inpcount;
+  }
+  //
+  if (mNew == 0) {
+    scalrec.intRecord.orbit = mCounters[mScalerName2Position[orb]];
+  } else {
+    scalrec.intRecord.orbit = mCounters[mScalerName2Position[orbitid]];
+  }
+  scalrec.intRecord.bc = 0;
+  mActiveRuns[irun]->scalers.addScalerRacordRaw(scalrec);
+  LOG(debug) << "Adding scalers for orbit:" << scalrec.intRecord.orbit;
+  // scalrec.printStream(std::cout);
+  // printCounters();
+  return 0;
+}
+int CTPRunManager::processMessage(std::string& topic, const std::string& message)
+{
+  LOG(debug) << "Processing message with topic:" << topic;
+  std::string firstcounters;
+  if (topic.find("clear") != std::string::npos) {
+    mRunsLoaded.clear();
+    LOG(info) << "Loaded runs cleared.";
+    return 0;
+  }
+  if (topic.find("ctpconfig") != std::string::npos) {
+    LOG(info) << "ctpconfig received";
+    loadRun(message);
+    return 0;
+  }
+  if (topic.find("soxorbit") != std::string::npos) {
+    std::vector tokens = o2::utils::Str::tokenize(message, ' ');
+    int ret = 0;
+    if (tokens.size() == 3) {
+      long timestamp = std::stol(tokens[0]);
+      uint32_t runnumber = std::stoul(tokens[1]);
+      uint32_t orbit = std::stoul(tokens[2]);
+      ret = saveSoxOrbit(runnumber, orbit, timestamp);
+      std::string logmessage;
+      if (ret) {
+        logmessage = "Failed to update CCDB with SOX orbit.";
+      } else {
+        logmessage = "CCDB updated with SOX orbit.";
+      }
+      LOG(important) << logmessage << " run:" << runnumber << " sox orbit:" << orbit << " ts:" << timestamp;
+    } else {
+      LOG(error) << "Topic soxorbit dize !=3: " << message << " token size:" << tokens.size();
+      ret = 1;
+    }
+    return ret;
+  }
+  if (topic.find("orbitreset") != std::string::npos) {
+    std::vector tokens = o2::utils::Str::tokenize(message, ' ');
+    int ret = 0;
+    if (tokens.size() == 1) {
+      long timestamp = std::stol(tokens[0]);
+      ret = saveOrbitReset(timestamp);
+      std::string logmessage;
+      if (ret) {
+        logmessage = "Failed to update CCDB with orbitreset. ";
+      } else {
+        logmessage = "CCDB updated with orbitreset. ";
+      }
+      LOG(important) << logmessage << timestamp;
+    } else {
+      LOG(error) << "Topic orbit reset != 2: " << message << " token size:" << tokens.size();
+      ret = 1;
+    }
+    return ret;
+  }
+  if (topic.find("rocnts") != std::string::npos) {
+    return 0;
+  }
+  static int nerror = 0;
+  if (topic.find("sox") != std::string::npos) {
+    // get config
+    size_t irun = message.find("run");
+    if (irun == std::string::npos) {
+      LOG(debug) << "run keyword not found in SOX";
+      irun = message.size();
+    }
+    LOG(info) << "SOX received, Run keyword position:" << irun;
+    std::string cfg = message.substr(irun, message.size() - irun);
+    firstcounters = message.substr(0, irun);
+  } else if (topic.find("eox") != std::string::npos) {
+    LOG(info) << "EOX received";
+    mEOX = 1;
+  } else if (topic.find("cnts") != std::string::npos) {
+    // just continue
+  } else {
+    if (nerror < 10) {
+      LOG(warning) << "Skipping topic:" << topic;
+      nerror++;
+    }
+    return 0;
+  }
+  //
+  std::vector tokens;
+  if (firstcounters.size() > 0) {
+    tokens = o2::utils::Str::tokenize(firstcounters, ' ');
+  } else {
+    tokens = o2::utils::Str::tokenize(message, ' ');
+  }
+  if (tokens.size() != (CTPRunScalers::NCOUNTERS + 1)) {
+    if (tokens.size() == (CTPRunScalers::NCOUNTERSv2 + 1)) {
+      mNew = 0;
+      LOG(warning) << "v2 scaler size";
+    } else {
+      LOG(warning) << "Scalers size wrong:" << tokens.size() << " expected:" << CTPRunScalers::NCOUNTERS + 1 << " or " << CTPRunScalers::NCOUNTERSv2 + 1;
+      return 1;
+    }
+  }
+  double timeStamp = std::stold(tokens.at(0));
+  std::time_t tt = timeStamp;
+  LOG(debug) << "Processing scalers, all good, time:" << tokens.at(0) << " " << std::asctime(std::localtime(&tt));
+  for (uint32_t i = 1; i < tokens.size(); i++) {
+    mCounters[i - 1] = std::stoull(tokens.at(i));
+    if (i < (NRUNS + 1)) {
+      // std::cout << mCounters[i - 1] << " ";
+    }
+  }
+  // std::cout << std::endl;
+  LOG(debug) << "Counter size:" << tokens.size();
+  //
+  for (uint32_t i = 0; i < NRUNS; i++) {
+    if ((mCounters[i] == 0) && (mActiveRunNumbers[i] == 0)) {
+      // not active
+    } else if ((mCounters[i] != 0) && (mActiveRunNumbers[i] == mCounters[i])) {
+      // active , do scalers
+      LOG(debug) << "Run continue:" << mCounters[i];
+      addScalers(i, tt);
+      // LOG(info) << " QC period:" << mActiveRunNumbers[i] << " " << mActiveRuns[i]->qcwpcount << " " << mQCWritePeriod;
+      if (mActiveRuns[i]->qcwpcount > mQCWritePeriod) {
+        saveRunScalersToQCDB(mActiveRuns[i]->scalers, tt * 1000, tt * 1000);
+        mActiveRuns[i]->qcwpcount = 0;
+      }
+      mActiveRuns[i]->qcwpcount++;
+    } else if ((mCounters[i] != 0) && (mActiveRunNumbers[i] == 0)) {
+      LOG(info) << "Run started:" << mCounters[i];
+      auto run = mRunsLoaded.find(mCounters[i]);
+      if (run != mRunsLoaded.end()) {
+        mActiveRunNumbers[i] = mCounters[i];
+        mActiveRuns[i] = run->second;
+        mRunsLoaded.erase(run);
+        setRunConfigBK(mActiveRuns[i]->cfg.getRunNumber(), mActiveRuns[i]->cfg.getConfigString());
+        addScalers(i, tt, 1);
+        saveRunScalersToQCDB(mActiveRuns[i]->scalers, tt * 1000, tt * 1000);
+      } else {
+        LOG(error) << "Trying to start run which is not loaded:" << mCounters[i];
+      }
+    } else if ((mCounters[i] == 0) && (mActiveRunNumbers[i] != 0)) {
+      if (mEOX != 1) {
+        LOG(error) << "Internal error in processMessage: mEOX != 1 expected 0: mEOX:" << mEOX;
+      }
+      LOG(info) << "Run stopped:" << mActiveRunNumbers[i];
+      addScalers(i, tt);
+      mActiveRunNumbers[i] = 0;
+      mEOX = 0;
+      stopRun(i, tt);
+    }
+  }
+  mEOX = 0;
+  // printActiveRuns();
+  return 0;
+}
+void CTPRunManager::printActiveRuns() const
+{
+  std::cout << "Active runs:";
+  for (auto const& arun : mActiveRunNumbers) {
+    std::cout << arun << " ";
+  }
+  std::cout << " #loaded runs:" << mRunsLoaded.size();
+  for (auto const& lrun : mRunsLoaded) {
+    std::cout << " " << lrun.second->cfg.getRunNumber();
+  }
+  std::cout << std::endl;
+}
+int CTPRunManager::loadScalerNames()
+{
+  if (CTPRunScalers::NCOUNTERS != CTPRunScalers::scalerNames.size()) {
+    LOG(fatal) << "NCOUNTERS:" << CTPRunScalers::NCOUNTERS << " different from names vector:" << CTPRunScalers::scalerNames.size();
+    return 1;
+  }
+  // try to open files of no success use default
+  for (uint32_t i = 0; i < CTPRunScalers::scalerNames.size(); i++) {
+    mScalerName2Position[CTPRunScalers::scalerNames[i]] = i;
+  }
+  return 0;
+}
+int CTPRunManager::getNRuns()
+{
+  int n = 0;
+  for (int i = 0; i < NRUNS; i++) {
+    if (mActiveRuns[i] != nullptr) {
+      n++;
+    }
+  }
+  return n;
+}
+void CTPRunManager::printCounters()
+{
+  int NDET = 18;
+  int NINPS = 48;
+  // int NCLKFP = 7;
+  int NLTG_start = NRUNS;
+  int NCLKFP_start = NLTG_start + NDET * 32;
+  int NINPS_start = NCLKFP_start + 7;
+  int NCLS_start = NINPS_start + NINPS;
+  std::cout << "====> CTP counters:" << std::endl;
+  std::cout << "RUNS:" << std::endl;
+  int ipos = 0;
+  for (uint32_t i = 0; i < NRUNS; i++) {
+    std::cout << ipos << ":" << mCounters[i] << " ";
+    ipos++;
+  }
+  std::cout << std::endl;
+  for (int i = 0; i < NDET; i++) {
+    std::cout << "LTG" << i + 1 << std::endl;
+    for (int j = NLTG_start + i * 32; j < NLTG_start + (i + 1) * 32; j++) {
+      std::cout << ipos << ":" << mCounters[j] << " ";
+      ipos++;
+    }
+    std::cout << std::endl;
+  }
+  std::cout << "BC40,BC240,Orbit,pulser, fastlm, busy,spare" << std::endl;
+  for (int i = NCLKFP_start; i < NCLKFP_start + 7; i++) {
+    std::cout << ipos << ":" << mCounters[i] << " ";
+    ipos++;
+  }
+  std::cout << std::endl;
+  std::cout << "INPUTS:" << std::endl;
+  for (int i = NINPS_start; i < NINPS_start + NINPS; i++) {
+    std::cout << ipos << ":" << mCounters[i] << " ";
+    ipos++;
+  }
+  std::cout << std::endl;
+  std::cout << "CLASS M Before" << std::endl;
+  for (int i = NCLS_start; i < NCLS_start + 64; i++) {
+    std::cout << ipos << ":" << mCounters[i] << " ";
+    ipos++;
+  }
+  std::cout << std::endl;
+  std::cout << "CLASS M After" << std::endl;
+  for (int i = NCLS_start + 64; i < NCLS_start + 2 * 64; i++) {
+    std::cout << ipos << ":" << mCounters[i] << " ";
+    ipos++;
+  }
+  std::cout << std::endl;
+  std::cout << "CLASS 0 Before" << std::endl;
+  for (int i = NCLS_start + 2 * 64; i < NCLS_start + 3 * 64; i++) {
+    std::cout << ipos << ":" << mCounters[i] << " ";
+    ipos++;
+  }
+  std::cout << std::endl;
+  std::cout << "CLASS 0 After" << std::endl;
+  for (int i = NCLS_start + 3 * 64; i < NCLS_start + 4 * 64; i++) {
+    std::cout << ipos << ":" << mCounters[i] << " ";
+    ipos++;
+  }
+  std::cout << std::endl;
+  std::cout << "CLASS 1 Before" << std::endl;
+  for (int i = NCLS_start + 4 * 64; i < NCLS_start + 5 * 64; i++) {
+    std::cout << ipos << ":" << mCounters[i] << " ";
+    ipos++;
+  }
+  std::cout << std::endl;
+  std::cout << "CLASS 1 After" << std::endl;
+  for (int i = NCLS_start + 5 * 64; i < NCLS_start + 6 * 64; i++) {
+    std::cout << ipos << ":" << mCounters[i] << " ";
+    ipos++;
+  }
+  std::cout << std::endl;
+  std::cout << " REST:" << std::endl;
+  for (uint32_t i = NCLS_start + 6 * 64; i < mCounters.size(); i++) {
+    if ((ipos % 10) == 0) {
+      std::cout << std::endl;
+      std::cout << ipos << ":";
+    }
+    std::cout << mCounters[i] << " ";
+  }
+}
diff --git a/Detectors/CTP/workflowScalers/src/ctp-bk-write.cxx b/Detectors/CTP/workflowScalers/src/ctp-bk-write.cxx
new file mode 100644
index 0000000000000..8460c07dcc896
--- /dev/null
+++ b/Detectors/CTP/workflowScalers/src/ctp-bk-write.cxx
@@ -0,0 +1,170 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+// example to run:
+//
+#include 
+#include 
+#include 
+#include 
+#include "CommonUtils/StringUtils.h"
+#include 
+#include "CTPWorkflowScalers/ctpCCDBManager.h"
+#include "BookkeepingApi/BkpClientFactory.h"
+#include "BookkeepingApi/BkpClient.h"
+#include 
+#include 
+#include 
+#include 
+namespace bpo = boost::program_options;
+//
+// Test in the lab
+// o2-ctp-bk-write -r 37 -s 1 -c 1 --ccdb='http://acsl-ccdb.cern.ch:8083' -b 'acsl-aliecs.cern.ch:4001' -t 1753185071753
+//
+int main(int argc, char** argv)
+{
+  const std::string testCCDB = "http://ccdb-test.cern.ch:8080";
+  // std::string prodCCDB = "http://o2-ccdb.internal";
+  const std::string aliceCCDB = "http://alice-ccdb.cern.ch";
+  bpo::variables_map vm;
+  bpo::options_description opt_general("Usage:\n  " + std::string(argv[0]) +
+                                       " Write ctp config or scalers to BK\n");
+  bpo::options_description opt_hidden("");
+  bpo::options_description opt_all;
+  bpo::positional_options_description opt_pos;
+  try {
+    auto add_option = opt_general.add_options();
+    add_option("help,h", "Print this help message");
+    add_option("input-file,f", bpo::value()->default_value("none"), "input file name, none - do not read file");
+    add_option("bkhost,b", bpo::value()->default_value("none"), "bk web address");
+    add_option("ccdb", bpo::value()->default_value("alice"), "choose databse: test- test ccdb; prod - production ccdb; alice - alice ccdb; else ccdb parameter");
+    add_option("run-number,r", bpo::value()->default_value(0), "run number");
+    add_option("timestamp,t", bpo::value()->default_value(0), "timestamp; if 0 timestamp is calulated inside this code");
+    add_option("cfg,c", bpo::value()->default_value(0), "Do cfg");
+    add_option("scalers,s", bpo::value()->default_value(0), "Do scalers");
+    //
+    opt_all.add(opt_general).add(opt_hidden);
+    bpo::store(bpo::command_line_parser(argc, argv).options(opt_all).positional(opt_pos).run(), vm);
+    if (vm.count("help")) {
+      std::cout << opt_general << std::endl;
+      exit(0);
+    }
+    bpo::notify(vm);
+  } catch (bpo::error& e) {
+    std::cerr << "ERROR: " << e.what() << std::endl
+              << std::endl;
+    std::cerr << opt_general << std::endl;
+    exit(1);
+  } catch (std::exception& e) {
+    std::cerr << e.what() << ", application will now exit" << std::endl;
+    exit(2);
+  }
+  uint64_t timestamp = vm["timestamp"].as();
+  //
+  int ret = 0;
+  std::vector runs;
+  int32_t run = vm["run-number"].as();
+  std::cout << "run:" << run << std::endl;
+  if (run) {
+    std::cout << "pushing" << std::endl;
+    runs.push_back(std::to_string(run));
+  }
+  // read input file
+  std::string filename = vm["input-file"].as();
+  if (filename != "none") {
+    std::ifstream file(filename);
+    if (!file.is_open()) {
+      LOG(fatal) << "Cannot open file:" << filename << std::endl;
+    } else {
+      std::string line;
+      while (std::getline(file, line)) {
+        std::cout << line << "\n";
+        std::vector tokens = o2::utils::Str::tokenize(line, ' ');
+        // int run = std::stoi(tokens[0]);
+        runs.push_back(tokens[0]);
+      }
+    }
+  }
+  bool cfg = vm["cfg"].as();
+  bool scalers = vm["scalers"].as();
+  std::cout << "Doing: cfg:" << cfg << " scal:" << scalers << std::endl;
+  if (cfg || scalers) {
+    std::string bkhost = vm["bkhost"].as();
+    std::unique_ptr mBKClient = o2::bkp::api::BkpClientFactory::create(bkhost);
+    // get from ccdb
+    std::string ccdbAddress;
+    if (vm["ccdb"].as() == "prod") {
+      // ccdbAddress = prodCCDB;
+    } else if (vm["ccdb"].as() == "test") {
+      ccdbAddress = testCCDB;
+    } else if (vm["ccdb"].as() == "alice") {
+      ccdbAddress = aliceCCDB;
+    } else {
+      ccdbAddress = vm["ccdb"].as();
+    }
+    o2::ctp::ctpCCDBManager::setCCDBHost(ccdbAddress);
+    std::cout << "CCDB: " << vm["ccdb"].as() << " " << ccdbAddress << std::endl;
+    std::map metadata;
+    for (auto const& run : runs) {
+      metadata["runNumber"] = run;
+      bool ok;
+      int runNumber = std::stoi(run);
+      auto ctpcfg = o2::ctp::ctpCCDBManager::getConfigFromCCDB(timestamp, run, ok);
+
+      if (cfg) {
+        std::string ctpcfgstr = ctpcfg.getConfigString();
+        try {
+          mBKClient->run()->setRawCtpTriggerConfiguration(runNumber, ctpcfgstr);
+        } catch (std::runtime_error& error) {
+          std::cerr << "An error occurred: " << error.what() << std::endl;
+          // return 1;
+        }
+        LOG(info) << "Run BK:" << run << " CFG:" << cfg;
+      }
+      if (scalers) {
+        auto ctpcnts = o2::ctp::ctpCCDBManager::getScalersFromCCDB(timestamp, run, "CTP/Calib/Scalers", ok);
+        ctpcnts.convertRawToO2();
+        std::vector clsinds = ctpcnts.getClassIndexes();
+        long ts = ctpcnts.getTimeLimit().second;
+        int i = 0;
+        for (auto const& ind : clsinds) {
+          std::array cntsbk = ctpcnts.getIntegralForClass(i);
+          std::string clsname = ctpcfg.getClassNameFromHWIndex(cntsbk[0]);
+          try {
+            mBKClient->ctpTriggerCounters()->createOrUpdateForRun(runNumber, clsname, ts, cntsbk[1], cntsbk[2], cntsbk[3], cntsbk[4], cntsbk[5], cntsbk[6]);
+            std::cout << runNumber << " clsname: " << cntsbk[0] << " " << clsname << " t:" << ts << " cnts:" << cntsbk[1] << " " << cntsbk[2] << " " << cntsbk[3] << " " << cntsbk[4] << " " << cntsbk[5] << " " << cntsbk[6] << std::endl;
+            ;
+
+          } catch (std::runtime_error& error) {
+            std::cerr << "An error occurred: " << error.what() << std::endl;
+            // return 1;
+          }
+          LOG(debug) << "Run BK scalers ok";
+          i++;
+        }
+      }
+    }
+    // add to bk
+  }
+  std::cout << "o2-ctp-bk-write done" << std::endl;
+  return ret;
+}
diff --git a/Detectors/CTP/workflowScalers/src/ctp-ccdb-orbit.cxx b/Detectors/CTP/workflowScalers/src/ctp-ccdb-orbit.cxx
new file mode 100644
index 0000000000000..aa953e89264ef
--- /dev/null
+++ b/Detectors/CTP/workflowScalers/src/ctp-ccdb-orbit.cxx
@@ -0,0 +1,151 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+// example to run:
+//
+#include 
+#include 
+#include 
+#include 
+#include "CommonUtils/StringUtils.h"
+#include 
+#include 
+#include 
+#include 
+namespace bpo = boost::program_options;
+//
+// get object from ccdb
+// auto & cc = o2::ccdb::BasicCCDNManager::instance();
+// auto pp = ccdbMgr.getSpecific>("CTP/Calib/OrbitResetTest")
+// std::cout  << (*pp)[0] << std::endl;
+int main(int argc, char** argv)
+{
+  const std::string testCCDB = "http://ccdb-test.cern.ch:8080";
+  const std::string prodCCDB = "http://o2-ccdb.internal";
+  bpo::variables_map vm;
+  bpo::options_description opt_general("Usage:\n  " + std::string(argv[0]) +
+                                       " Write orbit staff to ccdb\n");
+  bpo::options_description opt_hidden("");
+  bpo::options_description opt_all;
+  bpo::positional_options_description opt_pos;
+  try {
+    auto add_option = opt_general.add_options();
+    add_option("help,h", "Print this help message");
+    add_option("output-file,o", bpo::value()->default_value("none"), "output file name, none - file not created");
+    add_option("output-dir,d", bpo::value()->default_value("./"), "output dir");
+    add_option("ccdb", bpo::value()->default_value("test"), "choose databse: test- test ccdb; prod - production ccdb; else ccdb parameter");
+    add_option("action,a", bpo::value()->default_value(""), "sox - first orbit otherwise orbit reset");
+    add_option("run-number,r", bpo::value()->default_value(123), "run number");
+    add_option("testReset,t", bpo::value()->default_value(0), "0 = CTP/Calib/OrbitReset; 1 = CTP/Calib/OrbitResetTest");
+    add_option("sox-orbit,x", bpo::value()->default_value(0), "SOX orbit");
+    add_option("timestamp,s", bpo::value()->default_value(0), "timestamp of SOX/orbit reading; if 0 timestamp is calulated inside this code");
+
+    //
+    opt_all.add(opt_general).add(opt_hidden);
+    bpo::store(bpo::command_line_parser(argc, argv).options(opt_all).positional(opt_pos).run(), vm);
+    if (vm.count("help")) {
+      std::cout << opt_general << std::endl;
+      exit(0);
+    }
+    bpo::notify(vm);
+  } catch (bpo::error& e) {
+    std::cerr << "ERROR: " << e.what() << std::endl
+              << std::endl;
+    std::cerr << opt_general << std::endl;
+    exit(1);
+  } catch (std::exception& e) {
+    std::cerr << e.what() << ", application will now exit" << std::endl;
+    exit(2);
+  }
+  int ret = 0;
+  std::string action = vm["action"].as();
+  std::vector vect;
+  std::string ccdbPath;
+  long tt = vm["timestamp"].as();
+  if (tt == 0) {
+    auto now = std::chrono::system_clock::now();
+    tt = std::chrono::duration_cast(now.time_since_epoch()).count();
+  }
+  vect.push_back(tt);
+  if (action == "sox") {
+    // write to CTP/Calib/FirstRunOrbit
+    std::cout << "===> FirsRunOrbit" << std::endl;
+    vect.push_back(vm["run-number"].as());
+    vect.push_back(vm["sox-orbit"].as());
+    ccdbPath = "CTP/Calib/FirstRunOrbit";
+  } else {
+    // write to CTP/Calib/OrbitReset
+    std::cout << "===> ResetOrbit" << std::endl;
+    ccdbPath = "CTP/Calib/OrbitReset";
+    if (vm["testReset"].as()) {
+      ccdbPath += "Test";
+    }
+  }
+  //
+  std::string ccdbAddress;
+  if (vm["ccdb"].as() == "prod") {
+    ccdbAddress = prodCCDB;
+  } else if (vm["ccdb"].as() == "test") {
+    ccdbAddress = testCCDB;
+  } else {
+    ccdbAddress = vm["ccdb"].as();
+  }
+  std::cout << " Writing to db:" << ccdbAddress << std::endl;
+  if (ccdbAddress != "none") {
+    // auto& ccdbMgr = o2::ccdb::BasicCCDBManager::instance();
+    o2::ccdb::CcdbApi api;
+    api.init(ccdbAddress.c_str());
+    std::map metadata;
+    long tmin = tt / 1000;
+    long tmax = tmin + 381928219;
+    if (action == "sox") {
+      int64_t runnum = vm["run-number"].as();
+      metadata["runNumber"] = std::to_string(runnum);
+      std::cout << "Storing:" << ccdbPath << " " << metadata["runNumber"] << " tmin:" << tmin << " tmax:" << tmax << " ts:" << tt << std::endl;
+      ret = api.storeAsTFileAny(&(vect), ccdbPath, metadata, tmin, tmax);
+    } else {
+      std::cout << "Storing:" << ccdbPath << " tmin:" << tmin << " tmax:" << tmax << " ts:" << tt << std::endl;
+      std::string filename = "orbitReset.root";
+      auto classname = "std::vector";
+      metadata["adjustableEOV"] = "true";
+      ret = api.storeAsTFileAny(&(vect), ccdbPath, metadata, tmin, tmax);
+      o2::ccdb::CcdbObjectInfo oi(ccdbPath, classname, filename, metadata, tmin, tmax);
+      adjustOverriddenEOV(api, oi);
+    }
+  }
+  //
+  if (vm["output-file"].as() != "none") {
+    std::string file = vm["output-dir"].as() + vm["output-file"].as();
+    TFile* f = TFile::Open(file.c_str(), "RECREATE");
+    if (f == nullptr) {
+      std::cout << "Error: File" << file << " could not be open for writing !!!" << std::endl;
+      ret++;
+    } else {
+      std::cout << "File" << file << " being writen." << std::endl;
+      f->WriteObject(&vect, "ccdb_object");
+      f->Close();
+    }
+  } else {
+    std::cout << "No file created" << std::endl;
+  }
+  return ret;
+}
diff --git a/Detectors/CTP/workflowScalers/src/ctp-proxy.cxx b/Detectors/CTP/workflowScalers/src/ctp-proxy.cxx
index 146b01f8c3960..391d1b5ccf58b 100644
--- a/Detectors/CTP/workflowScalers/src/ctp-proxy.cxx
+++ b/Detectors/CTP/workflowScalers/src/ctp-proxy.cxx
@@ -29,6 +29,7 @@
 #include "Framework/WorkflowSpec.h"
 #include "Framework/DataProcessorSpec.h"
 #include "Framework/DataSpecUtils.h"
+#include "Framework/RawDeviceService.h"
 #include "Framework/ControlService.h"
 #include "Framework/Logger.h"
 #include "Framework/Lifetime.h"
@@ -39,30 +40,44 @@
 #include 
 #include 
 #include "CommonUtils/StringUtils.h"
-#include "DataFormatsCTP/Configuration.h"
+#include "CTPWorkflowScalers/RunManager.h"
 #include 
 #include 
-
+#include "BookkeepingApi/BkpClient.h"
 using namespace o2::framework;
 using DetID = o2::detectors::DetID;
-InjectorFunction dcs2dpl(std::string& ccdbhost)
-// InjectorFunction dcs2dpl()
+InjectorFunction dcs2dpl(std::string& ccdbhost, std::string& bkhost, std::string& qchost, int qcwriteperiod, std::string& ctpcfgdir)
 {
-  auto timesliceId = std::make_shared(0);
   auto runMgr = std::make_shared();
   runMgr->setCCDBHost(ccdbhost);
+  runMgr->setBKHost(bkhost);
+  runMgr->setQCDBHost(qchost);
+  runMgr->setQCWritePeriod(qcwriteperiod);
+  runMgr->setCtpCfgDir(ctpcfgdir);
   runMgr->init();
-  return [timesliceId, runMgr](TimingInfo&, fair::mq::Device& device, fair::mq::Parts& parts, ChannelRetriever channelRetriever) {
+  // runMgr->setClient(client);
+  static int nprint = 0;
+  return [runMgr](TimingInfo&, ServiceRegistryRef const& services, fair::mq::Parts& parts, ChannelRetriever channelRetriever, size_t newTimesliceId, bool& stop) -> bool {
+    // FIXME: Why isn't this function using the timeslice index?
     // make sure just 2 messages received
-    if (parts.Size() != 2) {
-      LOG(error) << "received " << parts.Size() << " instead of 2 expected";
-      return;
-    }
+    // if (parts.Size() != 2) {
+    //  LOG(error) << "received " << parts.Size() << " instead of 2 expected";
+    //  return;
+    //}
     std::string messageHeader{static_cast(parts.At(0)->GetData()), parts.At(0)->GetSize()};
     size_t dataSize = parts.At(1)->GetSize();
     std::string messageData{static_cast(parts.At(1)->GetData()), parts.At(1)->GetSize()};
-    LOG(info) << "received message " << messageHeader << " of size " << dataSize; // << " Payload:" << messageData;
+    nprint++;
+    int nlimit = 60;
+    int nrange = 8;
+    if (nprint > nlimit && nprint < (nlimit + nrange + 1)) {
+      LOG(info) << "received message " << messageHeader << " of size " << dataSize << " # parts:" << parts.Size(); // << " Payload:" << messageData;
+      if (nprint == (nlimit + nrange)) {
+        nprint = 0;
+      }
+    }
     runMgr->processMessage(messageHeader, messageData);
+    return true;
   };
 }
 
@@ -71,6 +86,10 @@ void customize(std::vector& workflowOptions)
 {
   workflowOptions.push_back(ConfigParamSpec{"subscribe-to", VariantType::String, "type=sub,method=connect,address=tcp://188.184.30.57:5556,rateLogging=10,transport=zeromq", {"channel subscribe to"}});
   workflowOptions.push_back(ConfigParamSpec{"ccdb-host", VariantType::String, "http://o2-ccdb.internal:8080", {"ccdb host"}});
+  workflowOptions.push_back(ConfigParamSpec{"bk-host", VariantType::String, "none", {"bk host"}});
+  workflowOptions.push_back(ConfigParamSpec{"qc-host", VariantType::String, "none", {"qc host"}});
+  workflowOptions.push_back(ConfigParamSpec{"ctpcfg-dir", VariantType::String, "none", {"ctp.cfg file directory"}});
+  workflowOptions.push_back(ConfigParamSpec{"qc-writeperiod", VariantType::Int, 30, {"Period of writing to QCDB in units of 10secs, default = 30 (5 mins)"}});
 }
 
 #include "Framework/runDataProcessing.h"
@@ -93,6 +112,10 @@ WorkflowSpec defineDataProcessing(ConfigContext const& config)
   const std::string devName = "ctp-proxy";
   auto chan = config.options().get("subscribe-to");
   std::string ccdbhost = config.options().get("ccdb-host");
+  std::string bkhost = config.options().get("bk-host");
+  std::string qchost = config.options().get("qc-host");
+  int qcwriteperiod = config.options().get("qc-writeperiod");
+  std::string ctpcfgdir = config.options().get("ctpcfg-dir");
   if (chan.empty()) {
     throw std::runtime_error("input channel is not provided");
   }
@@ -107,7 +130,8 @@ WorkflowSpec defineDataProcessing(ConfigContext const& config)
     std::move(ctpCountersOutputs),
     // this is just default, can be overriden by --ctp-config-proxy '--channel-config..'
     chan.c_str(),
-    dcs2dpl(ccdbhost));
+    dcs2dpl(ccdbhost, bkhost, qchost, qcwriteperiod, ctpcfgdir));
+  ctpProxy.labels.emplace_back(DataProcessorLabel{"input-proxy"});
   LOG(info) << "===> Proxy done";
   WorkflowSpec workflow;
   workflow.emplace_back(ctpProxy);
diff --git a/Detectors/CTP/workflowScalers/src/ctp-qc-proxy.cxx b/Detectors/CTP/workflowScalers/src/ctp-qc-proxy.cxx
index e9ea2b6d47ce6..0a4eeb64cf3af 100644
--- a/Detectors/CTP/workflowScalers/src/ctp-qc-proxy.cxx
+++ b/Detectors/CTP/workflowScalers/src/ctp-qc-proxy.cxx
@@ -28,6 +28,7 @@
 #include "Framework/DataProcessorSpec.h"
 #include "Framework/DataSpecUtils.h"
 #include "Framework/ControlService.h"
+#include "Framework/RawDeviceService.h"
 #include "Framework/Logger.h"
 #include "Framework/Lifetime.h"
 #include "Framework/ConfigParamSpec.h"
@@ -45,34 +46,29 @@ using DetID = o2::detectors::DetID;
 InjectorFunction dcs2dpl()
 // InjectorFunction dcs2dpl()
 {
-  auto timesliceId = std::make_shared(0);
-  return [timesliceId](TimingInfo&, fair::mq::Device& device, fair::mq::Parts& parts, ChannelRetriever channelRetriever) {
-    // make sure just 2 messages received
-    if (parts.Size() != 2) {
-      LOG(error) << "received " << parts.Size() << " instead of 2 expected";
-      return;
-    }
+  return [](TimingInfo&, ServiceRegistryRef const& services, fair::mq::Parts& parts, ChannelRetriever channelRetriever, size_t newTimesliceId, bool&) -> bool {
+    auto* device = services.get().device();
     std::string messageHeader{static_cast(parts.At(0)->GetData()), parts.At(0)->GetSize()};
     size_t dataSize = parts.At(1)->GetSize();
     std::string messageData{static_cast(parts.At(1)->GetData()), parts.At(1)->GetSize()};
-    LOG(info) << "received message " << messageHeader << " of size " << dataSize; // << " Payload:" << messageData;
+    LOG(info) << "received message " << messageHeader << " of size " << dataSize << "#parts:" << parts.Size(); // << " Payload:" << messageData;
     o2::header::DataHeader hdrF("CTP_COUNTERS", o2::header::gDataOriginCTP, 0);
     OutputSpec outsp{hdrF.dataOrigin, hdrF.dataDescription, hdrF.subSpecification};
-    auto channel = channelRetriever(outsp, *timesliceId);
+    auto channel = channelRetriever(outsp, newTimesliceId);
     if (channel.empty()) {
       LOG(error) << "No output channel found for OutputSpec " << outsp;
-      return;
+      return false;
     }
-    hdrF.tfCounter = *timesliceId; // this also
+    hdrF.tfCounter = newTimesliceId; // this also
     hdrF.payloadSerializationMethod = o2::header::gSerializationMethodNone;
     hdrF.splitPayloadParts = 1;
     hdrF.splitPayloadIndex = 0;
     hdrF.payloadSize = parts.At(0)->GetSize() + parts.At(1)->GetSize() + 1;
     hdrF.firstTForbit = 0; // this should be irrelevant for Counters ? Orbit is in payload
 
-    auto fmqFactory = device.GetChannel(channel).Transport();
+    auto fmqFactory = device->GetChannel(channel).Transport();
 
-    o2::header::Stack headerStackF{hdrF, DataProcessingHeader{*timesliceId, 1}};
+    o2::header::Stack headerStackF{hdrF, DataProcessingHeader{newTimesliceId, 1}};
     auto hdMessageF = fmqFactory->CreateMessage(headerStackF.size(), fair::mq::Alignment{64});
     auto plMessageF = fmqFactory->CreateMessage(hdrF.payloadSize, fair::mq::Alignment{64});
     memcpy(hdMessageF->GetData(), headerStackF.data(), headerStackF.size());
@@ -93,8 +89,9 @@ InjectorFunction dcs2dpl()
     fair::mq::Parts outParts;
     outParts.AddPart(std::move(hdMessageF));
     outParts.AddPart(std::move(plMessageF));
-    sendOnChannel(device, outParts, channel, *timesliceId);
+    sendOnChannel(*device, outParts, channel, newTimesliceId);
     LOG(info) << "Sent CTP counters DPL message" << std::flush;
+    return true;
   };
 }
 
@@ -138,6 +135,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const& config)
     // this is just default, can be overriden by --ctp-config-proxy '--channel-config..'
     chan.c_str(),
     dcs2dpl());
+  ctpProxy.labels.emplace_back(DataProcessorLabel{"input-proxy"});
   LOG(info) << "===> Proxy done";
   WorkflowSpec workflow;
   workflow.emplace_back(ctpProxy);
diff --git a/Detectors/CTP/workflowScalers/src/ctpCCDBManager.cxx b/Detectors/CTP/workflowScalers/src/ctpCCDBManager.cxx
new file mode 100644
index 0000000000000..74d4a905c93e2
--- /dev/null
+++ b/Detectors/CTP/workflowScalers/src/ctpCCDBManager.cxx
@@ -0,0 +1,274 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file ctpCCDBManager.cxx
+/// \author Roman Lietava
+#include "CTPWorkflowScalers/ctpCCDBManager.h"
+#include "DataFormatsCTP/Configuration.h"
+#include "CCDB/CcdbApi.h"
+#include "CCDB/BasicCCDBManager.h"
+#include 
+#include 
+#include "CommonUtils/StringUtils.h"
+#include 
+using namespace o2::ctp;
+std::string ctpCCDBManager::mCCDBHost = "http://o2-ccdb.internal";
+std::string ctpCCDBManager::mQCDBHost = "http://ali-qcdb.cern.ch:8083";
+// std::string ctpCCDBManager::mQCDBHost = "none";
+//
+int ctpCCDBManager::saveRunScalersToCCDB(CTPRunScalers& scalers, long timeStart, long timeStop)
+{
+  // data base
+  if (mCCDBHost == "none") {
+    LOG(debug) << "Scalers not written to CCDB none";
+    return 0;
+  }
+  // CTPActiveRun* run = mActiveRuns[i];
+  using namespace std::chrono_literals;
+  std::chrono::seconds days3 = 259200s;
+  std::chrono::seconds min10 = 600s;
+  long time3days = std::chrono::duration_cast(days3).count();
+  long time10min = std::chrono::duration_cast(min10).count();
+  long tmin = timeStart - time10min;
+  long tmax = timeStop + time3days;
+  o2::ccdb::CcdbApi api;
+  std::map metadata; // can be empty
+  metadata["runNumber"] = std::to_string(scalers.getRunNumber());
+  api.init(mCCDBHost.c_str()); // or http://localhost:8080 for a local installation
+  // store abitrary user object in strongly typed manner
+  int ret = api.storeAsTFileAny(&(scalers), mCCDBPathCTPScalers, metadata, tmin, tmax);
+  if (ret == 0) {
+    LOG(info) << "CTP scalers saved in ccdb:" << mCCDBHost << " run:" << scalers.getRunNumber() << " tmin:" << tmin << " tmax:" << tmax;
+  } else {
+    LOG(fatal) << "Problem writing to database ret:" << ret;
+  }
+  return ret;
+}
+int ctpCCDBManager::saveRunScalersToQCDB(CTPRunScalers& scalers, long timeStart, long timeStop)
+{
+  // data base
+  if (mQCDBHost == "none") {
+    LOG(debug) << "Scalers not written to QCDB none";
+    return 0;
+  }
+  // CTPActiveRun* run = mActiveRuns[i];q
+  using namespace std::chrono_literals;
+  std::chrono::seconds days3 = 259200s;
+  std::chrono::seconds min10 = 600s;
+  long time3days = std::chrono::duration_cast(days3).count();
+  long time10min = std::chrono::duration_cast(min10).count();
+  long tmin = timeStart - time10min;
+  long tmax = timeStop + time3days;
+  o2::ccdb::CcdbApi api;
+  std::map metadata; // can be empty
+  metadata["runNumber"] = std::to_string(scalers.getRunNumber());
+  api.init(mQCDBHost.c_str()); // or http://localhost:8080 for a local installation
+  // store abitrary user object in strongly typed manner
+  int ret = api.storeAsTFileAny(&(scalers), mQCDBPathCTPScalers, metadata, tmin, tmax);
+  if (ret == 0) {
+    LOG(info) << "CTP scalers saved in qcdb:" << mQCDBHost << " run:" << scalers.getRunNumber() << " tmin:" << tmin << " tmax:" << tmax;
+  } else {
+    LOG(fatal) << "CTP scalers Problem writing to database qcdb ret:" << ret;
+  }
+  return ret;
+}
+int ctpCCDBManager::saveRunConfigToCCDB(CTPConfiguration* cfg, long timeStart)
+{
+  // data base
+  if (mCCDBHost == "none") {
+    LOG(info) << "CTP config not written to CCDB none";
+    return 0;
+  }
+  using namespace std::chrono_literals;
+  std::chrono::seconds days3 = 259200s;
+  std::chrono::seconds min10 = 600s;
+  long time3days = std::chrono::duration_cast(days3).count();
+  long time10min = std::chrono::duration_cast(min10).count();
+  long tmin = timeStart - time10min;
+  long tmax = timeStart + time3days;
+  o2::ccdb::CcdbApi api;
+  std::map metadata; // can be empty
+  metadata["runNumber"] = std::to_string(cfg->getRunNumber());
+  api.init(mCCDBHost.c_str()); // or http://localhost:8080 for a local installation
+  // store abitrary user object in strongly typed manner
+  int ret = api.storeAsTFileAny(cfg, CCDBPathCTPConfig, metadata, tmin, tmax);
+  if (ret == 0) {
+    LOG(info) << "CTP config  saved in ccdb:" << mCCDBHost << " run:" << cfg->getRunNumber() << " tmin:" << tmin << " tmax:" << tmax;
+  } else {
+    LOG(fatal) << "CTPConfig: Problem writing to database ret:" << ret;
+  }
+  return ret;
+}
+int ctpCCDBManager::saveSoxOrbit(uint32_t runNumber, uint32_t soxOrbit, long timestamp)
+{
+  // data base
+  if (mCCDBHost == "none") {
+    LOG(info) << "SOX Orbit not written to CCDB none";
+    return 0;
+  }
+  std::vector vect;
+  if (timestamp == 0) {
+    auto now = std::chrono::system_clock::now();
+    timestamp = std::chrono::duration_cast(now.time_since_epoch()).count();
+  }
+  vect.push_back(timestamp);
+  vect.push_back((uint64_t)runNumber);
+  vect.push_back((uint64_t)soxOrbit);
+  long tmin = timestamp / 1000;
+  long tmax = tmin + 381928219;
+  o2::ccdb::CcdbApi api;
+  std::map metadata; // can be empty
+  metadata["runNumber"] = std::to_string(runNumber);
+  api.init(mCCDBHost.c_str()); // or http://localhost:8080 for a local installation
+
+  // store abitrary user object in strongly typed manner
+  int ret = api.storeAsTFileAny(&vect, mCCDBPathSoxOrbit, metadata, tmin, tmax);
+  if (ret == 0) {
+    LOG(info) << "SOX orbit  saved in ccdb:" << mCCDBHost << " run:" << runNumber << " tmin:" << tmin << " tmax:" << tmax;
+  } else {
+    LOG(fatal) << "SOX orbit Problem writing to database ret:" << ret;
+  }
+  return 0;
+}
+int ctpCCDBManager::saveOrbitReset(long timeStamp)
+{
+  // data base
+  if (mCCDBHost == "none") {
+    LOG(info) << "Orbit Reset not written to CCDB none";
+    return 0;
+  }
+  std::vector vect;
+  if (timeStamp == 0) {
+    auto now = std::chrono::system_clock::now();
+    timeStamp = std::chrono::duration_cast(now.time_since_epoch()).count();
+    LOG(warn) << "Received timestamp = 0 , using current time:" << timeStamp;
+  }
+  vect.push_back(timeStamp);
+  long tmin = timeStamp / 1000;
+  long tmax = tmin + 381928219;
+  o2::ccdb::CcdbApi api;
+  std::map metadata; // can be empty
+  api.init(mCCDBHost.c_str());  // or http://localhost:8080 for a local installation
+  // int ret = api.storeAsTFileAny(&vect, mCCDBPathOrbitReset, metadata, tmin, tmax);
+  std::cout << "Storing:" << mCCDBPathOrbitReset << " tmin:" << tmin << " tmax:" << tmax << " ts:" << timeStamp << std::endl;
+  std::string filename = "orbitReset.root";
+  auto classname = "std::vector";
+  metadata["adjustableEOV"] = "true";
+  int ret = api.storeAsTFileAny(&(vect), mCCDBPathOrbitReset, metadata, tmin, tmax);
+  o2::ccdb::CcdbObjectInfo oi(mCCDBPathOrbitReset, classname, filename, metadata, tmin, tmax);
+  adjustOverriddenEOV(api, oi);
+  if (ret == 0) {
+    LOG(info) << "Orbit reset  saved in ccdb:" << mCCDBHost << " tmin:" << tmin << " tmax:" << tmax;
+  } else {
+    LOG(fatal) << "Orbit reset Problem writing to database ret:" << ret;
+  }
+  return 0;
+}
+int ctpCCDBManager::saveCtpCfg(uint32_t runNumber, long timeStart)
+{
+  if (mCCDBHost == "none") {
+    LOG(info) << "CtpCfg not written to CCDB none";
+    return 0;
+  }
+  CtpCfg ctpcfg;
+  int ret = ctpcfg.readAndSave(mCtpCfgDir);
+  if (ret == 0) {
+    using namespace std::chrono_literals;
+    std::chrono::seconds days3 = 259200s;
+    std::chrono::seconds min10 = 600s;
+    long time3days = std::chrono::duration_cast(days3).count();
+    long time10min = std::chrono::duration_cast(min10).count();
+    long tmin = timeStart - time10min;
+    long tmax = timeStart + time3days;
+    o2::ccdb::CcdbApi api;
+    std::map metadata; // can be empty
+    metadata["runNumber"] = std::to_string(runNumber);
+    api.init(mCCDBHost.c_str()); // or http://localhost:8080 for a local installation
+    // store abitrary user object in strongly typed manner
+    ret = api.storeAsTFileAny(&ctpcfg, mCCDBPathCtpCfg, metadata, tmin, tmax);
+    if (ret == 0) {
+      LOG(info) << "CtpCfg  saved in ccdb:" << mCCDBHost << " tmin:" << tmin << " tmax:" << tmax;
+    } else {
+      LOG(error) << "CtpCfg Problem writing to database ret:" << ret;
+    }
+  }
+  return ret;
+}
+CTPConfiguration ctpCCDBManager::getConfigFromCCDB(long timestamp, std::string run, bool& ok)
+{
+
+  auto& mgr = o2::ccdb::BasicCCDBManager::instance();
+  mgr.setURL(mCCDBHost);
+  std::map metadata; // can be empty
+  metadata["runNumber"] = run;
+  if (timestamp == 0) {
+    // Timestamp
+    auto soreor = mgr.getRunDuration(std::stoi(run));
+    timestamp = (soreor.second - soreor.first) / 2 + soreor.first;
+  }
+  auto ctpconfigdb = mgr.getSpecific(CCDBPathCTPConfig, timestamp, metadata);
+  if (ctpconfigdb == nullptr) {
+    LOG(info) << "CTP config not in database, timestamp:" << timestamp;
+    ok = 0;
+  } else {
+    // ctpconfigdb->printStream(std::cout);
+    LOG(info) << "CTP config found. Run:" << run;
+    ok = 1;
+  }
+  return *ctpconfigdb;
+}
+CTPConfiguration ctpCCDBManager::getConfigFromCCDB(long timestamp, std::string run)
+{
+  bool ok;
+  auto ctpconfig = getConfigFromCCDB(timestamp, run, ok);
+  if (ok == 0) {
+    LOG(error) << "CTP config not in CCDB";
+    return CTPConfiguration();
+  }
+  return ctpconfig;
+}
+CTPRunScalers ctpCCDBManager::getScalersFromCCDB(long timestamp, std::string run, bool& ok)
+{
+  auto& mgr = o2::ccdb::BasicCCDBManager::instance();
+  mgr.setURL(mCCDBHost);
+  std::map metadata; // can be empty
+  metadata["runNumber"] = run;
+  auto ctpscalers = mgr.getSpecific(mCCDBPathCTPScalers, timestamp, metadata);
+  if (ctpscalers == nullptr) {
+    LOG(info) << "CTPRunScalers not in database, timestamp:" << timestamp;
+    ok = 0;
+  } else {
+    // ctpscalers->printStream(std::cout);
+    ok = 1;
+  }
+  return *ctpscalers;
+}
+CTPRunScalers ctpCCDBManager::getScalersFromCCDB(long timestamp, std::string run, std::string path, bool& ok)
+{
+  auto& mgr = o2::ccdb::BasicCCDBManager::instance();
+  mgr.setURL(mCCDBHost);
+  std::map metadata; // can be empty
+  metadata["runNumber"] = run;
+  if (timestamp == 0) {
+    // Timestamp
+    auto soreor = mgr.getRunDuration(std::stoi(run));
+    timestamp = (soreor.second - soreor.first) / 2 + soreor.first;
+  }
+  auto ctpscalers = mgr.getSpecific(path, timestamp, metadata);
+  if (ctpscalers == nullptr) {
+    LOG(info) << "CTPRunScalers not in database, timestamp:" << timestamp;
+    ok = 0;
+  } else {
+    // ctpscalers->printStream(std::cout);
+    ok = 1;
+  }
+  return *ctpscalers;
+}
\ No newline at end of file
diff --git a/Detectors/Calibration/CMakeLists.txt b/Detectors/Calibration/CMakeLists.txt
index b6892186e8454..f1ff2a6b77dbd 100644
--- a/Detectors/Calibration/CMakeLists.txt
+++ b/Detectors/Calibration/CMakeLists.txt
@@ -16,6 +16,7 @@ o2_add_library(DetectorsCalibration
                    src/MeanVertexData.cxx
                    src/MeanVertexCalibrator.cxx
                    src/MeanVertexParams.cxx
+                   src/IntegratedClusterCalibrator.cxx
                PUBLIC_LINK_LIBRARIES O2::Headers
                    O2::CCDB
                    O2::CommonUtils
@@ -30,10 +31,12 @@ o2_add_library(DetectorsCalibration
 o2_target_root_dictionary(DetectorsCalibration
                           HEADERS include/DetectorsCalibration/TimeSlotCalibration.h
                                   include/DetectorsCalibration/TimeSlot.h
+                                  include/DetectorsCalibration/TimeSlotMetaData.h
                                   include/DetectorsCalibration/Utils.h
                                   include/DetectorsCalibration/MeanVertexData.h
                                   include/DetectorsCalibration/MeanVertexCalibrator.h
-                                  include/DetectorsCalibration/MeanVertexParams.h)
+                                  include/DetectorsCalibration/MeanVertexParams.h
+                                  include/DetectorsCalibration/IntegratedClusterCalibrator.h)
 
 o2_add_executable(ccdb-populator-workflow
                   COMPONENT_NAME calibration
diff --git a/Detectors/Calibration/README.md b/Detectors/Calibration/README.md
index 80e27f4f0aee8..be519be405c07 100644
--- a/Detectors/Calibration/README.md
+++ b/Detectors/Calibration/README.md
@@ -4,52 +4,67 @@
 
 # Time-interval based calibration flow for O2
 
-The calibration flow of O2 foresees that every calibration device (expected to all run on one single aggregation node) will receive the TimeFrames with calibration input from every EPN in an asynchronous way. The calibration device will have to process the TFs in time intervals (TimeSlots) which allow to create CCDB entries with the needed granularity and update frequency (defined by the calibration device itself).
+The calibration flow of O2 foresees that all calibration devices are running on dedicated EPN calibration nodes. These nodes are also called aggregator nodes. A particular calibration device can only run on a single calibration node, as it is supposed to receive the complete input data from each EPN processing node.
+The processing nodes can send the data either for every TF or sporadically. In the latter case for example accumulated for every 10 TFs.
+In both cases the calibration input reaches the calibration nodes asynchronously. And since there are up to 250 EPNs processing TFs in two different NUMA domains, TFs which reach the calibration node consecutively might in reality be 500 TFs apart in absolute time.
+This is because the processing time of individual TFs can vary quite significantly.
+The calibration devices have to prepare the calibration objects for time intervals (TimeSlots) which are defined by the user.
+They have a specificied duration and a minimum statistics requirement.
+For example a calibration is supposed to aggregate data for 10 minutes and for a meaningful calibration within these 10 minutes a certain amount of global tracks is required. In case after 10 minutes the amount of global tracks is not sufficient, the framework can automatically increase the interval of 10 minutes until enough global tracks are available to create the calibration object.
 
-## TimeSlotCalibration
-Each calibration device (to be run in a workflow) has to derive from `o2::calibration::TimeSlotCalibration`, which is a templated class that takes as types the Input type (i.e. the object to be processed, coming from the upstream device) and the Container type (i.e. the object that will contain the calibration data per TimeSlot). Each calibration device has to be configured with the following parameters:
 
-`tf-per-slot` : default length of a TiemSlot in TFs (will be widened in case of too little statistics). If this is set to `o2::calibration::INFINITE_TF`, then there will be only 1 slot at a time, valid till infinity. Value 0 is reserved for a special mode: a single slot w/o explicit boundaries is
-filled until the requested statistics is reached. Once `hasEnoughData` return true, the slot will be closed with really seen min/max TFs and new one will be created with lower boundary equal the end of the previous slot.
-The slot duration can be also set via methods `setSlotLengthInSeconds(int s)` or `setSlotLengthInOrbits(int n)`, which will be internally converted (and rounded) to the number of TFs at the 1st TF processing (when the NHBF per orbit will be available from the GRPECS).
+## TimeSlotCalibration
+Each calibration device which is supposed to run on the aggregator should derive from `o2::calibration::TimeSlotCalibration`. It is a templated class. The `Container` type is the object in which the calibration data per TimeSlot will be accumulated.
 
-`updateInterval` : to be used together with `tf-per-slot = o2::calibration::INFINITE_TF`: it allows to try to finalize the slot (and produce calibration) when the `updateInterval` has passed. Note that this is an approximation (as explained in the code) due to the fact that TFs will come asynchronously (not ordered in time).
+### Configuration of the TimeSlot
 
-`max-delay` : maximum arrival delay of a TF with respect to the most recent one processed; units in number of TimeSlots; if beyond this, the TF will be considered too old, and discarded.
-If `tf-per-slot == o2::calibration::INFINITE_TF`, or `updateAtTheEndOfRunOnly == true`, its value is irrelevant.
+Internally the default length of a TimeSlot is calculated in number of TFs. The default TF length is 128 orbits, but theoretically this can change. Therefore it is advised to set the TimeSlot length via the methods `setSlotLengthInSeconds(int s)` or `setSlotLengthInOrbits(int n)`, which will be internally converted (and rounded) to the corresponding number of TFs at the 1st TF processing. At that time we know the TF length from the GRPECS object.
+One can also set the number of TFs per slot directly via `setSlotLength(o2::calibration::TFType n)`. With `setSlotLength(o2::calibration::INFINITE_TF)` there will be only 1 slot at a time, valid till infinity. A special mode is configured with `setSlotLength(0)` in which case there will only be a single slot, w/o explicit boundaries which is filled until the statistics is reached. In case `setSlotLength(0)` is configured we can also use `setCheckIntervalInfiniteSlot(o2::calibration::TFType updateInterval)` in which case the calibration checks whether the statistics is sufficient only after `updateInterval` TFs. Otherwise it would check after every TF.
 
-`updateAtTheEndOfRunOnly` : to tell the TimeCalibration to finalize the slots and prepare the CCDB entries only at the end of the run.
+The TFs arrive asynchronously at the aggregator node. The `TimeSlotCalibration` keeps a `std::deque` of TimeSlots for which it aggregates the input data simultaneously. Whenever a slot has reached its configured duration the statistics requirement is checked. In case it is not fulfilled, the slot can be extended or merged to the previous slot in order to obtain the required statistics.
 
-Example for the options above:
-`tf-per-slot = 20`
-`max-delay = 3`
-Then if we are processing TF 61 and TF 0 comes, TF 0 will be discarded.
+By default, TFs which arrive more than `3 * SlotLengthInTF` later than the most recent TF processed are discarded. This maximum delay can be configured via `setMaxSlotsDelay(int nSlots)`. If it is set to 4 and each slot has the length of 30 TFs, then upon processing of TF 121 the input from TF0 would be discarded, if it was not already processed.
 
-Each calibration device has to implement the following methods:
 
-`void initOutput()`: initialization of the output object (typically a vector of calibration objects and another one with the associated CcdbObjectInfo);
+In order to prepare only one CCDB object at the end of the run you can use `setUpdateAtTheEndOfRunOnly()`. In this case all the above settings for the slot duration are irrelevant. And upon the `endOfStream` of your calibration device you should make a call to `checkSlotsToFinalize()`.
 
-`bool hasEnoughData(const o2::calibration::TimeSlot& slot)` : method to determine whether a TimeSlot has enough data to be calibrated; if not, it will be merged to the following (in time) one;
 
-`void finalizeSlot(o2::calibration::TimeSlot& slot)` : method to process the calibration data accumulated in each TimeSlot;
+### Mandatory methods to implement when deriving from `o2::calibration::TimeSlotCalibration`
 
-`o2::calibration::TimeSlot& slot emplaceNewSlot(bool front, TFType tstart, TFType tend)` : method to creata a new TimeSlot; this is specific to the calibration procedure as it instantiates the detector-calibration-specific object.
+
+- `void initOutput()`: initialization of the output object (typically a vector of calibration objects and another one with the associated CcdbObjectInfo);
+
+- `bool hasEnoughData(const o2::calibration::TimeSlot& slot)` : method to determine whether a TimeSlot has enough data to be calibrated; if not, it will be merged to the following (in time) one;
+
+- `void finalizeSlot(o2::calibration::TimeSlot& slot)` : method to process the calibration data accumulated in each TimeSlot;
+
+- `o2::calibration::TimeSlot& slot emplaceNewSlot(bool front, TFType tstart, TFType tend)` : method to creata a new TimeSlot; this is specific to the calibration procedure as it instantiates the detector-calibration-specific object.
 
 See e.g. LHCClockCalibrator.h/cxx in AliceO2/Detectors/TOF/calibration/include/TOFCalibration/LHCClockCalibrator.h and  AliceO2/Detectors/TOF/calibration/srcLHCClockCalibrator.cxx
 
+Sometimes it might be useful to define the 1st slot of the run shorter than the nominal slot length, e.g. to not rely to long on the previous run or default calibration. In this case one can impose to the
+calibration class an offset using method `setStartOffsetFrac(float offset)`, where `offset` is a fractional of the nominal slot length to be subtracted from the nominal boundaries of all slots (except the
+very 1st one, whose startTF is set to 0). The fractional offset should be in `[0:0.95)` range, any value outside this range will be overridden to the nearest boundary.
+This feature is supported only for the finite slot-length calibrations. E.g. if the nominal slot length is 10 minutes `(==~210000 TFs)`, setting `setStartOffsetFrac(float 0.7)` will lead to 1st slot
+finalized after the first 3 minutes, while the rest of the slots will be defined with nominal 10 minutes coverage. If statistics of this 1st short slot is insufficient, it will be merged as usual
+with the next slot (note this if this happens, in the example above the 1st calibration will be available in 13 minutes...).
+
 ## TimeSlot
+
 The TimeSlot is a templated class which takes as input type the Container that will hold the calibration data needed to produce the calibration objects (histograms, vectors, array...). Each calibration device could implement its own Container, according to its needs.
 
 The Container class needs to implement the following methods:
 
-`void fill(const gsl::span data)`  : method to decide how to use the calibration data within the container (e.g. fill a vector);
-or
-`void fill(o2::dataformats::TFIDInfo& ti, const gsl::span data)`  : method to decide how to use the calibration data within the container (e.g. fill a vector) and having access to the TFIDInfo struct providing relevant info for current TF (tfCounter, runNumber, creationTime etc.)
-If provided, this latter method will be used.
+- `void merge(const Container* prev)` : method to allow merging of the content of a TimeSlot to the content of the following one, when stastics is limited.
+
+- `void print()` : method to print the content of the Container
+
+- `void fill(DATA data, ...)`  : method to decide how to use the calibration data within the container (e.g. fill a vector). The type of `DATA` is usually `const gsl::span`, but can also be anything else. Optionally the `fill` method can accept additional input of arbitrary type;
 
-`void merge(const Container* prev)` : method to allow merging of the content of a TimeSlot to the content of the following one, when stastics is limited.
+or, alternatively
 
-`void print()` : method to print the content of the Container
+- `void fill(o2::dataformats::TFIDInfo& ti, DATA data, ...)`  : method to decide how to use the calibration data within the container (e.g. fill a vector) and having access to the TFIDInfo struct providing relevant info for current TF (tfCounter, runNumber, creationTime etc.).
+If provided, this latter method will be used.
 
 See e.g. LHCClockCalibrator.h/cxx in AliceO2/Detectors/TOF/calibration/include/TOFCalibration/LHCClockCalibrator.h and  AliceO2/Detectors/TOF/calibration/srcLHCClockCalibrator.cxx
 
@@ -57,7 +72,7 @@ The Slot provides a generic methods to access its boundaries: `getTFStart()` and
 
 ## detector-specific-calibrator-workflow
 
-Each calibration will need to be implemented in the form of a workflow, whose options should include those for the calibration device itself (`tf-per-slot` and `max-delay`, see above).
+Each calibration will need to be implemented in the form of a workflow, whose options should include those for the calibration device itself (for example the slot length and statistics requirement).
 The output to be sent by the calibrator should include:
 
 *   a vector of the snapshots of the object to be put in the CCDB;
@@ -76,6 +91,50 @@ Note that in order to access the absolute time of the slot boundaries, one shoul
 
 See e.g. AliceO2/Detectors/TOF/calibration/testWorkflow/LHCClockCalibratorSpec.h,  AliceO2/Detectors/TOF/calibration/testWorkflow/lhc-clockphase-workflow.cxx
 
+## Integration of calibration workflows into the global framework
+
+For the synchronous processing on the EPN the calibration workflows are grouped according to their origin (BARREL, CALO, MUON and FORWARD) and to the nature of their input (TF for devices expecting input for every TF and SPORADIC for devices expecting input sporadically).
+For each group (e.g. `BARREL_TF`) a pair of `o2-dpl-output-proxy` running on the processing EPNs and `o2-dpl-raw-proxy` running on the calibration nodes is initialized and these proxys are used to transfer the input from processing nodes to the calibration node.
+Have a look at the `DATA/common/setenv_calib.sh` script in O2DPG where for each calibration the required data descriptors are added to the proxies.
+In addition there is always some logic to decide whether a specific calibration should be enabled or not.
+
+The workflow which is running on the processing nodes should be added in the `prodtests/full-system-test/calib-workflow.sh` script.
+The workflow running on the aggregator should be added to `prodtests/full-system-test/aggregator-workflow.sh`.
+
+## Calibrating over multiple runs
+
+Some statistics-hungry calibrations define single time-slot which integrates data of the whole run. If there is a possibility that for the short run the slot will not accumulate enough statistics,
+one can save the user-defined content of the slot to a file in the dedicated partition on the calibrator node
+and adopt data from this file in the next run. In order to do that the calibrator class derived from the TimeSlotCalibration must:
+
+* set fixed file name to write via `setSaveFileName(const std::string& n)` method. Also, the corresponding workflow should provide/parse an option to set the output directory.
+
+* implement virtual method `bool saveLastSlotData(TFile& fl)` which writes content of the (last and only) slot into the provided file handler. It is up to detector to define the format of the stored data. The framework will write to the same file a
+TimeSlotMetaData struct describing the start/end timestamps and start/end runs for the data written.
+
+* implement virtual method `bool adoptSavedData(const TimeSlotMetaData& metadata, TFile& fl)` which reads and adds saved data to the slot in the new run. Provided metadata should be used to judge if the saved data are useful.
+
+* decide e.g. in the finalizeSlot method if the slot content must be saved to be accounted in the following run and call `saveLastSlot()` in that case.
+
+* in the beginning of the processing, e.g. after the 1st call of the `process(..)` method (where the time-slot will be created) call `loadSavedSlot()` method, i.e.
+  ```
+  auto data = pc.inputs().get<...>; // get input data
+  o2::base::TFIDInfoHelper::fillTFIDInfo(pc, mCalibrator->getCurrentTFInfo());
+  mCalibrator->process(data);
+  static bool firstCall = true;
+  if (firstCall && getNSlots()) {
+    firstCall = false;
+    loadSavedSlot();
+  }
+  ```
+  Make sure the static method `o2::base::TFIDInfoHelper::fillTFIDInfo(pc, mCalibrator->getCurrentTFInfo());` was called from the `run()` method before the `process(...)` call.
+
+The slot saving and loading will be done only if `setSavedSlotAllowed(true)` was called explicitly from the calibrator device before the processing starts (e.g. in the `init()` method).
+Since one can have multiple instances of the calibrator device
+running at the same time (in staging and produnction partitions, synthetic and real runs) it is important to make sure that this method is called only for the physics run calibration.
+
+In order to not pollute calibration node disk, the file will be removed in the end of `loadSavedSlot()` call.
+
 ## ccdb-populator-workflow
 
 This is the workflow that, connected to all workflows producting calibrations with different granularities and frequencies, will update the CCDB.
@@ -102,7 +161,12 @@ o2-calibration-ccdb-populator-workflow --sspec-min 0 --sspec-max 1  -b
 then the `ObjA` will be uploaded only to the default server (`http://alice-ccdb.cern.ch`), `ObjB` will be uploaded to both default and `local` server and
 `ObjC` will be uploaded to the `local` server only.
 
-By default the ccdb-populator-workflow will not produce `fatal` on failed upload. To require it an option `--fatal-on-failure` can be used.
+By default the `ccdb-populator-workflow` will not produce `fatal` on failed upload. To require it an option `--fatal-on-failure` can be used.
+
+By default the `ccdb-populator-workflow` uploads objects as it gets them. In case there is a danger that objects of the same URL will arrive in the order not sorted in SOV
+(which may lead to screaning of the object with later SOV by other object if earlier ROF) one can use an option `--ordering-latency ` of the `ccdb-populator-workflow`.
+Then every incoming object will be buffered and uploaded only if no object with the same CCDB path and earlier start of validity was received in preceding N milliseconds. All remaining cached objects are uploaded at EOR (or stop() method call).
+
 
diff --git a/Detectors/Calibration/include/DetectorsCalibration/IntegratedClusterCalibrator.h b/Detectors/Calibration/include/DetectorsCalibration/IntegratedClusterCalibrator.h
new file mode 100644
index 0000000000000..9720142d391b1
--- /dev/null
+++ b/Detectors/Calibration/include/DetectorsCalibration/IntegratedClusterCalibrator.h
@@ -0,0 +1,816 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file IntegratedClusterCalibrator.h
+/// \brief calibrator class for accumulating integrated clusters
+/// \author Matthias Kleiner 
+/// \date Jan 21, 2023
+
+#ifndef INTEGRATEDCLUSTERCALIBRATOR_H_
+#define INTEGRATEDCLUSTERCALIBRATOR_H_
+
+#include "DetectorsCalibration/TimeSlotCalibration.h"
+#include "DetectorsCalibration/TimeSlot.h"
+
+class TTree;
+
+namespace o2
+{
+
+// see https://stackoverflow.com/questions/56483053/if-condition-checking-multiple-vector-sizes-are-equal
+// comparing if all vector have the same size
+template 
+bool sameSize(T0 const& first, Ts const&... rest)
+{
+  return ((first.size() == rest.size()) && ...);
+}
+
+namespace tof
+{
+
+/// struct containing the integrated TOF currents
+struct ITOFC {
+  std::vector mITOFCNCl;                                     ///< integrated 1D TOF cluster currents
+  std::vector mITOFCQ;                                       ///< integrated 1D TOF qTot currents
+  long mTimeMS{};                                                   ///< start time in ms
+  bool areSameSize() const { return sameSize(mITOFCNCl, mITOFCQ); } ///< check if stored currents have same number of entries
+  bool isEmpty() const { return mITOFCNCl.empty(); }                ///< check if values are empty
+  size_t getEntries() const { return mITOFCNCl.size(); }            ///< \return returns number of values stored
+  void setStartTime(long timeMS) { mTimeMS = timeMS; }
+
+  /// acummulate integrated currents at given index
+  /// \param posIndex index where data will be copied to
+  /// \param data integrated currents which will be copied
+  void fill(const unsigned int posIndex, const ITOFC& data)
+  {
+    std::copy(data.mITOFCNCl.begin(), data.mITOFCNCl.end(), mITOFCNCl.begin() + posIndex);
+    std::copy(data.mITOFCQ.begin(), data.mITOFCQ.end(), mITOFCQ.begin() + posIndex);
+  }
+
+  /// \param nDummyValues number of empty values which are inserted at the beginning of the accumulated integrated currents
+  void insert(const unsigned int nDummyValues)
+  {
+    std::vector vecTmp(nDummyValues, 0);
+    mITOFCNCl.insert(mITOFCNCl.begin(), vecTmp.begin(), vecTmp.end());
+    mITOFCQ.insert(mITOFCQ.begin(), vecTmp.begin(), vecTmp.end());
+  }
+
+  /// resize buffer for accumulated currents
+  void resize(const unsigned int nTotal)
+  {
+    mITOFCNCl.resize(nTotal);
+    mITOFCQ.resize(nTotal);
+  }
+
+  ClassDefNV(ITOFC, 2);
+};
+} // end namespace tof
+
+namespace tpc
+{
+
+/// struct containing the integrated TPC currents
+struct ITPCC {
+  std::vector mIQMaxA; ///< integrated 1D-currents for QMax A-side
+  std::vector mIQMaxC; ///< integrated 1D-currents for QMax C-side
+  std::vector mIQTotA; ///< integrated 1D-currents for QTot A-side
+  std::vector mIQTotC; ///< integrated 1D-currents for QTot A-side
+  std::vector mINClA;  ///< integrated 1D-currents for NCl A-side
+  std::vector mINClC;  ///< integrated 1D-currents for NCl A-side
+  long mTimeMS{};             ///< start time in ms
+
+  float compression(float value, const int nBits) const
+  {
+    const int shiftN = std::pow(2, nBits);
+    int exp2;
+    const auto mantissa = std::frexp(value, &exp2);
+    const auto mantissaRounded = std::round(mantissa * shiftN) / shiftN;
+    return std::ldexp(mantissaRounded, exp2);
+  }
+
+  void compress(const int nBits)
+  {
+    std::transform(mIQMaxA.begin(), mIQMaxA.end(), mIQMaxA.begin(), [this, nBits](float val) { return compression(val, nBits); });
+    std::transform(mIQMaxC.begin(), mIQMaxC.end(), mIQMaxC.begin(), [this, nBits](float val) { return compression(val, nBits); });
+    std::transform(mIQTotA.begin(), mIQTotA.end(), mIQTotA.begin(), [this, nBits](float val) { return compression(val, nBits); });
+    std::transform(mIQTotC.begin(), mIQTotC.end(), mIQTotC.begin(), [this, nBits](float val) { return compression(val, nBits); });
+    std::transform(mINClA.begin(), mINClA.end(), mINClA.begin(), [this, nBits](float val) { return compression(val, nBits); });
+    std::transform(mINClC.begin(), mINClC.end(), mINClC.begin(), [this, nBits](float val) { return compression(val, nBits); });
+  }
+
+  bool areSameSize() const { return sameSize(mIQMaxA, mIQMaxC, mIQTotA, mIQTotC, mINClA, mINClC); } ///< check if stored currents have same number of entries
+  bool isEmpty() const { return mIQMaxA.empty(); }                                                  ///< check if values are empty
+  size_t getEntries() const { return mIQMaxA.size(); }                                              ///< \return returns number of values stored
+  void setStartTime(long timeMS) { mTimeMS = timeMS; }
+
+  /// acummulate integrated currents at given index
+  /// \param posIndex index where data will be copied to
+  /// \param data integrated currents which will be copied
+  void fill(const unsigned int posIndex, const ITPCC& data)
+  {
+    std::copy(data.mIQMaxA.begin(), data.mIQMaxA.end(), mIQMaxA.begin() + posIndex);
+    std::copy(data.mIQMaxC.begin(), data.mIQMaxC.end(), mIQMaxC.begin() + posIndex);
+    std::copy(data.mIQTotA.begin(), data.mIQTotA.end(), mIQTotA.begin() + posIndex);
+    std::copy(data.mIQTotC.begin(), data.mIQTotC.end(), mIQTotC.begin() + posIndex);
+    std::copy(data.mINClA.begin(), data.mINClA.end(), mINClA.begin() + posIndex);
+    std::copy(data.mINClC.begin(), data.mINClC.end(), mINClC.begin() + posIndex);
+  }
+
+  /// \param nDummyValues number of empty values which are inserted at the beginning of the accumulated integrated currents
+  void insert(const unsigned int nDummyValues)
+  {
+    std::vector vecTmp(nDummyValues, 0);
+    mIQMaxA.insert(mIQMaxA.begin(), vecTmp.begin(), vecTmp.end());
+    mIQMaxC.insert(mIQMaxC.begin(), vecTmp.begin(), vecTmp.end());
+    mIQTotA.insert(mIQTotA.begin(), vecTmp.begin(), vecTmp.end());
+    mIQTotC.insert(mIQTotC.begin(), vecTmp.begin(), vecTmp.end());
+    mINClA.insert(mINClA.begin(), vecTmp.begin(), vecTmp.end());
+    mINClC.insert(mINClC.begin(), vecTmp.begin(), vecTmp.end());
+  }
+
+  /// resize buffer for accumulated currents
+  void resize(const unsigned int nTotal)
+  {
+    mIQMaxA.resize(nTotal);
+    mIQMaxC.resize(nTotal);
+    mIQTotA.resize(nTotal);
+    mIQTotC.resize(nTotal);
+    mINClA.resize(nTotal);
+    mINClC.resize(nTotal);
+  }
+
+  /// reset buffered currents
+  void reset()
+  {
+    std::fill(mIQMaxA.begin(), mIQMaxA.end(), 0);
+    std::fill(mIQMaxC.begin(), mIQMaxC.end(), 0);
+    std::fill(mIQTotA.begin(), mIQTotA.end(), 0);
+    std::fill(mIQTotC.begin(), mIQTotC.end(), 0);
+    std::fill(mINClA.begin(), mINClA.end(), 0);
+    std::fill(mINClC.begin(), mINClC.end(), 0);
+  }
+
+  /// normalize currents
+  void normalize(const float factor)
+  {
+    std::transform(mIQMaxA.begin(), mIQMaxA.end(), mIQMaxA.begin(), [factor](const float val) { return val * factor; });
+    std::transform(mIQMaxC.begin(), mIQMaxC.end(), mIQMaxC.begin(), [factor](const float val) { return val * factor; });
+    std::transform(mIQTotA.begin(), mIQTotA.end(), mIQTotA.begin(), [factor](const float val) { return val * factor; });
+    std::transform(mIQTotC.begin(), mIQTotC.end(), mIQTotC.begin(), [factor](const float val) { return val * factor; });
+    std::transform(mINClA.begin(), mINClA.end(), mINClA.begin(), [factor](const float val) { return val * factor; });
+    std::transform(mINClC.begin(), mINClC.end(), mINClC.begin(), [factor](const float val) { return val * factor; });
+  }
+
+  ClassDefNV(ITPCC, 2);
+};
+
+/// struct containing time series values
+struct TimeSeries {
+  std::vector mDCAr_A_Median;       ///< integrated 1D DCAr for A-side median in phi/tgl slices
+  std::vector mDCAr_C_Median;       ///< integrated 1D DCAr for C-side weighted mean in phi/tgl slices
+  std::vector mDCAr_A_WeightedMean; ///< integrated 1D DCAr for A-side weighted mean in phi/tgl slices
+  std::vector mDCAr_C_WeightedMean; ///< integrated 1D DCAr for C-side median in phi/tgl slices
+  std::vector mDCAr_A_RMS;          ///< integrated 1D DCAr for A-side RMS in phi/tgl slices
+  std::vector mDCAr_C_RMS;          ///< integrated 1D DCAr for C-side RMS in phi/tgl slices
+  std::vector mDCAr_A_NTracks;      ///< number of tracks used to calculate the DCAs
+  std::vector mDCAr_C_NTracks;      ///< number of tracks used to calculate the DCAs
+  std::vector mDCAz_A_Median;       ///< integrated 1D DCAz for A-side median in phi/tgl slices
+  std::vector mDCAz_C_Median;       ///< integrated 1D DCAz for C-side median in phi/tgl slices
+  std::vector mDCAz_A_WeightedMean; ///< integrated 1D DCAz for A-side weighted mean in phi/tgl slices
+  std::vector mDCAz_C_WeightedMean; ///< integrated 1D DCAz for C-side weighted mean in phi/tgl slices
+  std::vector mDCAz_A_RMS;          ///< integrated 1D DCAz for A-side RMS in phi/tgl slices
+  std::vector mDCAz_C_RMS;          ///< integrated 1D DCAz for C-side RMS in phi/tgl slices
+  std::vector mDCAz_A_NTracks;      ///< number of tracks used to calculate the DCAs
+  std::vector mDCAz_C_NTracks;      ///< number of tracks used to calculate the DCAs
+  std::vector mMIPdEdxRatioQMaxA;   ///< ratio of MIP/dEdx - qMax -
+  std::vector mMIPdEdxRatioQMaxC;   ///< ratio of MIP/dEdx - qMax -
+  std::vector mMIPdEdxRatioQTotA;   ///< ratio of MIP/dEdx - qTot -
+  std::vector mMIPdEdxRatioQTotC;   ///< ratio of MIP/dEdx - qTot -
+  std::vector mTPCChi2A;            ///< Chi2 of TPC tracks
+  std::vector mTPCChi2C;            ///< Chi2 of TPC tracks
+  std::vector mTPCNClA;             ///< number of TPC cluster
+  std::vector mTPCNClC;             ///< number of TPC cluster
+
+  unsigned char mNBinsPhi{}; ///< number of tgl bins
+  unsigned char mNBinsTgl{}; ///< number of phi bins
+  float mTglMax{};           ///< absolute max tgl
+  unsigned char mNBinsqPt{}; ///< number of qPt bins
+  float mQPtMax{};           ///< abs qPt max
+  unsigned char mMultBins{}; ///< multiplicity bins
+  float mMultMax{};          ///< max local multiplicity
+  long mTimeMS{};            ///< start time in ms
+
+  /// dump object to tree
+  /// \param outFileName name of the output file
+  /// \param nHBFPerTF number of orbits per TF
+  void setStartTime(long timeMS) { mTimeMS = timeMS; }
+
+  /// \return returns total number of bins
+  int getNBins() const { return mNBinsTgl + mNBinsPhi + mNBinsqPt + mMultBins + 1; }
+
+  /// \return returns index for given phi bin
+  int getIndexPhi(const int iPhi, int slice = 0) const { return iPhi + slice * getNBins(); }
+
+  /// \return returns index for given tgl bin
+  int getIndexTgl(const int iTgl, int slice = 0) const { return mNBinsPhi + iTgl + slice * getNBins(); }
+
+  /// \return returns index for given qPt bin
+  int getIndexqPt(const int iqPt, int slice = 0) const { return mNBinsPhi + mNBinsTgl + iqPt + slice * getNBins(); }
+
+  /// \return returns index for given qPt bin
+  int getIndexMult(const int iMult, int slice = 0) const { return mNBinsPhi + mNBinsTgl + mNBinsqPt + iMult + slice * getNBins(); }
+
+  /// \return returns index for integrated over all bins
+  int getIndexInt(int slice = 0) const { return getNBins() - 1 + slice * getNBins(); }
+
+  /// resize buffer for accumulated currents
+  void resize(const unsigned int nTotal)
+  {
+    mDCAr_A_Median.resize(nTotal);
+    mDCAr_C_Median.resize(nTotal);
+    mDCAr_A_RMS.resize(nTotal);
+    mDCAr_C_RMS.resize(nTotal);
+    mDCAz_A_Median.resize(nTotal);
+    mDCAz_C_Median.resize(nTotal);
+    mDCAz_A_RMS.resize(nTotal);
+    mDCAz_C_RMS.resize(nTotal);
+    mDCAr_C_NTracks.resize(nTotal);
+    mDCAr_A_NTracks.resize(nTotal);
+    mDCAz_A_NTracks.resize(nTotal);
+    mDCAz_C_NTracks.resize(nTotal);
+    mDCAr_A_WeightedMean.resize(nTotal);
+    mDCAr_C_WeightedMean.resize(nTotal);
+    mDCAz_A_WeightedMean.resize(nTotal);
+    mDCAz_C_WeightedMean.resize(nTotal);
+    mMIPdEdxRatioQMaxA.resize(nTotal);
+    mMIPdEdxRatioQMaxC.resize(nTotal);
+    mMIPdEdxRatioQTotA.resize(nTotal);
+    mMIPdEdxRatioQTotC.resize(nTotal);
+    mTPCChi2A.resize(nTotal);
+    mTPCChi2C.resize(nTotal);
+    mTPCNClA.resize(nTotal);
+    mTPCNClC.resize(nTotal);
+  }
+
+  ClassDefNV(TimeSeries, 1);
+};
+
+struct ITSTPC_Matching {
+  std::vector mITSTPC_A_MatchEff;  ///< matching efficiency of ITS-TPC tracks A-side
+  std::vector mITSTPC_C_MatchEff;  ///< matching efficiency of ITS-TPC tracks C-side
+  std::vector mITSTPC_A_Chi2Match; ///< ITS-TPC chi2 A-side
+  std::vector mITSTPC_C_Chi2Match; ///< ITS-TPC chi2 C-side
+
+  /// resize buffer for accumulated currents
+  void resize(const unsigned int nTotal)
+  {
+    mITSTPC_A_MatchEff.resize(nTotal);
+    mITSTPC_C_MatchEff.resize(nTotal);
+    mITSTPC_A_Chi2Match.resize(nTotal);
+    mITSTPC_C_Chi2Match.resize(nTotal);
+  }
+
+  ClassDefNV(ITSTPC_Matching, 2);
+};
+
+struct TimeSeriesdEdx {
+  std::vector mLogdEdx_A_Median;       ///< log(dEdx_exp(pion)/dEdx) - A-side
+  std::vector mLogdEdx_A_RMS;          ///< log(dEdx_exp(pion)/dEdx) - A-side
+  std::vector mLogdEdx_A_IROC_Median;  ///< log(dedxIROC / dEdx) - A-side
+  std::vector mLogdEdx_A_IROC_RMS;     ///< log(dedxIROC / dEdx) - A-side
+  std::vector mLogdEdx_A_OROC1_Median; ///< log(dedxOROC1 / dEdx) - A-side
+  std::vector mLogdEdx_A_OROC1_RMS;    ///< log(dedxOROC1 / dEdx) - A-side
+  std::vector mLogdEdx_A_OROC2_Median; ///< log(dedxOROC2 / dEdx) - A-side
+  std::vector mLogdEdx_A_OROC2_RMS;    ///< log(dedxOROC2 / dEdx) - A-side
+  std::vector mLogdEdx_A_OROC3_Median; ///< log(dedxOROC3 / dEdx) - A-side
+  std::vector mLogdEdx_A_OROC3_RMS;    ///< log(dedxOROC3 / dEdx) - A-side
+  std::vector mLogdEdx_C_Median;       ///< log(dEdx_exp(pion)/dEdx) - C-side
+  std::vector mLogdEdx_C_RMS;          ///< log(dEdx_exp(pion)/dEdx) - C-side
+  std::vector mLogdEdx_C_IROC_Median;  ///< log(dedxIROC / dEdx) - C-side
+  std::vector mLogdEdx_C_IROC_RMS;     ///< log(dedxIROC / dEdx) - C-side
+  std::vector mLogdEdx_C_OROC1_Median; ///< log(dedxOROC1 / dEdx) - C-side
+  std::vector mLogdEdx_C_OROC1_RMS;    ///< log(dedxOROC1 / dEdx) - C-side
+  std::vector mLogdEdx_C_OROC2_Median; ///< log(dedxOROC2 / dEdx) - C-side
+  std::vector mLogdEdx_C_OROC2_RMS;    ///< log(dedxOROC2 / dEdx) - C-side
+  std::vector mLogdEdx_C_OROC3_Median; ///< log(dedxOROC3 / dEdx) - C-side
+  std::vector mLogdEdx_C_OROC3_RMS;    ///< log(dedxOROC3 / dEdx) - C-side
+
+  void resize(const unsigned int nTotal)
+  {
+    mLogdEdx_A_Median.resize(nTotal);
+    mLogdEdx_A_RMS.resize(nTotal);
+    mLogdEdx_A_IROC_Median.resize(nTotal);
+    mLogdEdx_A_IROC_RMS.resize(nTotal);
+    mLogdEdx_A_OROC1_Median.resize(nTotal);
+    mLogdEdx_A_OROC1_RMS.resize(nTotal);
+    mLogdEdx_A_OROC2_Median.resize(nTotal);
+    mLogdEdx_A_OROC2_RMS.resize(nTotal);
+    mLogdEdx_A_OROC3_Median.resize(nTotal);
+    mLogdEdx_A_OROC3_RMS.resize(nTotal);
+    mLogdEdx_C_Median.resize(nTotal);
+    mLogdEdx_C_RMS.resize(nTotal);
+    mLogdEdx_C_IROC_Median.resize(nTotal);
+    mLogdEdx_C_IROC_RMS.resize(nTotal);
+    mLogdEdx_C_OROC1_Median.resize(nTotal);
+    mLogdEdx_C_OROC1_RMS.resize(nTotal);
+    mLogdEdx_C_OROC2_Median.resize(nTotal);
+    mLogdEdx_C_OROC2_RMS.resize(nTotal);
+    mLogdEdx_C_OROC3_Median.resize(nTotal);
+    mLogdEdx_C_OROC3_RMS.resize(nTotal);
+  }
+
+  ClassDefNV(TimeSeriesdEdx, 1);
+};
+
+struct TimeSeriesITSTPC {
+  float mVDrift = 0;                          ///< drift velocity in cm/us
+  float mPressure = 0;                        ///< pressure
+  float mTemperature = 0;                     ///< temperature
+  TimeSeries mTSTPC;                          ///< TPC standalone DCAs
+  TimeSeries mTSITSTPC;                       ///< ITS-TPC standalone DCAs
+  ITSTPC_Matching mITSTPCAll;                 ///< ITS-TPC matching efficiency for ITS standalone + afterburner
+  ITSTPC_Matching mITSTPCStandalone;          ///< ITS-TPC matching efficiency for ITS standalone
+  ITSTPC_Matching mITSTPCAfterburner;         ///< ITS-TPC matchin efficiency  fir ITS afterburner
+  TimeSeriesdEdx mdEdxQTot;                   ///< time series for dE/dx qTot monitoring
+  TimeSeriesdEdx mdEdxQMax;                   ///< time series for dE/dx qMax monitoring
+  std::vector mOccupancyMapTPC; ///< cluster occupancy map
+
+  std::vector nPrimVertices;                  ///< number of primary vertices
+  std::vector nPrimVertices_ITS;              ///< number of primary vertices selected with ITS cut 0.2 nVertexContributors_ITS_Median; ///< number of primary vertices selected with ITS cut 0.2 nVertexContributors_ITS_RMS;    ///< number of primary vertices selected with ITS cut 0.2 vertexX_ITS_Median;             ///< vertex x position selected with ITS cut 0.2 vertexY_ITS_Median;             ///< vertex y position selected with ITS cut 0.2 vertexZ_ITS_Median;             ///< vertex z position selected with ITS cut 0.2 vertexX_ITS_RMS;                ///< vertex x RMS selected with ITS cut 0.2 vertexY_ITS_RMS;                ///< vertex y RMS selected with ITS cut 0.2 vertexZ_ITS_RMS;                ///< vertex z RMS selected with ITS cut 0.2 nPrimVertices_ITSTPC;              ///< number of primary vertices with ITS-TPC cut (nContributorsITS + nContributorsITSTPC)<0.95
+  std::vector nVertexContributors_ITSTPC_Median; ///< number of primary vertices with ITS-TPC cut (nContributorsITS + nContributorsITSTPC)<0.95
+  std::vector nVertexContributors_ITSTPC_RMS;    ///< number of primary vertices with ITS-TPC cut (nContributorsITS + nContributorsITSTPC)<0.95
+  std::vector vertexX_ITSTPC_Median;             ///< vertex x position with ITS-TPC cut (nContributorsITS + nContributorsITSTPC)<0.95
+  std::vector vertexY_ITSTPC_Median;             ///< vertex y position with ITS-TPC cut (nContributorsITS + nContributorsITSTPC)<0.95
+  std::vector vertexZ_ITSTPC_Median;             ///< vertex z position with ITS-TPC cut (nContributorsITS + nContributorsITSTPC)<0.95
+  std::vector vertexX_ITSTPC_RMS;                ///< vertex x RMS with ITS-TPC cut (nContributorsITS + nContributorsITSTPC)<0.95
+  std::vector vertexY_ITSTPC_RMS;                ///< vertex y RMS with ITS-TPC cut (nContributorsITS + nContributorsITSTPC)<0.95
+  std::vector vertexZ_ITSTPC_RMS;                ///< vertex z RMS with ITS-TPC cut (nContributorsITS + nContributorsITSTPC)<0.95
+
+  int quantileValues = 23;                          /// nVertexContributors_Quantiles; ///< number of primary vertices for quantiles 0.1, 0.2, ... 0.9 and truncated mean values 0.05->0.95, 0.1->0.9, 0.2->0.8
+
+  std::vector mDCAr_comb_A_Median;       ///< DCAr for ITS-TPC track - A-side
+  std::vector mDCAz_comb_A_Median;       ///< DCAz for ITS-TPC track - A-side
+  std::vector mDCAr_comb_A_RMS;          ///< DCAr RMS for ITS-TPC track - A-side
+  std::vector mDCAz_comb_A_RMS;          ///< DCAz RMS for ITS-TPC track - A-side
+  std::vector mDCAr_comb_C_Median;       ///< DCAr for ITS-TPC track - C-side
+  std::vector mDCAz_comb_C_Median;       ///< DCAz for ITS-TPC track - C-side
+  std::vector mDCAr_comb_C_RMS;          ///< DCAr RMS for ITS-TPC track - C-side
+  std::vector mDCAz_comb_C_RMS;          ///< DCAz RMS for ITS-TPC track - C-side
+  std::vector mITS_A_NCl_Median;         ///< its number of clusters
+  std::vector mITS_A_NCl_RMS;            ///< its number of clusters
+  std::vector mITS_C_NCl_Median;         ///< its number of clusters
+  std::vector mITS_C_NCl_RMS;            ///< its number of clusters
+  std::vector mSqrtITSChi2_Ncl_A_Median; ///< sqrt(ITC chi2 / ncl)
+  std::vector mSqrtITSChi2_Ncl_C_Median; ///< sqrt(ITC chi2 / ncl)
+  std::vector mSqrtITSChi2_Ncl_A_RMS;    ///< sqrt(ITC chi2 / ncl)
+  std::vector mSqrtITSChi2_Ncl_C_RMS;    ///< sqrt(ITC chi2 / ncl)
+
+  std::vector mITSTPCDeltaP2_A_Median; ///< track param TPC - track param ITS-TPC for param 2 - A-side
+  std::vector mITSTPCDeltaP3_A_Median; ///< track param TPC - track param ITS-TPC for param 3 - A-side
+  std::vector mITSTPCDeltaP4_A_Median; ///< track param TPC - track param ITS-TPC for param 4 - A-side
+  std::vector mITSTPCDeltaP2_C_Median; ///< track param TPC - track param ITS-TPC for param 2 - A-side
+  std::vector mITSTPCDeltaP3_C_Median; ///< track param TPC - track param ITS-TPC for param 3 - A-side
+  std::vector mITSTPCDeltaP4_C_Median; ///< track param TPC - track param ITS-TPC for param 4 - A-side
+  std::vector mITSTPCDeltaP2_A_RMS;    ///< RMS of track param TPC - track param ITS-TPC for param 2 - A-side
+  std::vector mITSTPCDeltaP3_A_RMS;    ///< RMS of track param TPC - track param ITS-TPC for param 3 - A-side
+  std::vector mITSTPCDeltaP4_A_RMS;    ///< RMS of track param TPC - track param ITS-TPC for param 4 - A-side
+  std::vector mITSTPCDeltaP2_C_RMS;    ///< RMS of track param TPC - track param ITS-TPC for param 2 - A-side
+  std::vector mITSTPCDeltaP3_C_RMS;    ///< RMS of track param TPC - track param ITS-TPC for param 3 - A-side
+  std::vector mITSTPCDeltaP4_C_RMS;    ///< RMS of track param TPC - track param ITS-TPC for param 4 - A-side
+
+  std::vector mTPCSigmaY2A_Median; ///< sigmaY2 at vertex
+  std::vector mTPCSigmaZ2A_Median; ///< sigmaZ2 at vertex
+  std::vector mTPCSigmaY2C_Median; ///< sigmaY2 at vertex
+  std::vector mTPCSigmaZ2C_Median; ///< sigmaZ2 at vertex
+  std::vector mTPCSigmaY2A_RMS;    ///< sigmaY2 RMS at vertex
+  std::vector mTPCSigmaZ2A_RMS;    ///< sigmaZ2 RMS at vertex
+  std::vector mTPCSigmaY2C_RMS;    ///< sigmaY2 RMS at vertex
+  std::vector mTPCSigmaZ2C_RMS;    ///< sigmaZ2 RMS at vertex
+
+  void setStartTime(long timeMS)
+  {
+    mTSTPC.setStartTime(timeMS);
+    mTSITSTPC.setStartTime(timeMS);
+  }
+
+  void setBinning(const int nBinsPhi, const int nBinsTgl, const int qPtBins, const int nBinsMult, float tglMax, float qPtMax, float multMax)
+  {
+    mTSTPC.mNBinsPhi = nBinsPhi;
+    mTSTPC.mNBinsTgl = nBinsTgl;
+    mTSTPC.mNBinsqPt = qPtBins;
+    mTSTPC.mMultBins = nBinsMult;
+    mTSTPC.mTglMax = tglMax;
+    mTSTPC.mQPtMax = qPtMax;
+    mTSTPC.mMultMax = multMax;
+    mTSITSTPC.mNBinsPhi = nBinsPhi;
+    mTSITSTPC.mNBinsTgl = nBinsTgl;
+    mTSITSTPC.mNBinsqPt = qPtBins;
+    mTSITSTPC.mMultBins = nBinsMult;
+    mTSITSTPC.mTglMax = tglMax;
+    mTSITSTPC.mQPtMax = qPtMax;
+    mTSITSTPC.mMultMax = multMax;
+  }
+
+  /// resize buffer for accumulated currents
+  void resize(const unsigned int nTotal)
+  {
+    mTSTPC.resize(nTotal);
+    mTSITSTPC.resize(nTotal);
+    mITSTPCAll.resize(nTotal);
+    mITSTPCStandalone.resize(nTotal);
+    mITSTPCAfterburner.resize(nTotal);
+    mdEdxQTot.resize(nTotal);
+    mdEdxQMax.resize(nTotal);
+    mDCAr_comb_A_Median.resize(nTotal);
+    mDCAz_comb_A_Median.resize(nTotal);
+    mDCAr_comb_A_RMS.resize(nTotal);
+    mDCAz_comb_A_RMS.resize(nTotal);
+    mDCAr_comb_C_Median.resize(nTotal);
+    mDCAz_comb_C_Median.resize(nTotal);
+    mDCAr_comb_C_RMS.resize(nTotal);
+    mDCAz_comb_C_RMS.resize(nTotal);
+    mITS_A_NCl_Median.resize(nTotal);
+    mITS_A_NCl_RMS.resize(nTotal);
+    mITS_C_NCl_Median.resize(nTotal);
+    mITS_C_NCl_RMS.resize(nTotal);
+    mSqrtITSChi2_Ncl_A_Median.resize(nTotal);
+    mSqrtITSChi2_Ncl_C_Median.resize(nTotal);
+    mSqrtITSChi2_Ncl_A_RMS.resize(nTotal);
+    mSqrtITSChi2_Ncl_C_RMS.resize(nTotal);
+    mITSTPCDeltaP2_A_Median.resize(nTotal);
+    mITSTPCDeltaP3_A_Median.resize(nTotal);
+    mITSTPCDeltaP4_A_Median.resize(nTotal);
+    mITSTPCDeltaP2_C_Median.resize(nTotal);
+    mITSTPCDeltaP3_C_Median.resize(nTotal);
+    mITSTPCDeltaP4_C_Median.resize(nTotal);
+    mITSTPCDeltaP2_A_RMS.resize(nTotal);
+    mITSTPCDeltaP3_A_RMS.resize(nTotal);
+    mITSTPCDeltaP4_A_RMS.resize(nTotal);
+    mITSTPCDeltaP2_C_RMS.resize(nTotal);
+    mITSTPCDeltaP3_C_RMS.resize(nTotal);
+    mITSTPCDeltaP4_C_RMS.resize(nTotal);
+    mTPCSigmaY2A_Median.resize(nTotal);
+    mTPCSigmaZ2A_Median.resize(nTotal);
+    mTPCSigmaY2C_Median.resize(nTotal);
+    mTPCSigmaZ2C_Median.resize(nTotal);
+    mTPCSigmaY2A_RMS.resize(nTotal);
+    mTPCSigmaZ2A_RMS.resize(nTotal);
+    mTPCSigmaY2C_RMS.resize(nTotal);
+    mTPCSigmaZ2C_RMS.resize(nTotal);
+
+    const int nTotalVtx = nTotal / mTSTPC.getNBins();
+    nPrimVertices.resize(nTotalVtx);
+    nPrimVertices_ITS.resize(nTotalVtx);
+    nVertexContributors_ITS_Median.resize(nTotalVtx);
+    nVertexContributors_ITS_RMS.resize(nTotalVtx);
+    vertexX_ITS_Median.resize(nTotalVtx);
+    vertexY_ITS_Median.resize(nTotalVtx);
+    vertexZ_ITS_Median.resize(nTotalVtx);
+    vertexX_ITS_RMS.resize(nTotalVtx);
+    vertexY_ITS_RMS.resize(nTotalVtx);
+    vertexZ_ITS_RMS.resize(nTotalVtx);
+    nPrimVertices_ITSTPC.resize(nTotalVtx);
+    nVertexContributors_ITSTPC_Median.resize(nTotalVtx);
+    nVertexContributors_ITSTPC_RMS.resize(nTotalVtx);
+    vertexX_ITSTPC_Median.resize(nTotalVtx);
+    vertexY_ITSTPC_Median.resize(nTotalVtx);
+    vertexZ_ITSTPC_Median.resize(nTotalVtx);
+    vertexX_ITSTPC_RMS.resize(nTotalVtx);
+    vertexY_ITSTPC_RMS.resize(nTotalVtx);
+    vertexZ_ITSTPC_RMS.resize(nTotalVtx);
+
+    const int nTotalQ = quantileValues * nTotal / mTSTPC.getNBins();
+    nVertexContributors_Quantiles.resize(nTotalQ);
+  }
+
+  ClassDefNV(TimeSeriesITSTPC, 6);
+};
+
+} // end namespace tpc
+
+namespace fit
+{
+
+/// struct containing the integrated FT0 currents
+struct IFT0C {
+  std::vector mINChanA;                                                        ///< integrated 1D FIT currents for NChan A
+  std::vector mINChanC;                                                        ///< integrated 1D FIT currents for NChan C
+  std::vector mIAmplA;                                                         ///< integrated 1D FIT currents for Ampl A
+  std::vector mIAmplC;                                                         ///< integrated 1D FIT currents for Ampl C
+  long mTimeMS{};                                                                     ///< start time in ms
+  bool areSameSize() const { return sameSize(mINChanA, mINChanC, mIAmplA, mIAmplC); } ///< check if stored currents have same number of entries
+  bool isEmpty() const { return mINChanA.empty(); }                                   ///< check if values are empty
+  size_t getEntries() const { return mINChanA.size(); }                               ///< \return returns number of values stored
+  void setStartTime(long timeMS) { mTimeMS = timeMS; }
+
+  /// acummulate integrated currents at given index
+  /// \param posIndex index where data will be copied to
+  /// \param data integrated currents which will be copied
+  void fill(const unsigned int posIndex, const IFT0C& data)
+  {
+    std::copy(data.mINChanA.begin(), data.mINChanA.end(), mINChanA.begin() + posIndex);
+    std::copy(data.mINChanC.begin(), data.mINChanC.end(), mINChanC.begin() + posIndex);
+    std::copy(data.mIAmplA.begin(), data.mIAmplA.end(), mIAmplA.begin() + posIndex);
+    std::copy(data.mIAmplC.begin(), data.mIAmplC.end(), mIAmplC.begin() + posIndex);
+  }
+
+  /// \param nDummyValues number of empty values which are inserted at the beginning of the accumulated integrated currents
+  void insert(const unsigned int nDummyValues)
+  {
+    std::vector vecTmp(nDummyValues, 0);
+    mINChanA.insert(mINChanA.begin(), vecTmp.begin(), vecTmp.end());
+    mINChanC.insert(mINChanC.begin(), vecTmp.begin(), vecTmp.end());
+    mIAmplA.insert(mIAmplA.begin(), vecTmp.begin(), vecTmp.end());
+    mIAmplC.insert(mIAmplC.begin(), vecTmp.begin(), vecTmp.end());
+  }
+
+  /// resize buffer for accumulated currents
+  void resize(const unsigned int nTotal)
+  {
+    mINChanA.resize(nTotal);
+    mINChanC.resize(nTotal);
+    mIAmplA.resize(nTotal);
+    mIAmplC.resize(nTotal);
+  }
+
+  /// reset buffered currents
+  void reset()
+  {
+    std::fill(mINChanA.begin(), mINChanA.end(), 0);
+    std::fill(mINChanC.begin(), mINChanC.end(), 0);
+    std::fill(mIAmplA.begin(), mIAmplA.end(), 0);
+    std::fill(mIAmplC.begin(), mIAmplC.end(), 0);
+  }
+
+  /// normalize currents
+  void normalize(const float factor)
+  {
+    std::transform(mINChanA.begin(), mINChanA.end(), mINChanA.begin(), [factor](const float val) { return val * factor; });
+    std::transform(mINChanC.begin(), mINChanC.end(), mINChanC.begin(), [factor](const float val) { return val * factor; });
+    std::transform(mIAmplA.begin(), mIAmplA.end(), mIAmplA.begin(), [factor](const float val) { return val * factor; });
+    std::transform(mIAmplC.begin(), mIAmplC.end(), mIAmplC.begin(), [factor](const float val) { return val * factor; });
+  }
+
+  ClassDefNV(IFT0C, 2);
+};
+
+/// struct containing the integrated FV0 currents
+struct IFV0C {
+  std::vector mINChanA;                                     ///< integrated 1D FIT currents for NChan A
+  std::vector mIAmplA;                                      ///< integrated 1D FIT currents for Ampl A
+  long mTimeMS{};                                                  ///< start time in ms
+  bool areSameSize() const { return sameSize(mINChanA, mIAmplA); } ///< check if stored currents have same number of entries
+  bool isEmpty() const { return mINChanA.empty(); }                ///< check if values are empty
+  size_t getEntries() const { return mINChanA.size(); }            ///< \return returns number of values stored
+  void setStartTime(long timeMS) { mTimeMS = timeMS; }
+
+  /// acummulate integrated currents at given index
+  /// \param posIndex index where data will be copied to
+  /// \param data integrated currents which will be copied
+  void fill(const unsigned int posIndex, const IFV0C& data)
+  {
+    std::copy(data.mINChanA.begin(), data.mINChanA.end(), mINChanA.begin() + posIndex);
+    std::copy(data.mIAmplA.begin(), data.mIAmplA.end(), mIAmplA.begin() + posIndex);
+  }
+
+  /// \param nDummyValues number of empty values which are inserted at the beginning of the accumulated integrated currents
+  void insert(const unsigned int nDummyValues)
+  {
+    std::vector vecTmp(nDummyValues, 0);
+    mINChanA.insert(mINChanA.begin(), vecTmp.begin(), vecTmp.end());
+    mIAmplA.insert(mIAmplA.begin(), vecTmp.begin(), vecTmp.end());
+  }
+
+  /// resize buffer for accumulated currents
+  void resize(const unsigned int nTotal)
+  {
+    mINChanA.resize(nTotal);
+    mIAmplA.resize(nTotal);
+  }
+
+  /// reset buffered currents
+  void reset()
+  {
+    std::fill(mINChanA.begin(), mINChanA.end(), 0);
+    std::fill(mIAmplA.begin(), mIAmplA.end(), 0);
+  }
+
+  /// normalize currents
+  void normalize(const float factor)
+  {
+    std::transform(mINChanA.begin(), mINChanA.end(), mINChanA.begin(), [factor](const float val) { return val * factor; });
+    std::transform(mIAmplA.begin(), mIAmplA.end(), mIAmplA.begin(), [factor](const float val) { return val * factor; });
+  }
+
+  ClassDefNV(IFV0C, 2);
+};
+
+/// struct containing the integrated FDD currents
+struct IFDDC {
+  std::vector mINChanA;                                                        ///< integrated 1D FIT currents for NChan A
+  std::vector mINChanC;                                                        ///< integrated 1D FIT currents for NChan C
+  std::vector mIAmplA;                                                         ///< integrated 1D FIT currents for Ampl A
+  std::vector mIAmplC;                                                         ///< integrated 1D FIT currents for Ampl C
+  long mTimeMS{};                                                                     ///< start time in ms
+  bool areSameSize() const { return sameSize(mINChanA, mINChanC, mIAmplA, mIAmplC); } ///< check if stored currents have same number of entries
+  bool isEmpty() const { return mINChanA.empty(); }                                   ///< check if values are empty
+  size_t getEntries() const { return mINChanA.size(); }                               ///< \return returns number of values stored
+  void setStartTime(long timeMS) { mTimeMS = timeMS; }
+
+  /// acummulate integrated currents at given index
+  /// \param posIndex index where data will be copied to
+  /// \param data integrated currents which will be copied
+  void fill(const unsigned int posIndex, const IFDDC& data)
+  {
+    std::copy(data.mINChanA.begin(), data.mINChanA.end(), mINChanA.begin() + posIndex);
+    std::copy(data.mINChanC.begin(), data.mINChanC.end(), mINChanC.begin() + posIndex);
+    std::copy(data.mIAmplA.begin(), data.mIAmplA.end(), mIAmplA.begin() + posIndex);
+    std::copy(data.mIAmplC.begin(), data.mIAmplC.end(), mIAmplC.begin() + posIndex);
+  }
+
+  /// \param nDummyValues number of empty values which are inserted at the beginning of the accumulated integrated currents
+  void insert(const unsigned int nDummyValues)
+  {
+    std::vector vecTmp(nDummyValues, 0);
+    mINChanA.insert(mINChanA.begin(), vecTmp.begin(), vecTmp.end());
+    mINChanC.insert(mINChanC.begin(), vecTmp.begin(), vecTmp.end());
+    mIAmplA.insert(mIAmplA.begin(), vecTmp.begin(), vecTmp.end());
+    mIAmplC.insert(mIAmplC.begin(), vecTmp.begin(), vecTmp.end());
+  }
+
+  /// resize buffer for accumulated currents
+  void resize(const unsigned int nTotal)
+  {
+    mINChanA.resize(nTotal);
+    mINChanC.resize(nTotal);
+    mIAmplA.resize(nTotal);
+    mIAmplC.resize(nTotal);
+  }
+
+  /// reset buffered currents
+  void reset()
+  {
+    std::fill(mINChanA.begin(), mINChanA.end(), 0);
+    std::fill(mINChanC.begin(), mINChanC.end(), 0);
+    std::fill(mIAmplA.begin(), mIAmplA.end(), 0);
+    std::fill(mIAmplC.begin(), mIAmplC.end(), 0);
+  }
+
+  /// normalize currents
+  void normalize(const float factor)
+  {
+    std::transform(mINChanA.begin(), mINChanA.end(), mINChanA.begin(), [factor](const float val) { return val * factor; });
+    std::transform(mINChanC.begin(), mINChanC.end(), mINChanC.begin(), [factor](const float val) { return val * factor; });
+    std::transform(mIAmplA.begin(), mIAmplA.end(), mIAmplA.begin(), [factor](const float val) { return val * factor; });
+    std::transform(mIAmplC.begin(), mIAmplC.end(), mIAmplC.begin(), [factor](const float val) { return val * factor; });
+  }
+
+  ClassDefNV(IFDDC, 2);
+};
+
+} // end namespace fit
+} // end namespace o2
+
+namespace o2
+{
+namespace calibration
+{
+
+/// class for accumulating integrated currents
+template 
+class IntegratedClusters
+{
+ public:
+  /// \constructor
+  /// \param tFirst first TF of the stored currents
+  /// \param tLast last TF of the stored currents
+  IntegratedClusters(o2::calibration::TFType tFirst, o2::calibration::TFType tLast) : mTFFirst{tFirst}, mTFLast{tLast} {};
+
+  /// \default constructor for ROOT I/O
+  IntegratedClusters() = default;
+
+  /// print summary informations
+  void print() const { LOGP(info, "TF Range from {} to {} with {} of remaining data", mTFFirst, mTFLast, mRemainingData); }
+
+  /// accumulate currents for given TF
+  /// \param tfID TF ID of incoming data
+  /// \param currentsContainer container containing the currents for given detector for one TF for number of clusters
+  void fill(const o2::calibration::TFType tfID, const DataT& currentsContainer);
+
+  /// merging TOF currents with previous interval
+  void merge(const IntegratedClusters* prev);
+
+  /// \return always return true. To specify the number of time slot intervals to wait for one should use the --max-delay option
+  bool hasEnoughData() const { return (mRemainingData != -1); }
+
+  /// \return returns accumulated currents
+  const auto& getCurrents() const& { return mCurrents; }
+
+  /// \return returns accumulated currents using move semantics
+  auto getCurrents() && { return std::move(mCurrents); }
+
+  /// \param currents currents for given detector which will be set
+  void setCurrents(const DataT& currents) { mCurrents = currents; }
+
+  /// dump object to disc
+  /// \param outFileName name of the output file
+  /// \param outName name of the object in the output file
+  void dumpToFile(const char* outFileName = "IntegratedClusters.root", const char* outName = "IC") const;
+
+  /// dump object to TTree for visualisation
+  /// \param outFileName name of the output file
+  void dumpToTree(const char* outFileName = "ICTree.root");
+
+  /// setting the start time
+  void setStartTime(long timeMS) { mCurrents.setStartTime(timeMS); }
+
+ private:
+  DataT mCurrents;                             ///< buffer for integrated currents
+  o2::calibration::TFType mTFFirst{};          ///< first TF of currents
+  o2::calibration::TFType mTFLast{};           ///< last TF of currents
+  o2::calibration::TFType mRemainingData = -1; ///< counter for received data
+  unsigned int mNValuesPerTF{};                ///< number of expected currents per TF (estimated from first received data)
+  bool mInitialize{true};                      ///< flag if this object will be initialized when fill method is called
+
+  /// init member when first data is received
+  /// \param valuesPerTF number of expected values per TF
+  void initData(const unsigned int valuesPerTF);
+
+  ClassDefNV(IntegratedClusters, 1);
+};
+
+template 
+class IntegratedClusterCalibrator : public o2::calibration::TimeSlotCalibration>
+{
+  using TFType = o2::calibration::TFType;
+  using Slot = o2::calibration::TimeSlot>;
+  using CalibVector = std::vector;
+  using TFinterval = std::vector>;
+  using TimeInterval = std::vector>;
+
+ public:
+  /// default constructor
+  IntegratedClusterCalibrator() = default;
+
+  /// default destructor
+  ~IntegratedClusterCalibrator() final = default;
+
+  /// check if given slot has already enough data
+  bool hasEnoughData(const Slot& slot) const final { return slot.getContainer()->hasEnoughData(); }
+
+  /// clearing all calibration objects in the output buffer
+  void initOutput() final;
+
+  /// storing the integrated currents for given slot
+  void finalizeSlot(Slot& slot) final;
+
+  /// Creates new time slot
+  Slot& emplaceNewSlot(bool front, TFType tstart, TFType tend) final;
+
+  /// \return CCDB output informations
+  const TFinterval& getTFinterval() const { return mIntervals; }
+
+  /// \return Time frame time information
+  const TimeInterval& getTimeIntervals() const { return mTimeIntervals; }
+
+  /// \return returns calibration objects (pad-by-pad gain maps)
+  auto getCalibs() && { return std::move(mCalibs); }
+
+  /// check if calibration data is available
+  bool hasCalibrationData() const { return mCalibs.size() > 0; }
+
+  /// set if debug objects will be created
+  void setDebug(const bool debug) { mDebug = debug; }
+
+ private:
+  TFinterval mIntervals;       ///< start and end time frames of each calibration time slots
+  TimeInterval mTimeIntervals; ///< start and end times of each calibration time slots
+  CalibVector mCalibs;         ///< Calibration object containing for each pad a histogram with normalized charge
+  bool mDebug{false};          ///< write debug output objects
+
+  ClassDefOverride(IntegratedClusterCalibrator, 1);
+};
+
+} // end namespace calibration
+} // end namespace o2
+
+#endif
diff --git a/Detectors/Calibration/include/DetectorsCalibration/MeanVertexCalibrator.h b/Detectors/Calibration/include/DetectorsCalibration/MeanVertexCalibrator.h
index 056b1bdfae556..33b3e22a79f5c 100644
--- a/Detectors/Calibration/include/DetectorsCalibration/MeanVertexCalibrator.h
+++ b/Detectors/Calibration/include/DetectorsCalibration/MeanVertexCalibrator.h
@@ -26,7 +26,7 @@ namespace o2
 namespace calibration
 {
 
-class MeanVertexCalibrator final : public o2::calibration::TimeSlotCalibration
+class MeanVertexCalibrator final : public o2::calibration::TimeSlotCalibration
 {
   using PVertex = o2::dataformats::PrimaryVertex;
   using MeanVertexData = o2::calibration::MeanVertexData;
@@ -37,34 +37,21 @@ class MeanVertexCalibrator final : public o2::calibration::TimeSlotCalibration;
 
  public:
-  MeanVertexCalibrator(int minEnt = 500, int nBinsX = 100, float rangeX = 1.f,
-                       int nBinsY = 100, float rangeY = 1.f, int nBinsZ = 100, float rangeZ = 20.f, uint32_t nPointsForSlope = 10,
-                       int nSlotsSMA = 5) : mMinEntries(minEnt), mNBinsX(nBinsX), mRangeX(rangeX), mNBinsY(nBinsY), mRangeY(rangeY), mNBinsZ(nBinsZ), mRangeZ(rangeZ), mSMAslots(nSlotsSMA), mNPointsForSlope(nPointsForSlope)
-  {
-    mBinWidthX = 2 * rangeX / nBinsX;
-    mBinWidthY = 2 * rangeY / nBinsY;
-    mBinWidthZ = 2 * rangeZ / nBinsZ;
-    mBinWidthXInv = 1. / mBinWidthX;
-    mBinWidthYInv = 1. / mBinWidthY;
-    mBinWidthZInv = 1. / mBinWidthZ;
-  }
-
+  struct HistoParams {
+    int nBins = 0.;
+    float binWidth = 0.;
+    float minRange = 0.;
+    float maxRange = 0.;
+  };
+
+  MeanVertexCalibrator() = default;
   ~MeanVertexCalibrator() final = default;
 
-  bool hasEnoughData(const Slot& slot) const final
-  {
-    if (mVerbose) {
-      LOG(info) << "container entries = " << slot.getContainer()->entries << ", minEntries = " << mMinEntries * 2;
-    }
-    return slot.getContainer()->entries >= mMinEntries * 2; // we need in fact that the min number of entries is 2x the required ones because we will do the slices in z, and we need at least two to fit
-  }
+  bool hasEnoughData(const Slot& slot) const final;
   void initOutput() final;
   void finalizeSlot(Slot& slot) final;
   Slot& emplaceNewSlot(bool front, TFType tstart, TFType tend) final;
 
-  uint64_t getNSlotsSMA() const { return mSMAslots; }
-  void setNSlotsSMA(uint64_t nslots) { mSMAslots = nslots; }
-
   void doSimpleMovingAverage(std::deque& dq, float& sma);
   void doSimpleMovingAverage(std::deque& dq, MVObject& sma);
 
@@ -75,28 +62,13 @@ class MeanVertexCalibrator final : public o2::calibration::TimeSlotCalibration& vectOut, const std::vector& vectIn, int nbins, float min, float max, float binWidthInv);
-  void printVector(std::vector& vect, float minRange, float maxRange, float binWidth);
-  void printVector(float* vect, int sizeVect, float minRange, float maxRange, float binWidth);
+  bool fitMeanVertex(o2::calibration::MeanVertexData* c, o2::dataformats::MeanVertexObject& mvo);
+  void fitMeanVertexCoord(int icoord, const float* array, const HistoParams& hpar, o2::dataformats::MeanVertexObject& mvo);
+  HistoParams binVector(std::vector& vectOut, const std::vector& vectIn, o2::calibration::MeanVertexData* c, int dim);
+  void printVector(const std::vector& vect, const HistoParams& hpar);
+  void printVector(const float* vect, const HistoParams& hpar);
 
  private:
-  int mMinEntries = 0;
-  int mNBinsX = 0;
-  float mRangeX = 0.;
-  int mNBinsY = 0;
-  float mRangeY = 0.;
-  int mNBinsZ = 0;
-  float mRangeZ = 0.;
-  float mBinWidthX = 0.;
-  float mBinWidthY = 0.;
-  float mBinWidthZ = 0.;
-  float mBinWidthXInv = 0.;
-  float mBinWidthYInv = 0.;
-  float mBinWidthZInv = 0.;
-  uint64_t mSMAslots = 5;
-  uint32_t mNPointsForSlope = 10;
   CcdbObjectInfoVector mInfoVector;                                    // vector of CCDB Infos , each element is filled with the CCDB description
                                                                        // of the accompanying LHCPhase
   MVObjectVector mMeanVertexVector;                                    // vector of Mean Vertex Objects, each element is filled in "process"
diff --git a/Detectors/Calibration/include/DetectorsCalibration/MeanVertexData.h b/Detectors/Calibration/include/DetectorsCalibration/MeanVertexData.h
index 1ce6efeedc4ea..877a8cc99c060 100644
--- a/Detectors/Calibration/include/DetectorsCalibration/MeanVertexData.h
+++ b/Detectors/Calibration/include/DetectorsCalibration/MeanVertexData.h
@@ -28,15 +28,18 @@ struct MeanVertexData {
   using PVertex = o2::dataformats::PrimaryVertex;
   int entries = 0;
   std::vector> histoVtx{0};
+  std::array means{};
+  std::array meanSquares{};
   bool mVerbose = false;
 
-  MeanVertexData();
+  MeanVertexData() = default;
 
   ~MeanVertexData()
   {
     histoVtx.clear();
   }
-
+  double getMean(int i) const { return means[i]; }
+  double getRMS(int i) const;
   MeanVertexData(MeanVertexData&& other) = default;
   MeanVertexData(const MeanVertexData& other) = default;
   MeanVertexData& operator=(MeanVertexData& other) = default;
diff --git a/Detectors/Calibration/include/DetectorsCalibration/MeanVertexParams.h b/Detectors/Calibration/include/DetectorsCalibration/MeanVertexParams.h
index a0cf14800a59c..ffbea723e09cc 100644
--- a/Detectors/Calibration/include/DetectorsCalibration/MeanVertexParams.h
+++ b/Detectors/Calibration/include/DetectorsCalibration/MeanVertexParams.h
@@ -24,18 +24,16 @@ namespace calibration
 
 // There are configurable params for TPC-ITS matching
 struct MeanVertexParams : public o2::conf::ConfigurableParamHelper {
-
+  float histoNSigma[3] = {4., 4., 4.};         // histo ranges defined as mean+-nsigma*sigma
+  float histoBinSize[3] = {0.002, 0.002, 0.5}; // cm
+  float minSigma[3] = {0.001, 0.001, 1.0};     // use this for histo definition if sigma is smaller
   int minEntries = 100;
-  int nbinsX = 100;
-  float rangeX = 1.f;
-  int nbinsY = 100;
-  float rangeY = 1.f;
-  int nbinsZ = 100;
-  float rangeZ = 20.f;
   int nSlots4SMA = 5;
-  uint32_t tfPerSlot = 5u;
+  uint32_t tfPerSlot = 5400u; // ~10 min
   uint32_t maxTFdelay = 3u;
-  uint32_t nPointsForSlope = 10;
+  uint32_t nPointsForSlope = 5;
+  bool dumpNonEmptyBins = false;
+  bool skipObjectSending = false;
 
   O2ParamDef(MeanVertexParams, "MeanVertexCalib");
 };
diff --git a/Detectors/Calibration/include/DetectorsCalibration/TimeSlot.h b/Detectors/Calibration/include/DetectorsCalibration/TimeSlot.h
index e514ccac8d37f..0b410f4a397e9 100644
--- a/Detectors/Calibration/include/DetectorsCalibration/TimeSlot.h
+++ b/Detectors/Calibration/include/DetectorsCalibration/TimeSlot.h
@@ -35,22 +35,18 @@ class TimeSlot
  public:
   TimeSlot() = default;
   TimeSlot(TFType tfS, TFType tfE) : mTFStart(tfS), mTFEnd(tfE) {}
-  TimeSlot(const TimeSlot& src) : mTFStart(src.mTFStart), mTFEnd(src.mTFEnd), mContainer(std::make_unique(*src.getContainer())) {}
-  TimeSlot& operator=(const TimeSlot& src)
+  TimeSlot(const TimeSlot& src) : mTFStart(src.mTFStart), mTFEnd(src.mTFEnd), mEntries(src.mEntries), mRunStartOrbit(src.mRunStartOrbit), mTFStartMS(src.mTFStartMS)
   {
-    if (&src != this) {
-      mTFStart = src.mTFStart;
-      mTFEnd = src.mTFEnd;
-      mContainer = std::make_unique(*src.getContainer());
-    }
-    return *this;
+    mContainer = src.mContainer ? std::make_unique(*src.mContainer) : nullptr;
   }
+  TimeSlot& operator=(TimeSlot&& src) = default;
 
   ~TimeSlot() = default;
 
   TFType getTFStart() const { return mTFStart; }
   TFType getTFEnd() const { return mTFEnd; }
 
+  long getStaticStartTimeMS() const { return mTFStartMS; }
   long getStartTimeMS() const { return o2::base::GRPGeomHelper::instance().getOrbitResetTimeMS() + (mRunStartOrbit + long(o2::base::GRPGeomHelper::getNHBFPerTF()) * mTFStart) * o2::constants::lhc::LHCOrbitMUS / 1000; }
   long getEndTimeMS() const { return o2::base::GRPGeomHelper::instance().getOrbitResetTimeMS() + (mRunStartOrbit + long(o2::base::GRPGeomHelper::getNHBFPerTF()) * (mTFEnd + 1)) * o2::constants::lhc::LHCOrbitMUS / 1000; }
 
@@ -60,6 +56,7 @@ class TimeSlot
 
   void setTFStart(TFType v) { mTFStart = v; }
   void setTFEnd(TFType v) { mTFEnd = v; }
+  void setStaticStartTimeMS(long t) { mTFStartMS = t; }
   void setRunStartOrbit(long t) { mRunStartOrbit = t; }
   auto getRunStartOrbit() const { return mRunStartOrbit; }
 
@@ -71,11 +68,12 @@ class TimeSlot
   {
     mContainer->merge(prev.mContainer.get());
     mTFStart = prev.mTFStart;
+    mTFStartMS = prev.mTFStartMS;
   }
 
   void print() const
   {
-    LOGF(info, "Calibration slot %5d <=TF<=  %5d", mTFStart, mTFEnd);
+    LOGF(info, "Calibration slot %5d <=TF<=  %5d (start in ms = %ld)", mTFStart, mTFEnd, mTFStartMS);
     mContainer->print();
   }
 
@@ -85,8 +83,9 @@ class TimeSlot
   size_t mEntries = 0;
   long mRunStartOrbit = 0;
   std::unique_ptr mContainer; // user object to accumulate the calibration data for this slot
+  long mTFStartMS = 0;                   // start time of the slot in ms that avoids to calculate it on the fly; needed when a slot covers more runs, otherwise the OrbitReset that is read is the one of the latest run, and the validity will be wrong
 
-  ClassDefNV(TimeSlot, 1);
+  ClassDefNV(TimeSlot, 2);
 };
 
 } // namespace calibration
diff --git a/Detectors/Calibration/include/DetectorsCalibration/TimeSlotCalibration.h b/Detectors/Calibration/include/DetectorsCalibration/TimeSlotCalibration.h
index 40f62e254a85c..87562afddf2ca 100644
--- a/Detectors/Calibration/include/DetectorsCalibration/TimeSlotCalibration.h
+++ b/Detectors/Calibration/include/DetectorsCalibration/TimeSlotCalibration.h
@@ -15,13 +15,17 @@
 /// @brief Processor for the multiple time slots calibration
 
 #include "DetectorsCalibration/TimeSlot.h"
+#include "DetectorsCalibration/TimeSlotMetaData.h"
 #include "DetectorsBase/TFIDInfoHelper.h"
 #include "DetectorsBase/GRPGeomHelper.h"
 #include "CommonDataFormat/TFIDInfo.h"
+#include 
+#include 
 #include 
 #include 
 #include 
 #include 
+#include 
 
 namespace o2
 {
@@ -32,7 +36,7 @@ class ProcessingContext;
 namespace calibration
 {
 
-template 
+template 
 class TimeSlotCalibration
 {
  public:
@@ -70,7 +74,7 @@ class TimeSlotCalibration
   void checkSlotLength()
   {
     if (mSlotLengthInSeconds > 0) {
-      TFType ntf = mSlotLengthInSeconds / (long(o2::base::GRPGeomHelper::getNHBFPerTF()) * o2::constants::lhc::LHCOrbitMUS / 1000000);
+      TFType ntf = mSlotLengthInSeconds / (o2::base::GRPGeomHelper::getNHBFPerTF() * o2::constants::lhc::LHCOrbitMUS * 1e-6);
       LOGP(info, "Redefining slot duration from {} s. to {} TFs", mSlotLengthInSeconds, ntf);
       setSlotLength(ntf);
       mSlotLengthInSeconds = 0;
@@ -83,12 +87,35 @@ class TimeSlotCalibration
       setSlotLength(ntf);
       mSlotLengthInOrbits = 0;
     }
+    setStartOffsetFrac(mStartOffsetFrac); // set once more to account for eventual dependencies
+    mStartOffsetTFs = TFType(mSlotLength * mStartOffsetFrac);
+  }
+
+  void setStartOffsetFrac(float f)
+  {
+    if (mUpdateAtTheEndOfRunOnly || mFinalizeWhenReady || mSlotLength == INFINITE_TF) { // offset makes no sense for run-wide objects
+      if (f) {
+        LOGP(info, "Start offset is not supported in the INFINITE_TF slot length or UpdateAtTheEndOfRunOnly or FinalizeWhenReady modes");
+      }
+      return;
+    }
+    if (f < 0.) {
+      mStartOffsetFrac = 0.;
+    } else if (f > 0.95) {
+      mStartOffsetFrac = 0.95;
+    } else {
+      mStartOffsetFrac = f;
+    }
+    if (mStartOffsetFrac || f != mStartOffsetFrac) {
+      LOGP(info, "Imposing offset of {:4.2} x nominal slot length", mStartOffsetFrac);
+    }
   }
 
   void setFinalizeWhenReady()
   {
     mFinalizeWhenReady = true;
     setSlotLength(INFINITE_TF);
+    mStartOffsetFrac = 0;
   }
 
   void setUpdateAtTheEndOfRunOnly() { mUpdateAtTheEndOfRunOnly = kTRUE; }
@@ -100,11 +127,22 @@ class TimeSlotCalibration
   const Slot& getLastSlot() const { return (Slot&)mSlots.back(); }
   const Slot& getFirstSlot() const { return (Slot&)mSlots.front(); }
 
-  template 
-  bool process(const DATA& data);
-  virtual void checkSlotsToFinalize(TFType tf, int maxDelay = 0);
+  template 
+  bool process(const DATA&... data);
+  virtual void checkSlotsToFinalize(TFType tf = INFINITE_TF, int maxDelay = 0);
   virtual void finalizeOldestSlot();
 
+  virtual void reset()
+  { // reset to virgin state (need for start - stop - start)
+    mSlots.clear();
+    mLastClosedTF = 0;
+    mFirstTF = 0;
+    mMaxSeenTF = 0;
+    mLastCheckedTFInfiniteSlot = 0;
+    mWasCheckedInfiniteSlot = false;
+    initOutput();
+  }
+
   // Methods to be implemented by the derived user class
 
   // implement and call this method te reset the output slots once they are not needed
@@ -149,20 +187,61 @@ class TimeSlotCalibration
     static constexpr bool value = type::value;
   };
 
+  // methods for saving/reading data in the of the run in case of insufficient statistics
+  bool getSavedSlotAllowed() const { return mSavedSlotAllowed; }
+  void setSavedSlotAllowed(bool v) { mSavedSlotAllowed = v; }
+  std::string getSaveFilePath() const;
+  const std::string& getSaveFileName() const { return mSaveFileName; }
+  void setSaveFileName(const std::string& n) { mSaveFileName = n; }
+  void setSaveDirectory(const std::string& n) { mSaveDirectory = n; }
+  virtual bool updateSaveMetaData();
+
+  // derived class using slot saving functionality must implement this method to write the
+  // content of the slot, returning true on success
+  virtual bool saveLastSlotData(TFile& fl)
+  {
+    LOG(fatal) << "This method must be implemented by derived class to write content of the slot to save";
+    return false;
+  }
+  // derived class using slot saving functionality must implement this method to adopt the content of the
+  // saved slot, returning true on success. Provided metadata should be used to judge if the saved data is useful.
+  virtual bool adoptSavedData(const TimeSlotMetaData& metadata, TFile& fl)
+  {
+    LOG(fatal) << "This method must be implemented by derived class to adopt content of the saved slot";
+    return false;
+  }
+  virtual bool loadSavedSlot();
+  virtual bool saveLastSlot();
+
  protected:
   auto& getSlots() { return mSlots; }
   uint32_t getRunStartOrbit() const
   {
     long orb = long(mCurrentTFInfo.firstTForbit) - long(o2::base::GRPGeomHelper::getNHBFPerTF() * mCurrentTFInfo.tfCounter);
+    static unsigned int threshold = 512 * o2::base::GRPGeomHelper::getNHBFPerTF();
     if (orb < 0) {
-      LOGP(alarm, "Negative runStartOrbit = {} deduced from tfCounter={} and firstTForbit={}, enforcing runStartOrbit to 0", orb, mCurrentTFInfo.tfCounter, mCurrentTFInfo.firstTForbit);
+      // If we have a firstTForbit between 1 and 512 * tf len, we disable the warning for negative runStartOrbit permanently, since this is a SYNTHETIC run.
+      static bool suppressRunStartWarning = false;
+      if (!suppressRunStartWarning) {
+        const auto* grpecs = o2::base::GRPGeomHelper::instance().getGRPECS();
+        if (grpecs) {
+          if (grpecs->getRunType() == o2::parameters::GRPECS::SYNTHETIC) {
+            suppressRunStartWarning = true;
+          }
+        } else if (mCurrentTFInfo.firstTForbit < threshold && mCurrentTFInfo.firstTForbit > 0) {
+          suppressRunStartWarning = true;
+        }
+      }
+
+      if (!suppressRunStartWarning && mCurrentTFInfo.firstTForbit >= threshold) {
+        LOGP(alarm, "Negative runStartOrbit = {} deduced from tfCounter={} and firstTForbit={}, enforcing runStartOrbit to 0", orb, mCurrentTFInfo.tfCounter, mCurrentTFInfo.firstTForbit);
+      }
       orb = 0;
     }
     return uint32_t(orb);
   }
 
   TFType tf2SlotMin(TFType tf) const;
-
   std::deque mSlots;
 
   o2::dataformats::TFIDInfo mCurrentTFInfo{};
@@ -172,6 +251,8 @@ class TimeSlotCalibration
   TFType mFirstTF = 0;
   TFType mMaxSeenTF = 0;                        // largest TF processed
   TFType mSlotLength = 1;                       // slot length in TFs
+  TFType mStartOffsetTFs = 0;                   // shift start of all TFs backwards by this amount (to make 1st slot effectively shorter: run_1st_tf to run_1st_tf - offset + mSlotLength), derived from mStartOffsetFrac
+  float mStartOffsetFrac = 0.;                  // shift start of all TFs backwards mSlotLength*mStartOffsetFrac TFs.
   TFType mCheckIntervalInfiniteSlot = 1;        // will be used if the TF length is INFINITE_TF_int64 to decide
                                                 // when to check if to call the finalize; otherwise it is called
                                                 // at every new TF; note that this is an approximation,
@@ -186,13 +267,19 @@ class TimeSlotCalibration
   bool mWasCheckedInfiniteSlot = false;         // flag to know whether the statistics of the infinite slot was already checked
   bool mUpdateAtTheEndOfRunOnly = false;
   bool mFinalizeWhenReady = false; // if true: single bin is filled until ready, then closed and new one is added
+
+  std::string mSaveDirectory = ""; // directory where the file is saved
+  std::string mSaveFileName = "";  // filename for data saves in the end of the run
+  TimeSlotMetaData mSaveMetaData{};
+  bool mSavedSlotAllowed = false;
+
   ClassDef(TimeSlotCalibration, 1);
 };
 
 //_________________________________________________
-template 
-template 
-bool TimeSlotCalibration::process(const DATA& data)
+template 
+template 
+bool TimeSlotCalibration::process(const DATA&... data)
 {
   static bool firstCall = true;
   if (firstCall) {
@@ -215,10 +302,10 @@ bool TimeSlotCalibration::process(const DATA& data)
   }
   auto& slotTF = getSlotForTF(tf);
   using Cont_t = typename std::remove_pointer::type;
-  if constexpr (has_fill_method::value) {
-    slotTF.getContainer()->fill(mCurrentTFInfo, data);
+  if constexpr (has_fill_method::value) {
+    slotTF.getContainer()->fill(mCurrentTFInfo, data...);
   } else {
-    slotTF.getContainer()->fill(data);
+    slotTF.getContainer()->fill(data...);
   }
   if (tf > mMaxSeenTF) {
     mMaxSeenTF = tf; // keep track of the most recent TF processed
@@ -232,8 +319,8 @@ bool TimeSlotCalibration::process(const DATA& data)
 }
 
 //_________________________________________________
-template 
-void TimeSlotCalibration::checkSlotsToFinalize(TFType tf, int maxDelay)
+template 
+void TimeSlotCalibration::checkSlotsToFinalize(TFType tf, int maxDelay)
 {
   // Check which slots can be finalized, provided the newly arrived TF is tf
 
@@ -301,8 +388,8 @@ void TimeSlotCalibration::checkSlotsToFinalize(TFType tf, int
 }
 
 //_________________________________________________
-template 
-void TimeSlotCalibration::finalizeOldestSlot()
+template 
+void TimeSlotCalibration::finalizeOldestSlot()
 {
   // Enforce finalization and removal of the oldest slot
   if (mSlots.empty()) {
@@ -315,25 +402,29 @@ void TimeSlotCalibration::finalizeOldestSlot()
 }
 
 //________________________________________
-template 
-inline TFType TimeSlotCalibration::tf2SlotMin(TFType tf) const
+template 
+inline TFType TimeSlotCalibration::tf2SlotMin(TFType tf) const
 {
-
   // returns the min TF of the slot to which "tf" belongs
-
   if (tf < mFirstTF) {
     throw std::runtime_error("invalid TF");
   }
   if (mUpdateAtTheEndOfRunOnly) {
     return mFirstTF;
   }
-  uint64_t tft = int64_t(((tf - mFirstTF) / mSlotLength) * mSlotLength) + mFirstTF;
+  int64_t tft = 0;
+  tft = int64_t(((tf - mFirstTF + mStartOffsetTFs) / mSlotLength) * mSlotLength) + mFirstTF;
+  if (tft > mStartOffsetTFs) {
+    tft -= mStartOffsetTFs;
+  } else {
+    tft = 0;
+  }
   return tft < o2::calibration::INFINITE_TF ? TFType(tft) : INFINITE_TF;
 }
 
 //_________________________________________________
-template 
-TimeSlot& TimeSlotCalibration::getSlotForTF(TFType tf)
+template 
+TimeSlot& TimeSlotCalibration::getSlotForTF(TFType tf)
 {
 
   LOG(debug) << "Getting slot for TF " << tf;
@@ -343,6 +434,7 @@ TimeSlot& TimeSlotCalibration::getSlotForTF(TFType
     } else if (mSlots.empty()) {
       auto& sl = emplaceNewSlot(true, mFirstTF, tf);
       sl.setRunStartOrbit(getRunStartOrbit());
+      sl.setStaticStartTimeMS(sl.getStartTimeMS());
     }
     return mSlots.back();
   }
@@ -351,11 +443,12 @@ TimeSlot& TimeSlotCalibration::getSlotForTF(TFType
     auto tfmn = tf2SlotMin(mSlots.front().getTFStart() - 1); // min TF of the slot corresponding to a TF smaller than the first seen
     auto tftgt = tf2SlotMin(tf);                             // min TF of the slot to which the TF "tf" would belong
     while (tfmn >= tftgt) {
-      uint64_t tft = uint64_t(tfmn) + mSlotLength - 1;
+      uint64_t tft = mSlots.front().getTFStart() - 1;
       TFType tfmx = tft < o2::calibration::INFINITE_TF ? TFType(tft) : o2::calibration::INFINITE_TF;
       LOG(info) << "Adding new slot for " << tfmn << " <= TF <= " << tfmx;
       auto& sl = emplaceNewSlot(true, tfmn, tfmx);
       sl.setRunStartOrbit(getRunStartOrbit());
+      sl.setStaticStartTimeMS(sl.getStartTimeMS());
       if (!tfmn) {
         break;
       }
@@ -373,26 +466,134 @@ TimeSlot& TimeSlotCalibration::getSlotForTF(TFType
   auto tfmn = mSlots.empty() ? tf2SlotMin(tf) : tf2SlotMin(mSlots.back().getTFEnd() + 1);
   do {
     uint64_t tft = uint64_t(tfmn) + mSlotLength - 1;
+    if (mSlots.empty() && mStartOffsetTFs && tf < mStartOffsetTFs) { // if this was lowest possible TF, its length might be smaller than mSlotLength
+      tft -= mStartOffsetTFs;
+    }
     TFType tfmx = tft < o2::calibration::INFINITE_TF ? TFType(tft) : o2::calibration::INFINITE_TF;
     LOG(info) << "Adding new slot for " << tfmn << " <= TF <= " << tfmx;
     auto& sl = emplaceNewSlot(false, tfmn, tfmx);
     sl.setRunStartOrbit(getRunStartOrbit());
-    tfmn = tf2SlotMin(mSlots.back().getTFEnd() + 1);
+    sl.setStaticStartTimeMS(sl.getStartTimeMS());
+    tfmn = tft < o2::calibration::INFINITE_TF ? mSlots.back().getTFEnd() + 1 : tft;
   } while (tf > mSlots.back().getTFEnd());
 
   return mSlots.back();
 }
 
 //_________________________________________________
-template 
-void TimeSlotCalibration::print() const
+template 
+void TimeSlotCalibration::print() const
 {
   for (int i = 0; i < getNSlots(); i++) {
-    LOG(info) << "Slot #" << i << " of " << getNSlots();
+    LOG(info) << "Slot #" << i + 1 << " of " << getNSlots();
     getSlot(i).print();
   }
 }
 
+//_________________________________________________
+template 
+bool TimeSlotCalibration::updateSaveMetaData()
+{
+  if (mSlots.empty()) {
+    LOG(warn) << "Nothing to save, no TimeSlots defined";
+    return false;
+  }
+  if (mSaveMetaData.startRun < 0) {
+    mSaveMetaData.startRun = mCurrentTFInfo.runNumber;
+  }
+  mSaveMetaData.endRun = mCurrentTFInfo.runNumber;
+  if (mSaveMetaData.startTime < 0) {
+    mSaveMetaData.startTime = mSlots.back().getStartTimeMS();
+  }
+  mSaveMetaData.endTime = mSlots.back().getEndTimeMS();
+  return true;
+}
+
+//_________________________________________________
+template 
+bool TimeSlotCalibration::saveLastSlot()
+{
+  if (!getSavedSlotAllowed()) {
+    LOG(info) << "Slot saving is disabled";
+    return false;
+  }
+  if (!updateSaveMetaData()) {
+    return false;
+  }
+
+  if (!mSaveDirectory.empty() && !std::filesystem::exists(mSaveDirectory)) {
+    std::filesystem::create_directories(mSaveDirectory);
+    if (!std::filesystem::exists(mSaveDirectory)) {
+      LOGP(fatal, "could not create output directory {}", mSaveDirectory);
+    } else {
+      LOGP(info, "created calibration directory {}", mSaveDirectory);
+    }
+  }
+
+  auto pth = getSaveFilePath();
+  auto pthTmp = pth + ".part";
+  TFile flout(pthTmp.c_str(), "recreate");
+  if (flout.IsZombie()) {
+    LOGP(error, "failed to open save file {}", pth);
+    unlink(pthTmp.c_str());
+    return false;
+  }
+  if (!saveLastSlotData(flout)) { // call used method to store data
+    flout.Close();
+    unlink(pthTmp.c_str());
+    return false;
+  }
+  flout.WriteObjectAny(&mSaveMetaData, "o2::calibration::TimeSlotMetaData", "metadata");
+  flout.Close();
+  std::filesystem::rename(pthTmp, pth);
+  LOGP(info, "Saved data of the last slot to {}", pth);
+  return true;
+}
+
+//_________________________________________________
+template 
+bool TimeSlotCalibration::loadSavedSlot()
+{
+  if (!getSavedSlotAllowed()) {
+    LOG(info) << "Saved slot usage is disabled";
+    return false;
+  }
+  auto pth = getSaveFilePath();
+  if (!std::filesystem::exists(pth)) {
+    LOGP(info, "No save file {} is found", pth);
+    return false;
+  }
+  TFile flin(pth.c_str());
+  if (flin.IsZombie()) {
+    LOGP(error, "failed to open save file {}", pth);
+    return false;
+  }
+  auto meta = (o2::calibration::TimeSlotMetaData*)flin.GetObjectChecked("metadata", "o2::calibration::TimeSlotMetaData");
+  if (!meta) {
+    LOGP(error, "Failed to read metadata from {}", pth);
+    return false;
+  }
+  auto res = adoptSavedData(*meta, flin); // up to the detector to decide if data should be accepted
+  if (res) {
+    mSaveMetaData.startRun = meta->startRun;
+    mSaveMetaData.startTime = meta->startTime;
+    updateSaveMetaData();
+  }
+  flin.Close();
+  unlink(pth.c_str()); // cleanup used file
+  return true;
+}
+
+//_________________________________________________
+template 
+std::string TimeSlotCalibration::getSaveFilePath() const
+{
+  if (mSaveFileName.empty()) {
+    LOGP(fatal, "Save file name was not set");
+  }
+  return fmt::format("{}{}{}", mSaveDirectory, ((!mSaveDirectory.empty() && mSaveDirectory.back() != '/') ? "/" : ""), mSaveFileName);
+}
+
 } // namespace calibration
 } // namespace o2
 
diff --git a/Detectors/Calibration/include/DetectorsCalibration/TimeSlotMetaData.h b/Detectors/Calibration/include/DetectorsCalibration/TimeSlotMetaData.h
new file mode 100644
index 0000000000000..08cf3465e3ff8
--- /dev/null
+++ b/Detectors/Calibration/include/DetectorsCalibration/TimeSlotMetaData.h
@@ -0,0 +1,35 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef DETECTOR_CALIB_TIMESLOTMETADATA_H_
+#define DETECTOR_CALIB_TIMESLOTMETADATA_H_
+
+/// @brief meta-data for the saved content of the timeslot
+
+namespace o2
+{
+namespace calibration
+{
+struct TimeSlotMetaData {
+  using TFType = uint32_t;
+
+  int startRun = -1;
+  int endRun = -1;
+  long startTime = -1;
+  long endTime = -1;
+
+  ClassDefNV(TimeSlotMetaData, 1);
+};
+
+} // namespace calibration
+} // namespace o2
+
+#endif
diff --git a/Detectors/Calibration/src/DetectorsCalibrationLinkDef.h b/Detectors/Calibration/src/DetectorsCalibrationLinkDef.h
index fdb0a8e41f063..b21dc731d6688 100644
--- a/Detectors/Calibration/src/DetectorsCalibrationLinkDef.h
+++ b/Detectors/Calibration/src/DetectorsCalibrationLinkDef.h
@@ -16,10 +16,46 @@
 #pragma link off all functions;
 
 #pragma link C++ class o2::calibration::MeanVertexData + ;
+#pragma link C++ class o2::calibration::TimeSlotMetaData + ;
 #pragma link C++ class o2::calibration::TimeSlot < o2::calibration::MeanVertexData> + ;
-#pragma link C++ class o2::calibration::TimeSlotCalibration < o2::dataformats::PrimaryVertex, o2::calibration::MeanVertexData> + ;
+#pragma link C++ class o2::calibration::TimeSlotCalibration < o2::calibration::MeanVertexData> + ;
 #pragma link C++ class o2::calibration::MeanVertexCalibrator + ;
 #pragma link C++ class o2::calibration::MeanVertexParams + ;
 #pragma link C++ class o2::conf::ConfigurableParamHelper < o2::calibration::MeanVertexParams> + ;
 
+#pragma link C++ struct o2::tof::ITOFC + ;
+#pragma link C++ class o2::calibration::IntegratedClusters < o2::tof::ITOFC> + ;
+#pragma link C++ class o2::calibration::IntegratedClusterCalibrator < o2::tof::ITOFC> + ;
+#pragma link C++ class o2::calibration::TimeSlot < o2::calibration::IntegratedClusters < o2::tof::ITOFC>> + ;
+#pragma link C++ class o2::calibration::TimeSlotCalibration < o2::calibration::IntegratedClusters < o2::tof::ITOFC>> + ;
+
+#pragma link C++ struct o2::fit::IFT0C + ;
+#pragma link C++ class o2::calibration::IntegratedClusters < o2::fit::IFT0C> + ;
+#pragma link C++ class o2::calibration::IntegratedClusterCalibrator < o2::fit::IFT0C> + ;
+#pragma link C++ class o2::calibration::TimeSlot < o2::calibration::IntegratedClusters < o2::fit::IFT0C>> + ;
+#pragma link C++ class o2::calibration::TimeSlotCalibration < o2::calibration::IntegratedClusters < o2::fit::IFT0C>> + ;
+
+#pragma link C++ struct o2::fit::IFV0C + ;
+#pragma link C++ class o2::calibration::IntegratedClusters < o2::fit::IFV0C> + ;
+#pragma link C++ class o2::calibration::IntegratedClusterCalibrator < o2::fit::IFV0C> + ;
+#pragma link C++ class o2::calibration::TimeSlot < o2::calibration::IntegratedClusters < o2::fit::IFV0C>> + ;
+#pragma link C++ class o2::calibration::TimeSlotCalibration < o2::calibration::IntegratedClusters < o2::fit::IFV0C>> + ;
+
+#pragma link C++ struct o2::tpc::ITPCC + ;
+#pragma link C++ class o2::calibration::IntegratedClusters < o2::tpc::ITPCC> + ;
+#pragma link C++ class o2::calibration::IntegratedClusterCalibrator < o2::tpc::ITPCC> + ;
+#pragma link C++ class o2::calibration::TimeSlot < o2::calibration::IntegratedClusters < o2::tpc::ITPCC>> + ;
+#pragma link C++ class o2::calibration::TimeSlotCalibration < o2::calibration::IntegratedClusters < o2::tpc::ITPCC>> + ;
+
+#pragma link C++ struct o2::fit::IFDDC + ;
+#pragma link C++ class o2::calibration::IntegratedClusters < o2::fit::IFDDC> + ;
+#pragma link C++ class o2::calibration::IntegratedClusterCalibrator < o2::fit::IFDDC> + ;
+#pragma link C++ class o2::calibration::TimeSlot < o2::calibration::IntegratedClusters < o2::fit::IFDDC>> + ;
+#pragma link C++ class o2::calibration::TimeSlotCalibration < o2::calibration::IntegratedClusters < o2::fit::IFDDC>> + ;
+
+#pragma link C++ struct o2::tpc::TimeSeries + ;
+#pragma link C++ struct o2::tpc::ITSTPC_Matching + ;
+#pragma link C++ struct o2::tpc::TimeSeriesITSTPC + ;
+#pragma link C++ struct o2::tpc::TimeSeriesdEdx + ;
+
 #endif
diff --git a/Detectors/Calibration/src/IntegratedClusterCalibrator.cxx b/Detectors/Calibration/src/IntegratedClusterCalibrator.cxx
new file mode 100644
index 0000000000000..508d79e9ccab2
--- /dev/null
+++ b/Detectors/Calibration/src/IntegratedClusterCalibrator.cxx
@@ -0,0 +1,205 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file IntegratedClusterCalibrator.cxx
+///
+/// \author Matthias Kleiner 
+/// \date Jan 21, 2023
+
+#include "DetectorsCalibration/IntegratedClusterCalibrator.h"
+#include "CommonUtils/TreeStreamRedirector.h"
+
+namespace o2
+{
+namespace calibration
+{
+
+template 
+void IntegratedClusters::dumpToFile(const char* outFileName, const char* outName) const
+{
+  TFile fOut(outFileName, "RECREATE");
+  fOut.WriteObject(this, outName);
+  fOut.Close();
+}
+
+template 
+void IntegratedClusters::fill(const o2::calibration::TFType tfID, const DataT& currentsContainer)
+{
+  // check if size is same
+  if (!currentsContainer.areSameSize()) {
+    LOGP(warning, "Received data with different size. Returning");
+    return;
+  }
+
+  if (currentsContainer.isEmpty()) {
+    LOGP(info, "Empty data received. Returning");
+    return;
+  }
+
+  // initialize when first data is received
+  const unsigned int entries = currentsContainer.getEntries();
+  if (mInitialize) {
+    initData(entries);
+  }
+
+  // check if all data is already received (this should never happen)
+  if (mRemainingData == 0) {
+    LOGP(warning, "All packages already received. Returning");
+    return;
+  }
+
+  if (entries != mNValuesPerTF) {
+    LOGP(info, "Received data with size {} expected size {} (expected size can be ignored if merging was performed)", entries, mNValuesPerTF);
+  }
+
+  const unsigned int posIndex = (tfID - mTFFirst) * mNValuesPerTF;
+  if (posIndex + entries > mCurrents.getEntries()) {
+    LOGP(warning, "Index for TF {} is larger {} than expected max index {} with {} values per package", tfID, posIndex, mCurrents.getEntries(), mNValuesPerTF);
+    return;
+  }
+
+  // copy data to buffer
+  mCurrents.fill(posIndex, currentsContainer);
+
+  mRemainingData -= entries;
+  LOGP(debug, "Processed TF {} at index {} with first TF {} and {} expected currents per TF. Remaining data {}", tfID, posIndex, mTFFirst, mNValuesPerTF, mRemainingData);
+}
+
+template 
+void IntegratedClusters::merge(const IntegratedClusters* prev)
+{
+  LOGP(info, "Printing last object...");
+  prev->print();
+  LOGP(info, "Printing current object...");
+  print();
+
+  const auto tfMin = std::min(mTFFirst, prev->mTFFirst);
+  const auto tfMax = std::max(mTFLast, prev->mTFLast);
+
+  if (prev->mInitialize) {
+    // if last object was not initialized adjust tf range
+    mTFFirst = tfMin;
+    mTFLast = tfMax;
+    if (!mInitialize) {
+      // current buffer is already initialized just add empty values in front of buffer
+      LOGP(info, "Adding dummy data to front");
+      const unsigned int nDummyValues = mNValuesPerTF * ((prev->mTFLast - prev->mTFFirst) + 1);
+      mCurrents.insert(nDummyValues);
+    }
+    LOGP(info, "Do not merge last object since it was not initialized. Adjusting TF range:");
+    print();
+    return;
+  }
+
+  // check if current object needs to be initialized
+  if (mInitialize) {
+    // init with dummy vector
+    initData(prev->mNValuesPerTF);
+  }
+
+  // creating new temporary object with the range of current and last object
+  const unsigned int totalTFs = (tfMax - tfMin) + 1; // add 1 since first<= t <=last
+  const unsigned int nTotal = mNValuesPerTF * totalTFs;
+  IntegratedClusters dataTmp(tfMin, tfMax);
+  dataTmp.mInitialize = false; // do no initialization as it is done manually
+  dataTmp.mNValuesPerTF = mNValuesPerTF;
+  dataTmp.mRemainingData = -1;
+  dataTmp.mCurrents.resize(nTotal);
+
+  // fill buffered values to tmp objects
+  dataTmp.fill(mTFFirst, mCurrents);
+  dataTmp.fill(prev->mTFFirst, prev->mCurrents);
+
+  dataTmp.mRemainingData = mRemainingData;
+  *this = std::move(dataTmp);
+  LOGP(info, "Merging done", totalTFs);
+  print();
+}
+
+template 
+void IntegratedClusters::initData(const unsigned int valuesPerTF)
+{
+  mInitialize = false;
+  const unsigned int totalTFs = (mTFLast - mTFFirst) + 1; // add 1 since first<= t <=last
+  mNValuesPerTF = valuesPerTF;
+  const unsigned int nTotal = mNValuesPerTF * totalTFs;
+  mCurrents.resize(nTotal);
+  mRemainingData = nTotal;
+  LOGP(info, "Init: Expecting {} packages with {} values per package with {} total values", mRemainingData, mNValuesPerTF, nTotal);
+}
+
+template 
+void IntegratedClusters::dumpToTree(const char* outFileName)
+{
+  o2::utils::TreeStreamRedirector pcstream(outFileName, "RECREATE");
+  pcstream << "tree"
+           << "currents=" << mCurrents
+           << "firstTF=" << mTFFirst
+           << "lastTF=" << mTFLast
+           << "remainingData=" << mRemainingData
+           << "valuesPerTF=" << mNValuesPerTF
+           << "\n";
+}
+
+template 
+void IntegratedClusterCalibrator::initOutput()
+{
+  mIntervals.clear();
+  mCalibs.clear();
+  mTimeIntervals.clear();
+}
+
+template 
+void IntegratedClusterCalibrator::finalizeSlot(o2::calibration::TimeSlot>& slot)
+{
+  const TFType startTF = slot.getTFStart();
+  const TFType endTF = slot.getTFEnd();
+  LOGP(info, "Finalizing slot {} <= TF <= {}", startTF, endTF);
+
+  auto& integratedClusters = *slot.getContainer();
+  integratedClusters.setStartTime(slot.getStartTimeMS());
+  if (mDebug) {
+    integratedClusters.dumpToFile(fmt::format("IntegratedClusters_TF_{}_{}_TS_{}_{}.root", startTF, endTF, slot.getStartTimeMS(), slot.getEndTimeMS()).data());
+  }
+  mCalibs.emplace_back(std::move(integratedClusters).getCurrents());
+
+  mIntervals.emplace_back(startTF, endTF);
+  mTimeIntervals.emplace_back(slot.getStartTimeMS(), slot.getEndTimeMS());
+}
+
+/// Creates new time slot
+template 
+o2::calibration::TimeSlot>& IntegratedClusterCalibrator::emplaceNewSlot(bool front, TFType tstart, TFType tend)
+{
+  auto& cont = this->getSlots();
+  auto& slot = front ? cont.emplace_front(tstart, tend) : cont.emplace_back(tstart, tend);
+  slot.setContainer(std::make_unique>(tstart, tend));
+  return slot;
+}
+
+template class IntegratedClusters;
+template class IntegratedClusterCalibrator;
+
+template class IntegratedClusters;
+template class IntegratedClusterCalibrator;
+
+template class IntegratedClusters;
+template class IntegratedClusterCalibrator;
+
+template class IntegratedClusters;
+template class IntegratedClusterCalibrator;
+
+template class IntegratedClusters;
+template class IntegratedClusterCalibrator;
+
+} // end namespace calibration
+
+} // end namespace o2
diff --git a/Detectors/Calibration/src/MeanVertexCalibrator.cxx b/Detectors/Calibration/src/MeanVertexCalibrator.cxx
index fe1c89a72b2a0..1f6eaf759304d 100644
--- a/Detectors/Calibration/src/MeanVertexCalibrator.cxx
+++ b/Detectors/Calibration/src/MeanVertexCalibrator.cxx
@@ -15,6 +15,7 @@
 #include "CommonUtils/MemFileHelper.h"
 #include "CCDB/CcdbApi.h"
 #include "DetectorsCalibration/Utils.h"
+#include "DetectorsCalibration/MeanVertexParams.h"
 
 namespace o2
 {
@@ -36,17 +37,17 @@ void MeanVertexCalibrator::initOutput()
 }
 
 //_____________________________________________
-void MeanVertexCalibrator::printVector(float* vect, int sizeVect, float minRange, float maxRange, float binWidth)
+void MeanVertexCalibrator::printVector(const float* vect, const MeanVertexCalibrator::HistoParams& hpar)
 {
-  for (int i = 0; i < sizeVect; ++i) {
-    LOG(info) << "i-th bin [" << minRange + i * binWidth << ", " << minRange + (i + 1) * binWidth << "] = " << i << ", content of histogram = " << vect[i];
+  for (int i = 0; i < hpar.nBins; ++i) {
+    LOG(info) << "i-th bin [" << hpar.minRange + i * hpar.binWidth << ", " << hpar.minRange + (i + 1) * hpar.binWidth << "] = " << i << ", content of histogram = " << vect[i];
   }
   LOG(info) << "Printing to be used as a vector holding the content";
-  for (int i = 0; i < sizeVect; ++i) {
+  for (int i = 0; i < hpar.nBins; ++i) {
     LOG(info) << "vect[" << i << "] = " << vect[i] << ";";
   }
   LOG(info) << "Printing to be used to fill a ROOT histogram";
-  for (int i = 0; i < sizeVect; ++i) {
+  for (int i = 0; i < hpar.nBins; ++i) {
     if (vect[i] != 0) {
       LOG(info) << "h->SetBinContent(" << i + 1 << ", " << vect[i] << ");";
     }
@@ -54,30 +55,50 @@ void MeanVertexCalibrator::printVector(float* vect, int sizeVect, float minRange
 }
 
 //_____________________________________________
-void MeanVertexCalibrator::printVector(std::vector& vect, float minRange, float maxRange, float binWidth)
+void MeanVertexCalibrator::printVector(const std::vector& vect, const MeanVertexCalibrator::HistoParams& hpar)
 {
-  printVector(&vect[0], vect.size(), minRange, maxRange, binWidth);
+  printVector(vect.data(), hpar);
 }
 
 //_____________________________________________
-void MeanVertexCalibrator::binVector(std::vector& vectOut, const std::vector& vectIn, int nbins, float min, float max, float binWidthInv)
+MeanVertexCalibrator::HistoParams MeanVertexCalibrator::binVector(std::vector& vectOut, const std::vector& vectIn, o2::calibration::MeanVertexData* c, int dim)
 {
+  const char* dimName[3] = {"X", "Y", "Z"};
   vectOut.clear();
-  vectOut.resize(nbins);
+  // define binning
+  const auto& params = MeanVertexParams::Instance();
+  float mean = c->getMean(dim), rms = c->getRMS(dim);
+  if (rms < params.minSigma[dim]) {
+    LOGP(alarm, "Too small RMS = {} for dimension {} ({} entries), override to {}", rms, dimName[dim], c->entries, params.minSigma[dim]);
+    rms = params.minSigma[dim];
+  }
+  float minD = mean - params.histoNSigma[dim] * rms;
+  float maxD = mean + params.histoNSigma[dim] * rms;
+  int nBins = std::max(1.f, (maxD - minD) / params.histoBinSize[dim]);
+  float binWidth = (maxD - minD) / nBins, binWidthInv = 1. / binWidth;
+  if (mVerbose) {
+    LOGP(info, "Histo for dim:{} with {} entries: mean:{} rms:{} -> {} bins in range {}:{} ", dimName[dim], c->entries, mean, rms, nBins, minD, maxD, nBins);
+  }
+  vectOut.resize(nBins);
   for (int i = 0; i < vectIn.size(); ++i) {
-    if (vectIn[i] < min) {
+    if (vectIn[i] < minD) {
+      continue;
+    }
+    int bin = (vectIn[i] - minD) * binWidthInv;
+    if (bin >= nBins) {
       continue;
     }
-    int bin = (vectIn[i] - min) * binWidthInv;
     vectOut[bin]++;
   }
+  return {nBins, binWidth, minD, maxD};
 }
 
 //_____________________________________________
-void MeanVertexCalibrator::fitMeanVertex(o2::calibration::MeanVertexData* c, MeanVertexObject& mvo)
+bool MeanVertexCalibrator::fitMeanVertex(o2::calibration::MeanVertexData* c, MeanVertexObject& mvo)
 {
   // now we do the fits in slices of Z
   // we fit as soon as we have enough entries in z
+  const auto& params = MeanVertexParams::Instance();
   double fitres;
   // first we order the vector
   std::sort(c->histoVtx.begin(), c->histoVtx.end(), [](std::array a, std::array b) { return b[2] > a[2]; });
@@ -97,91 +118,109 @@ void MeanVertexCalibrator::fitMeanVertex(o2::calibration::MeanVertexData* c, Mea
   std::vector covMatrixY;
   std::vector binnedVect;
   std::vector meanZvect;
-  int startZ = 0;
-  int counter = 0;
-  auto minEntriesPerPoint = std::max((unsigned long int)mMinEntries, c->histoVtx.size() / mNPointsForSlope);
+  int startZ = 0, nBinsOK = 0;
+  auto minEntriesPerPoint = std::max((unsigned long int)params.minEntries, c->histoVtx.size() / params.nPointsForSlope);
   if (mVerbose) {
-    LOG(info) << "Beginning: startZ = " << startZ << " c->histoVtx.size() = " << c->histoVtx.size();
+    LOGP(info, "Beginning: startZ = {} c->histoVtx.size() = {}, will process {} Z slices with {} entries each", startZ, c->histoVtx.size(), params.nPointsForSlope, minEntriesPerPoint);
   }
-  while (startZ <= c->histoVtx.size()) {
+  auto dumpNonEmpty = [&binnedVect](const std::string& msg) {
+    if (!msg.empty()) {
+      LOG(info) << msg;
+    }
+    for (size_t i = 0; i < binnedVect.size(); i++) {
+      if (binnedVect[i]) {
+        LOGP(info, "bin:{} {}", i, binnedVect[i]);
+      }
+    }
+  };
+
+  for (int counter = 0; counter < params.nPointsForSlope; counter++) {
     if (mVerbose) {
       LOG(info) << "Beginning of while: startZ = " << startZ << " c->histoVtx.size() = " << c->histoVtx.size();
     }
     double meanZ = 0;
     int counts = 0;
-    for (int ii = startZ; ii <= c->histoVtx.size(); ++ii) {
-      if (mVerbose) {
-        // LOG(info) << "htmpX.size() = " << htmpX.size() << " ii = " << ii << " c->histoVtx.size() = " << c->histoVtx.size();
-      }
-      if (htmpX.size() < minEntriesPerPoint) {
-        if (mVerbose) {
-          // LOG(info) << "filling X with c->histoVtx[" << ii << "][0] = " << c->histoVtx[ii][0];
-          // LOG(info) << "filling Y with c->histoVtx[" << ii << "][0] = " << c->histoVtx[ii][1];
-        }
+    for (int ii = startZ; ii < c->histoVtx.size(); ++ii) {
+      bool failed = false;
+      if (++counts < minEntriesPerPoint) {
         htmpX.push_back(c->histoVtx[ii][0]);
         htmpY.push_back(c->histoVtx[ii][1]);
         meanZ += c->histoVtx[ii][2];
-        ++counts;
       } else {
+        counts--;
         if (mVerbose) {
-          LOG(info) << "fitting ";
+          LOGP(info, "fitting after collecting {} entries for Z slice {} of {}", htmpX.size(), counter, params.nPointsForSlope);
         }
         // we can fit and restart filling
         // X:
         fitResSlicesX.push_back({});
         covMatrixX.push_back({});
+        auto hparX = binVector(binnedVect, htmpX, c, 0);
         if (mVerbose) {
-          LOG(info) << "Fitting X for counter " << counter << ", will use " << mNBinsX << " bins, from " << -mRangeX << " to " << mRangeX;
+          LOG(info) << "Fitting X for counter " << counter << ", will use " << hparX.nBins << " bins, from " << hparX.minRange << " to " << hparX.maxRange;
           for (int i = 0; i < htmpX.size(); ++i) {
             LOG(info) << "vect[" << i << "] = " << htmpX[i] << ";";
           }
         }
-        binVector(binnedVect, htmpX, mNBinsX, -mRangeX, mRangeX, mBinWidthXInv);
         if (mVerbose) {
           LOG(info) << " Printing output binned vector for X:";
-          printVector(binnedVect, -(mRangeX), mRangeX, mBinWidthX);
+          printVector(binnedVect, hparX);
+        } else if (params.dumpNonEmptyBins) {
+          dumpNonEmpty(fmt::format("X{} nonEmpty bins", counter));
         }
-        fitres = fitGaus(mNBinsX, &binnedVect[0], -(mRangeX), mRangeX, fitResSlicesX.back(), &covMatrixX.back());
-        if (fitres != 10) {
-          LOG(info) << "X, counter " << counter << ": Fit result (z slice [" << c->histoVtx[startZ][2] << ", " << c->histoVtx[ii][2] << "[) => " << fitres << ". Mean = " << fitResSlicesX[counter][1] << " Sigma = " << fitResSlicesX[counter][2] << ", covMatrix = " << covMatrixX[counter](2, 2);
+        fitres = fitGaus(hparX.nBins, binnedVect.data(), hparX.minRange, hparX.maxRange, fitResSlicesX.back(), &covMatrixX.back());
+        if (fitres != -10) {
+          LOG(info) << "X, counter " << counter << ": Fit result (z slice [" << c->histoVtx[startZ][2] << ", " << c->histoVtx[ii][2] << "]) => " << fitres << ". Mean = " << fitResSlicesX[counter][1] << " Sigma = " << fitResSlicesX[counter][2] << ", covMatrix = " << covMatrixX[counter](2, 2) << " entries = " << counts;
         } else {
-          LOG(error) << "X, counter " << counter << ": Fit failed with result = " << fitres;
+          LOG(error) << "X, counter " << counter << ": Fit failed with result = " << fitres << " entries = " << counts;
+          failed = true;
         }
         htmpX.clear();
 
         // Y:
         fitResSlicesY.push_back({});
         covMatrixY.push_back({});
+        binnedVect.clear();
+        auto hparY = binVector(binnedVect, htmpY, c, 1);
         if (mVerbose) {
-          LOG(info) << "Fitting Y for counter " << counter << ", will use " << mNBinsY << " bins, from " << -(mRangeY) << " to " << mRangeY;
+          LOG(info) << "Fitting Y for counter " << counter << ", will use " << hparY.nBins << " bins, from " << hparY.minRange << " to " << hparY.maxRange;
           for (int i = 0; i < htmpY.size(); ++i) {
             LOG(info) << i << " : " << htmpY[i];
           }
         }
-        binnedVect.clear();
-        binVector(binnedVect, htmpY, mNBinsY, -(mRangeY), mRangeY, mBinWidthYInv);
         if (mVerbose) {
           LOG(info) << " Printing output binned vector for Y:";
-          printVector(binnedVect, -(mRangeY), mRangeY, mBinWidthY);
+          printVector(binnedVect, hparY);
+        } else if (params.dumpNonEmptyBins) {
+          dumpNonEmpty(fmt::format("Y{} nonEmpty bins", counter));
         }
-        fitres = fitGaus(mNBinsY, &binnedVect[0], -(mRangeY), mRangeY, fitResSlicesY.back(), &covMatrixY.back());
-        if (fitres != 10) {
-          LOG(info) << "Y, counter " << counter << ": Fit result (z slice [" << c->histoVtx[startZ][2] << ", " << c->histoVtx[ii][2] << "[) => " << fitres << ". Mean = " << fitResSlicesY[counter][1] << " Sigma = " << fitResSlicesY[counter][2] << ", covMatrix = " << covMatrixY[counter](2, 2);
+        fitres = fitGaus(hparY.nBins, binnedVect.data(), hparY.minRange, hparY.maxRange, fitResSlicesY.back(), &covMatrixY.back());
+        if (fitres != -10) {
+          LOG(info) << "Y, counter " << counter << ": Fit result (z slice [" << c->histoVtx[startZ][2] << ", " << c->histoVtx[ii][2] << "]) => " << fitres << ". Mean = " << fitResSlicesY[counter][1] << " Sigma = " << fitResSlicesY[counter][2] << ", covMatrix = " << covMatrixY[counter](2, 2) << " entries = " << counts;
         } else {
-          LOG(error) << "Y, counter " << counter << ": Fit failed with result = " << fitres;
+          LOG(error) << "Y, counter " << counter << ": Fit failed with result = " << fitres << " entries = " << counts;
+          failed = true;
         }
         htmpY.clear();
 
         // Z: let's calculate the mean position
         if (mVerbose) {
-          LOG(info) << "Z, counter " << counter << ": " << meanZ / counts;
+          LOGP(info, "Z, counter {} {} ({}/{})", counter, meanZ / counts, meanZ, counts);
+        }
+
+        if (failed) {
+          fitResSlicesX.pop_back();
+          covMatrixX.pop_back();
+          fitResSlicesY.pop_back();
+          covMatrixY.pop_back();
+        } else {
+          meanZvect.push_back(meanZ / counts);
+          nBinsOK++;
         }
-        ++counter;
-        meanZvect.push_back(meanZ / counts);
+        startZ += counts;
         break;
       }
     }
-    startZ += mMinEntries * counter;
     if (mVerbose) {
       LOG(info) << "End of while: startZ = " << startZ << " c->histoVtx.size() = " << c->histoVtx.size();
     }
@@ -191,28 +230,25 @@ void MeanVertexCalibrator::fitMeanVertex(o2::calibration::MeanVertexData* c, Mea
   for (int ii = 0; ii < c->histoVtx.size(); ++ii) {
     htmpZ.push_back(c->histoVtx[ii][2]);
   }
-  binVector(binnedVect, htmpZ, mNBinsZ, -(mRangeZ), mRangeZ, mBinWidthZInv);
-  fitMeanVertexCoord(2, mNBinsZ, &binnedVect[0], -(mRangeZ), mRangeZ, mvo);
+  auto hparZ = binVector(binnedVect, htmpZ, c, 2);
+  fitMeanVertexCoord(2, binnedVect.data(), hparZ, mvo);
   htmpZ.clear();
   binnedVect.clear();
 
   // now we update the error on x
   double sumX = 0, sumY = 0, weightSumX = 0, weightSumY = 0;
-  for (int iFit = 0; iFit < counter; ++iFit) {
+  for (int iFit = 0; iFit < nBinsOK; ++iFit) {
     if (mVerbose) {
       LOG(info) << "SigmaX = " << fitResSlicesX[iFit][2] << " error = " << covMatrixX[iFit](2, 2);
       LOG(info) << "SigmaY = " << fitResSlicesY[iFit][2] << " error = " << covMatrixY[iFit](2, 2);
     }
-    if (covMatrixX[iFit](2, 2) != 0) {
-      double weightSigma = 1. / covMatrixX[iFit](2, 2); // covMatrix is already an error squared
-      sumX += (fitResSlicesX[iFit][2] * weightSigma);
-      weightSumX += weightSigma;
-    }
-    if (covMatrixY[iFit](2, 2) != 0) {
-      double weightSigma = 1. / covMatrixY[iFit](2, 2); // covMatrix is already an error squared
-      sumY += (fitResSlicesY[iFit][2] * weightSigma);
-      weightSumY += weightSigma;
-    }
+    double weightSigmaX = covMatrixX[iFit](2, 2) > 0 ? 1. / covMatrixX[iFit](2, 2) : 1.; // covMatrix is already an error squared
+    sumX += (fitResSlicesX[iFit][2] * weightSigmaX);
+    weightSumX += weightSigmaX;
+
+    double weightSigmaY = covMatrixY[iFit](2, 2) > 0 ? 1. / covMatrixY[iFit](2, 2) : 1.; // covMatrix is already an error squared
+    sumY += (fitResSlicesY[iFit][2] * weightSigmaY);
+    weightSumY += weightSigmaY;
   }
   if (mVerbose) {
     LOG(info) << "sumX = " << sumX;
@@ -233,13 +269,17 @@ void MeanVertexCalibrator::fitMeanVertex(o2::calibration::MeanVertexData* c, Mea
     LOG(info) << "SigmaX for MeanVertex = " << sigmaX;
     LOG(info) << "SigmaY for MeanVertex = " << sigmaY;
   }
+  if (sigmaX == 0 || sigmaY == 0 || mvo.getSigmaZ() == 0 || nBinsOK < 2) {
+    LOGP(error, "Fit with {} valid slices produced wrong vertex parameters: SigmaX={}, SigmaY={}, SigmaZ={}", nBinsOK, sigmaX, sigmaY, mvo.getSigmaZ());
+    return false;
+  }
   mvo.setSigmaX(sigmaX);
   mvo.setSigmaY(sigmaY);
 
   // now we get the slope for the x-coordinate dependence on z
   TLinearFitter lf(1, "pol1");
   lf.StoreData(kFALSE);
-  for (int i = 0; i < fitResSlicesX.size(); ++i) {
+  for (int i = 0; i < nBinsOK; ++i) {
     if (mVerbose) {
       LOG(info) << "Adding point " << i << ": zvtx = " << meanZvect[i] << " xvtx = " << fitResSlicesX[i][2];
     }
@@ -252,7 +292,7 @@ void MeanVertexCalibrator::fitMeanVertex(o2::calibration::MeanVertexData* c, Mea
   lf.ClearPoints();
 
   // now slope for the y-coordinate dependence on z
-  for (int i = 0; i < fitResSlicesX.size(); ++i) {
+  for (int i = 0; i < nBinsOK; ++i) {
     if (mVerbose) {
       LOG(info) << "Adding point " << i << ": zvtx = " << meanZvect[i] << " yvtx = " << fitResSlicesY[i][2];
     }
@@ -266,25 +306,20 @@ void MeanVertexCalibrator::fitMeanVertex(o2::calibration::MeanVertexData* c, Mea
     LOG(info) << "slope X = " << slopeX;
     LOG(info) << "slope Y = " << slopeY;
   }
+  LOG(info) << "Fitted meanVertex: " << mvo.asString();
+  return true;
 }
 //_____________________________________________
-void MeanVertexCalibrator::fitMeanVertexCoord(int icoord, int nbins, float* array, float minRange, float maxRange, MeanVertexObject& mvo)
+void MeanVertexCalibrator::fitMeanVertexCoord(int icoord, const float* array, const MeanVertexCalibrator::HistoParams& hpar, MeanVertexObject& mvo)
 {
   // fit mean vertex coordinate icoord
   std::vector fitValues;
   float binWidth = 0;
   if (mVerbose) {
     LOG(info) << "**** Printing content of MeanVertex object for coordinate " << icoord;
-    if (icoord == 0) {
-      binWidth = mBinWidthX;
-    } else if (icoord == 1) {
-      binWidth = mBinWidthY;
-    } else {
-      binWidth = mBinWidthZ;
-    }
-    printVector(array, nbins, minRange, maxRange, binWidth);
+    printVector(array, hpar);
   }
-  double fitres = fitGaus(nbins, array, minRange, maxRange, fitValues);
+  double fitres = fitGaus(hpar.nBins, array, hpar.minRange, hpar.maxRange, fitValues);
   if (fitres != -4) {
     LOG(info) << "coordinate " << icoord << ": Fit result of full statistics => " << fitres << ". Mean = " << fitValues[1] << " Sigma = " << fitValues[2];
   } else {
@@ -310,28 +345,32 @@ void MeanVertexCalibrator::fitMeanVertexCoord(int icoord, int nbins, float* arra
 void MeanVertexCalibrator::finalizeSlot(Slot& slot)
 {
   // Extract results for the single slot
+  const auto& params = MeanVertexParams::Instance();
   o2::calibration::MeanVertexData* c = slot.getContainer();
   LOG(info) << "Finalize slot " << slot.getTFStart() << " <= TF <= " << slot.getTFEnd() << " with "
             << c->getEntries() << " entries";
-  mTmpMVobjDqTime.emplace_back(slot.getStartTimeMS(), slot.getEndTimeMS());
   MeanVertexObject mvo;
   // fitting
-  fitMeanVertex(c, mvo);
+  if (!fitMeanVertex(c, mvo)) {
+    return;
+  }
+  mTmpMVobjDqTime.emplace_back(slot.getStartTimeMS(), slot.getEndTimeMS());
   // now we add the object to the deque
   mTmpMVobjDq.push_back(std::move(mvo));
 
   // moving average
   doSimpleMovingAverage(mTmpMVobjDq, mSMAMVobj);
 
-  if (mTmpMVobjDqTime.size() > mSMAslots) {
+  if (mTmpMVobjDqTime.size() > params.nSlots4SMA) {
     mTmpMVobjDqTime.pop_front();
   }
-  long startValidity = (mTmpMVobjDqTime.front().getMin() + mTmpMVobjDqTime.back().getMax()) / 2;
+  long offset = (slot.getEndTimeMS() - slot.getStartTimeMS()) / 2;
+  long startValidity = (mTmpMVobjDqTime.front().getMin() + mTmpMVobjDqTime.back().getMax()) / 2 - offset;
   LOG(info) << "start validity = " << startValidity;
   std::map md;
   auto clName = o2::utils::MemFileHelper::getClassName(mSMAMVobj);
   auto flName = o2::ccdb::CcdbApi::generateFileName(clName);
-  mInfoVector.emplace_back("GLO/Calib/MeanVertex", clName, flName, md, startValidity - 10 * o2::ccdb::CcdbObjectInfo::SECOND, startValidity + o2::ccdb::CcdbObjectInfo::MONTH);
+  mInfoVector.emplace_back("GLO/Calib/MeanVertex", clName, flName, md, startValidity - 10 * o2::ccdb::CcdbObjectInfo::SECOND, startValidity + offset + 10 * o2::ccdb::CcdbObjectInfo::SECOND);
   mMeanVertexVector.emplace_back(mSMAMVobj);
   if (mVerbose) {
     LOG(info) << "Printing MeanVertex Object:";
@@ -346,8 +385,7 @@ void MeanVertexCalibrator::doSimpleMovingAverage(std::deque& dq, float& s
 {
 
   // doing simple moving average
-
-  if (dq.size() <= mSMAslots) {
+  if (dq.size() <= MeanVertexParams::Instance().nSlots4SMA) {
     sma = std::accumulate(dq.begin(), dq.end(), 0.0) / dq.size();
     //avg = (avg * (vect.size() - 1) + vect.back()) / vect.size();
     return;
@@ -355,7 +393,7 @@ void MeanVertexCalibrator::doSimpleMovingAverage(std::deque& dq, float& s
 
   // if the vector has size > mSMAslots, we calculate the SMA, and then we drop 1 element
   // (note that it can have a size > mSMAslots only of 1 element!)
-  sma += (dq[dq.size() - 1] - dq[0]) / mSMAslots;
+  sma += (dq[dq.size() - 1] - dq[0]) / MeanVertexParams::Instance().nSlots4SMA;
   dq.pop_front();
 
   return;
@@ -366,8 +404,8 @@ void MeanVertexCalibrator::doSimpleMovingAverage(std::deque& dq, MVObj
 {
 
   // doing simple moving average
-
-  if (dq.size() <= mSMAslots) {
+  const auto& params = MeanVertexParams::Instance();
+  if (dq.size() <= params.nSlots4SMA) {
     sma.setX((sma.getX() * (dq.size() - 1) + dq.back().getX()) / dq.size());
     sma.setY((sma.getY() * (dq.size() - 1) + dq.back().getY()) / dq.size());
     sma.setZ((sma.getZ() * (dq.size() - 1) + dq.back().getZ()) / dq.size());
@@ -385,14 +423,14 @@ void MeanVertexCalibrator::doSimpleMovingAverage(std::deque& dq, MVObj
 
   // if the vector has size > mSMAslots, we calculate the SMA, and then we drop 1 element
   // (note that it can have a size > mSMAslots only of 1 element!)
-  sma.setX(sma.getX() + (dq[dq.size() - 1].getX() - dq[0].getX()) / mSMAslots);
-  sma.setY(sma.getY() + (dq[dq.size() - 1].getY() - dq[0].getY()) / mSMAslots);
-  sma.setZ(sma.getZ() + (dq[dq.size() - 1].getZ() - dq[0].getZ()) / mSMAslots);
-  sma.setSigmaX(sma.getSigmaX() + (dq[dq.size() - 1].getSigmaX() - dq[0].getSigmaX()) / mSMAslots);
-  sma.setSigmaY(sma.getSigmaY() + (dq[dq.size() - 1].getSigmaY() - dq[0].getSigmaY()) / mSMAslots);
-  sma.setSigmaZ(sma.getSigmaZ() + (dq[dq.size() - 1].getSigmaZ() - dq[0].getSigmaZ()) / mSMAslots);
-  sma.setSlopeX(sma.getSlopeX() + (dq[dq.size() - 1].getSlopeX() - dq[0].getSlopeX()) / mSMAslots);
-  sma.setSlopeY(sma.getSlopeY() + (dq[dq.size() - 1].getSlopeY() - dq[0].getSlopeY()) / mSMAslots);
+  sma.setX(sma.getX() + (dq[dq.size() - 1].getX() - dq[0].getX()) / params.nSlots4SMA);
+  sma.setY(sma.getY() + (dq[dq.size() - 1].getY() - dq[0].getY()) / params.nSlots4SMA);
+  sma.setZ(sma.getZ() + (dq[dq.size() - 1].getZ() - dq[0].getZ()) / params.nSlots4SMA);
+  sma.setSigmaX(sma.getSigmaX() + (dq[dq.size() - 1].getSigmaX() - dq[0].getSigmaX()) / params.nSlots4SMA);
+  sma.setSigmaY(sma.getSigmaY() + (dq[dq.size() - 1].getSigmaY() - dq[0].getSigmaY()) / params.nSlots4SMA);
+  sma.setSigmaZ(sma.getSigmaZ() + (dq[dq.size() - 1].getSigmaZ() - dq[0].getSigmaZ()) / params.nSlots4SMA);
+  sma.setSlopeX(sma.getSlopeX() + (dq[dq.size() - 1].getSlopeX() - dq[0].getSlopeX()) / params.nSlots4SMA);
+  sma.setSlopeY(sma.getSlopeY() + (dq[dq.size() - 1].getSlopeY() - dq[0].getSlopeY()) / params.nSlots4SMA);
 
   dq.pop_front();
 
@@ -409,10 +447,19 @@ Slot& MeanVertexCalibrator::emplaceNewSlot(bool front, TFType tstart, TFType ten
 {
   auto& cont = getSlots();
   auto& slot = front ? cont.emplace_front(tstart, tend) : cont.emplace_back(tstart, tend);
-  // slot.setContainer(std::make_unique(mNBinsX, mRangeX, mNBinsY, mRangeY, mNBinsZ, mRangeZ));
   slot.setContainer(std::make_unique());
   return slot;
 }
 
+bool MeanVertexCalibrator::hasEnoughData(const Slot& slot) const
+{
+  auto minReq = MeanVertexParams::Instance().minEntries * MeanVertexParams::Instance().nPointsForSlope;
+  if (mVerbose) {
+    LOG(info) << "container * " << (void*)slot.getContainer();
+    LOG(info) << "container entries = " << slot.getContainer()->entries << ", minEntries = " << minReq;
+  }
+  return slot.getContainer()->entries >= minReq;
+}
+
 } // end namespace calibration
 } // end namespace o2
diff --git a/Detectors/Calibration/src/MeanVertexData.cxx b/Detectors/Calibration/src/MeanVertexData.cxx
index 1ad4bbceabbcc..443770afe7cf0 100644
--- a/Detectors/Calibration/src/MeanVertexData.cxx
+++ b/Detectors/Calibration/src/MeanVertexData.cxx
@@ -14,6 +14,7 @@
 #include "CommonUtils/MemFileHelper.h"
 #include "CCDB/CcdbApi.h"
 #include "DetectorsCalibration/Utils.h"
+#include 
 
 using namespace o2::calibration;
 
@@ -26,12 +27,6 @@ using Slot = o2::calibration::TimeSlot;
 using PVertex = o2::dataformats::PrimaryVertex;
 using clbUtils = o2::calibration::Utils;
 
-//_____________________________________________
-MeanVertexData::MeanVertexData()
-{
-  LOG(info) << "Default c-tor, not to be used";
-}
-
 //_____________________________________________
 void MeanVertexData::print() const
 {
@@ -43,17 +38,20 @@ void MeanVertexData::fill(const gsl::span data)
 {
   // fill container
 
-  LOG(info) << "input size = " << data.size();
+  LOG(debug) << "input size = " << data.size();
   for (int i = data.size(); i--;) {
     // filling the histogram in binned mode
-    auto x = data[i].getX();
-    auto y = data[i].getY();
-    auto z = data[i].getZ();
+    std::array xyz{data[i].getX(), data[i].getY(), data[i].getZ()};
+    auto entries1 = entries + 1;
+    for (int j = 0; j < 3; j++) {
+      means[j] = (means[j] * entries + xyz[j]) / entries1;
+      meanSquares[j] = (meanSquares[j] * entries + xyz[j] * xyz[j]) / entries1;
+    }
     if (mVerbose) {
-      LOG(info) << "i = " << i << " --> x = " << x << ", y = " << y << ", z = " << z;
+      LOG(info) << "i = " << i << " --> x = " << xyz[0] << ", y = " << xyz[1] << ", z = " << xyz[2];
     }
-    entries++;
-    histoVtx.push_back({x, y, z});
+    entries = entries + 1;
+    histoVtx.push_back(xyz);
   }
 }
 
@@ -61,7 +59,17 @@ void MeanVertexData::fill(const gsl::span data)
 void MeanVertexData::subtract(const MeanVertexData* prev)
 {
   // remove entries from prev
-
+  int totEntries = entries - prev->entries;
+  if (totEntries > 0) {
+    for (int i = 0; i < 3; i++) {
+      means[i] = (means[i] * entries - prev->means[i] * prev->entries) / totEntries;
+      meanSquares[i] = (meanSquares[i] * entries - prev->meanSquares[i] * prev->entries) / totEntries;
+    }
+  } else {
+    for (int i = 0; i < 3; i++) {
+      means[i] = meanSquares[i] = 0.;
+    }
+  }
   histoVtx.erase(histoVtx.begin(), histoVtx.begin() + prev->entries);
   entries -= prev->entries;
 }
@@ -71,7 +79,20 @@ void MeanVertexData::merge(const MeanVertexData* prev)
 {
   // merge data of 2 slots
   histoVtx.insert(histoVtx.end(), prev->histoVtx.begin(), prev->histoVtx.end());
-  entries += prev->entries;
+  auto totEntries = entries + prev->entries;
+  if (totEntries) {
+    for (int i = 0; i < 3; i++) {
+      means[i] = (means[i] * entries + prev->means[i] * prev->entries) / totEntries;
+      meanSquares[i] = (meanSquares[i] * entries + prev->meanSquares[i] * prev->entries) / totEntries;
+    }
+  }
+  entries = totEntries;
+}
+
+double MeanVertexData::getRMS(int i) const
+{
+  double rms2 = meanSquares[i] - means[i] * means[i];
+  return rms2 > 0. ? std::sqrt(rms2) : 0;
 }
 
 } // end namespace calibration
diff --git a/Detectors/Calibration/testMacros/CMakeLists.txt b/Detectors/Calibration/testMacros/CMakeLists.txt
index 2d9cec773a80d..f2fdb6687d4e2 100644
--- a/Detectors/Calibration/testMacros/CMakeLists.txt
+++ b/Detectors/Calibration/testMacros/CMakeLists.txt
@@ -14,10 +14,10 @@ install(FILES populateCCDB.C
               DESTINATION share/macro/)
 
 install(FILES cdbSizeV0.txt
-        DESTINATION share/Detectors/Calibration/data)	    
+        DESTINATION share/Detectors/Calibration/data)
 
 install(FILES runEPNsimulation.sh
-        DESTINATION share/Detectors/Calibration/scripts)	    
+        DESTINATION share/Detectors/Calibration/scripts)
 
 o2_add_test_root_macro(populateCCDB.C
                        PUBLIC_LINK_LIBRARIES O2::CCDB
@@ -30,13 +30,21 @@ o2_add_test_root_macro(retrieveFromCCDB.C
 o2_add_executable(populate-ccdb
                   COMPONENT_NAME calibration
                   SOURCES populateCCDB.cxx
-		  PUBLIC_LINK_LIBRARIES ROOT::MathCore
-		  O2::Framework
-		  O2::CCDB)
+      PUBLIC_LINK_LIBRARIES ROOT::MathCore
+      O2::Framework
+      O2::CCDB)
 
 o2_add_executable(retrieve-from-ccdb
                   COMPONENT_NAME calibration
                   SOURCES retrieveFromCCDB.cxx
-		  PUBLIC_LINK_LIBRARIES ROOT::MathCore
-		  O2::Framework
-		  O2::CCDB)
+      PUBLIC_LINK_LIBRARIES ROOT::MathCore
+      O2::Framework
+      O2::CCDB)
+
+o2_add_executable(get-run-parameters
+                  COMPONENT_NAME calibration
+                  SOURCES getRunParameters.cxx
+      PUBLIC_LINK_LIBRARIES
+      O2::DataFormatsCTP
+      O2::CommonDataFormat
+      O2::CCDB)
diff --git a/Detectors/Calibration/testMacros/getRunParameters.cxx b/Detectors/Calibration/testMacros/getRunParameters.cxx
new file mode 100644
index 0000000000000..c06926511c95d
--- /dev/null
+++ b/Detectors/Calibration/testMacros/getRunParameters.cxx
@@ -0,0 +1,209 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+// executable to get the interaction rate and duration of a run from CCDB
+
+#include 
+#include 
+#include "CCDB/BasicCCDBManager.h"
+#include "CommonDataFormat/InteractionRecord.h"
+#include "CCDB/CcdbApi.h"
+#include "CCDB/BasicCCDBManager.h"
+#include "DataFormatsCTP/Scalers.h"
+#include "DataFormatsCTP/Configuration.h"
+#include "DataFormatsParameters/GRPMagField.h"
+#include "DataFormatsParameters/GRPECSObject.h"
+#include "CommonTypes/Units.h"
+#include 
+
+using namespace o2::ctp;
+namespace bpo = boost::program_options;
+
+const double orbitDuration = 88.924596234; // us
+
+void writeIRtoFile(float ir)
+{
+
+  FILE* fptr = fopen("IR.txt", "w");
+  if (fptr == nullptr) {
+    LOGP(fatal, "ERROR: Could not open file to write IR!");
+    return;
+  }
+  fprintf(fptr, "%.2f", ir);
+  fclose(fptr);
+}
+
+void writeDurationToFile(long duration)
+{
+
+  FILE* fptr = fopen("Duration.txt", "w");
+  if (fptr == nullptr) {
+    LOGP(fatal, "ERROR: Could not open file to write duration!");
+    return;
+  }
+  fprintf(fptr, "%ld", duration);
+  fclose(fptr);
+}
+
+void writeBFieldToFile(float b)
+{
+
+  FILE* fptr = fopen("BField.txt", "w");
+  if (fptr == nullptr) {
+    LOGP(fatal, "ERROR: Could not open file to write B field!");
+    return;
+  }
+  fprintf(fptr, "%.2f", b);
+  fclose(fptr);
+}
+
+void writeDetListToFile(std::string detList)
+{
+  FILE* fptr = fopen("DetList.txt", "w");
+  if (fptr == nullptr) {
+    LOGP(fatal, "ERROR: Could not open file to write detector list!");
+    return;
+  }
+  fprintf(fptr, "%s", detList.c_str());
+  fclose(fptr);
+}
+
+bool initOptionsAndParse(bpo::options_description& options, int argc, char* argv[], bpo::variables_map& vm)
+{
+  options.add_options()(
+    "run,r", bpo::value()->default_value(8), "Run number to inspect")(
+    "enable-debug,d", bpo::value()->default_value(false)->implicit_value(true), "Enable debug logs")(
+    "help,h", "Produce help message.");
+
+  try {
+    bpo::store(parse_command_line(argc, argv, options), vm);
+
+    // help
+    if (vm.count("help")) {
+      std::cout << options << std::endl;
+      return false;
+    }
+
+    bpo::notify(vm);
+  } catch (const bpo::error& e) {
+    std::cerr << e.what() << "\n\n";
+    std::cerr << "Error parsing command line arguments; Available options:\n";
+
+    std::cerr << options << std::endl;
+    return false;
+  }
+  return true;
+}
+
+int main(int argc, char* argv[])
+{
+  bpo::options_description options("Allowed options");
+  bpo::variables_map vm;
+  if (!initOptionsAndParse(options, argc, argv, vm)) {
+    return -1;
+  }
+
+  auto run = vm["run"].as();
+  auto debug = vm["enable-debug"].as();
+
+  float ir = 0.f;
+  long duration = 0;
+  // duration as O2end - O2start:
+  auto& ccdb_inst = o2::ccdb::BasicCCDBManager::instance();
+  ccdb_inst.setURL("http://alice-ccdb.cern.ch");
+  std::pair run_times = ccdb_inst.getRunDuration(run);
+  long run_O2duration = long(run_times.second - run_times.first);
+  // access SOR and EOR timestamps
+  int64_t tsSOR = run_times.first;  // ms
+  int64_t tsEOR = run_times.second; // ms
+  LOGP(info, "tsSOR = {} ms, tsEOR = {} ms", tsSOR, tsEOR);
+
+  // first we get the B field
+  LOGP(info, "Getting B field");
+  std::map metadata;
+  ccdb_inst.setFatalWhenNull(true);
+  o2::parameters::GRPMagField* magField = ccdb_inst.getSpecific("GLO/Config/GRPMagField", tsSOR, metadata);
+  o2::units::Current_t magFieldL3Curr = magField->getL3Current();
+  LOGP(info, "run {}: B field = {}", run, magFieldL3Curr);
+  writeBFieldToFile((float)magFieldL3Curr);
+
+  // getting the detector list
+  LOGP(info, "Getting detector participating in the run");
+  std::map metadataRun;
+  metadataRun["runNumber"] = std::to_string(run);
+  o2::parameters::GRPECSObject* ecsObj = ccdb_inst.getSpecific("GLO/Config/GRPECS", tsSOR, metadataRun);
+  std::string dets = "";
+  for (int i = o2::detectors::DetID::First; i < o2::detectors::DetID::nDetectors; ++i) {
+    if (ecsObj->isDetReadOut(i)) {
+      dets = dets + o2::detectors::DetID::getName(i) + " ";
+    }
+  }
+  LOGP(info, "run {}: detectors in readout = {}", run, dets);
+  writeDetListToFile(dets);
+
+  LOGP(info, "Checking IR and duration");
+
+  // Extract CTP info
+  ccdb_inst.setFatalWhenNull(false);
+  metadata["runNumber"] = Form("%d", run);
+  o2::ctp::CTPRunScalers* scl = ccdb_inst.getSpecific("CTP/Calib/Scalers", tsSOR, metadata);
+  if (!scl) {
+    LOGP(info, "CTP/Calib/Scalers object does not exist in production CCDB, trying test CCDB");
+    ccdb_inst.setURL("http://ccdb-test.cern.ch:8080");
+    scl = ccdb_inst.getSpecific("CTP/Calib/Scalers", tsSOR, metadata);
+    if (!scl) {
+      LOGP(info, "Cannot get IR for run {} neither from production nor test CCDB, writing -1.f", run);
+      LOGP(info, "In addition, the duration for these runs is O2end - O2start: if the run was short, this might overestimate the duration");
+      ir = -1.f;
+      writeIRtoFile(ir);
+      writeDurationToFile(run_O2duration);
+      return 0;
+    }
+  }
+
+  scl->convertRawToO2();
+  std::vector mScalerRecordO2 = scl->getScalerRecordO2();
+  int n = mScalerRecordO2.size();
+  if (n != 0) {
+    std::int64_t totScalers = 0;
+    std::vector vOrbit;
+    std::vector vScaler;
+    int i = 0;
+    for (auto& record : mScalerRecordO2) {
+      if (debug) {
+        record.printStream(std::cout);
+      }
+      std::vector& scalers = record.scalers;
+      o2::InteractionRecord& intRecord = record.intRecord;
+      vOrbit.push_back(intRecord.orbit);
+      if (debug) {
+        LOGP(info, "{} orbit = {} scalers = {}", i, intRecord.orbit, scalers[0].lmBefore);
+      }
+      vScaler.push_back(scalers[0].lmBefore); // use scalers for class 0 (usually TVX). TODO: extract info on class id from trigger config
+      totScalers += scalers[0].lmBefore;
+      ++i;
+    }
+
+    duration = std::round((vOrbit.back() - vOrbit.front()) * orbitDuration * 1e-6); // s
+    ir = float(vScaler.back() - vScaler.front()) / duration;
+    LOGP(info, "run {}: orbit.front = {} orbit.back = {} duration = {} s scalers = {} IR = {} Hz", run, vOrbit.front(), vOrbit.back(), duration, vScaler.back() - vScaler.front(), ir);
+  }
+
+  if (ir < 100000) {
+    LOGP(info, "IR < 100 kHz");
+  } else {
+    LOGP(info, "IR > 100 kHz");
+  }
+  writeIRtoFile(ir);
+  writeDurationToFile(duration);
+
+  return 0;
+}
diff --git a/Detectors/Calibration/workflow/CCDBPopulatorSpec.h b/Detectors/Calibration/workflow/CCDBPopulatorSpec.h
index e5b0be673855d..990d70e17e050 100644
--- a/Detectors/Calibration/workflow/CCDBPopulatorSpec.h
+++ b/Detectors/Calibration/workflow/CCDBPopulatorSpec.h
@@ -29,6 +29,11 @@
 #include "CCDB/CcdbObjectInfo.h"
 #include "CCDB/CCDBTimeStampUtils.h"
 #include "CommonUtils/NameConf.h"
+#include 
+#include 
+#include 
+#include 
+#include 
 
 namespace o2
 {
@@ -37,93 +42,215 @@ namespace calibration
 
 class CCDBPopulator : public o2::framework::Task
 {
+ public:
   using CcdbObjectInfo = o2::ccdb::CcdbObjectInfo;
   using CcdbApi = o2::ccdb::CcdbApi;
 
- public:
-  void init(o2::framework::InitContext& ic) final
-  {
-    mCCDBpath = ic.options().get("ccdb-path");
-    mSSpecMin = ic.options().get("sspec-min");
-    mSSpecMax = ic.options().get("sspec-max");
-    mFatalOnFailure = ic.options().get("fatal-on-failure");
-    mAPI.init(mCCDBpath);
-  }
+  using BLOB = std::vector;
+  using TBLOB = std::pair; // pair of creation time and object to upload
+  using OBJCACHE = std::map;
+
+  void init(o2::framework::InitContext& ic) final;
+  void run(o2::framework::ProcessingContext& pc) final;
+  void endOfStream(o2::framework::EndOfStreamContext& ec) final;
+  void stop() final;
+
+  void checkCache(long delay);
+  void doUpload(const CcdbObjectInfo& wrp, const gsl::span& pld, bool cached = false);
+  void logAsNeeded(long nowMS, const std::string& path, std::string& msg);
+
+ private:
+  CcdbApi mAPI;
+  long mThrottlingDelayMS = 0;  // LOG(important) at most once per this period for given path
+  int mOrderingLatencyMS = -1;  // if >0, bufferize and upload if no object with smaller SOV was received in this time interval in ms
+  bool mFatalOnFailure = true;  // produce fatal on failed upload
+  bool mValidateUpload = false; // validate upload by querying its headers
+  bool mEnded = false;
+  std::unordered_map> mThrottling;
+  std::unordered_map mOrdCache;
+  std::int64_t mSSpecMin = -1;                             // min subspec to accept
+  std::int64_t mSSpecMax = -1;                             // max subspec to accept
+  std::string mCCDBpath = "http://ccdb-test.cern.ch:8080"; // CCDB path
+  int mRunNoFromDH = 0;
+  std::string mRunNoStr = {};
+};
+
+void CCDBPopulator::init(o2::framework::InitContext& ic)
+{
+  mCCDBpath = ic.options().get("ccdb-path");
+  mSSpecMin = ic.options().get("sspec-min");
+  mSSpecMax = ic.options().get("sspec-max");
+  mFatalOnFailure = ic.options().get("fatal-on-failure");
+  mValidateUpload = ic.options().get("validate-upload");
+  mThrottlingDelayMS = ic.options().get("throttling-delay");
+  mOrderingLatencyMS = ic.options().get("ordering-latency");
+  mAPI.init(mCCDBpath);
+}
 
-  void run(o2::framework::ProcessingContext& pc) final
-  {
-    int nSlots = pc.inputs().getNofParts(0);
-    if (nSlots != pc.inputs().getNofParts(1)) {
-      LOGP(alarm, "Number of slots={} in part0 is different from that ({}) in part1", nSlots, pc.inputs().getNofParts(1));
-      return;
-    } else if (nSlots == 0) {
-      LOG(alarm) << "0 slots received";
-      return;
+void CCDBPopulator::run(o2::framework::ProcessingContext& pc)
+{
+  int nSlots = pc.inputs().getNofParts(0);
+  if (nSlots != pc.inputs().getNofParts(1)) {
+    LOGP(alarm, "Number of slots={} in part0 is different from that ({}) in part1", nSlots, pc.inputs().getNofParts(1));
+    return;
+  } else if (nSlots == 0) {
+    LOG(alarm) << "0 slots received";
+    return;
+  }
+  mRunNoFromDH = pc.services().get().runNumber;
+  if (mRunNoFromDH > 0) {
+    mRunNoStr = std::to_string(mRunNoFromDH);
+  }
+  auto nowMS = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count();
+  for (int isl = 0; isl < nSlots; isl++) {
+    auto refWrp = pc.inputs().get("clbWrapper", isl);
+    auto refPld = pc.inputs().get("clbPayload", isl);
+    if (!o2::framework::DataRefUtils::isValid(refWrp)) {
+      LOGP(alarm, "Wrapper is not valid for slot {}", isl);
+      continue;
     }
-    auto runNoFromDH = pc.services().get().runNumber;
-    std::string runNoStr;
-    if (runNoFromDH > 0) {
-      runNoStr = std::to_string(runNoFromDH);
+    if (!o2::framework::DataRefUtils::isValid(refPld)) {
+      LOGP(alarm, "Payload is not valid for slot {}", isl);
+      continue;
     }
-    std::map metadata;
-    for (int isl = 0; isl < nSlots; isl++) {
-      auto refWrp = pc.inputs().get("clbWrapper", isl);
-      auto refPld = pc.inputs().get("clbPayload", isl);
-      if (!o2::framework::DataRefUtils::isValid(refWrp)) {
-        LOGP(info, "Wrapper is not valid for slot {}", isl);
-        continue;
-      }
-      if (!o2::framework::DataRefUtils::isValid(refPld)) {
-        LOGP(info, "Payload is not valid for slot {}", isl);
-        continue;
-      }
-      if (mSSpecMin >= 0 && mSSpecMin <= mSSpecMax) { // there is a selection
-        auto ss = std::int64_t(o2::framework::DataRefUtils::getHeader(refWrp)->subSpecification);
-        if (ss < mSSpecMin || ss > mSSpecMax) {
-          continue;
-        }
-      }
-      const auto wrp = pc.inputs().get(refWrp);
-      const auto pld = pc.inputs().get>(refPld); // this is actually an image of TMemFile
-      if (!wrp) {
-        LOGP(alarm, "No CcdbObjectInfo info for {} at slot {}",
-             o2::framework::DataRefUtils::getHeader(refWrp)->dataDescription.as(), isl);
+    if (mSSpecMin >= 0 && mSSpecMin <= mSSpecMax) { // there is a selection
+      auto ss = std::int64_t(o2::framework::DataRefUtils::getHeader(refWrp)->subSpecification);
+      if (ss < mSSpecMin || ss > mSSpecMax) {
         continue;
       }
-      const auto* md = &wrp->getMetaData();
-      if (runNoFromDH > 0 && md->find(o2::base::NameConf::CCDBRunTag.data()) == md->end()) { // if valid run number is provided and it is not filled in the metadata, add it to the clone
-        metadata = *md;                                                                      // clone since the md from the message is const
-        metadata[o2::base::NameConf::CCDBRunTag.data()] = runNoStr;
-        md = &metadata;
+    }
+    const auto wrp = pc.inputs().get(refWrp);
+    const auto pld = pc.inputs().get>(refPld); // this is actually an image of TMemFile
+    if (!wrp) {
+      LOGP(alarm, "No CcdbObjectInfo info for {} at slot {}",
+           o2::framework::DataRefUtils::getHeader(refWrp)->dataDescription.as(), isl);
+      continue;
+    }
+    if (mOrderingLatencyMS <= 0) { // ordering is not requested
+      doUpload(*wrp, pld);
+    } else {
+      auto& pathCache = mOrdCache[wrp->getPath()];
+      auto stt = pathCache.emplace(*wrp, std::make_pair(nowMS, std::vector(pld.size())));
+      if (stt.second) { // insertion success
+        stt.first->second.second.assign(pld.begin(), pld.end());
+        std::string msg = fmt::format("Bufferizing for ordering ccdb object {}/{} of size {} valid for {} : {}",
+                                      wrp->getPath(), wrp->getFileName(), pld.size(), wrp->getStartValidityTimestamp(), wrp->getEndValidityTimestamp());
+        logAsNeeded(nowMS, wrp->getPath(), msg);
+      } else {
+        bool v = stt.first != pathCache.end();
+        LOGP(error, "failed to bufferize a {} object with SOV={}/EOV={} received at {}, conflicting with previously bufferized one SOV={}/EOV={} received at {}",
+             wrp->getPath(), wrp->getStartValidityTimestamp(), wrp->getEndValidityTimestamp(), nowMS,
+             v ? std::to_string(stt.first->first.getStartValidityTimestamp()) : std::string{"N/A"},
+             v ? std::to_string(stt.first->first.getEndValidityTimestamp()) : std::string{"N/A"},
+             v ? std::to_string(stt.first->second.first) : std::string{"N/A"});
       }
+    }
+  }
+  if (mOrderingLatencyMS > 0) {
+    checkCache(mOrderingLatencyMS);
+  }
+}
 
-      LOG(info) << "Storing in ccdb " << wrp->getPath() << "/" << wrp->getFileName() << " of size " << pld.size()
-                << " Valid for " << wrp->getStartValidityTimestamp() << " : " << wrp->getEndValidityTimestamp();
-      int res = mAPI.storeAsBinaryFile(&pld[0], pld.size(), wrp->getFileName(), wrp->getObjectType(), wrp->getPath(),
-                                       *md, wrp->getStartValidityTimestamp(), wrp->getEndValidityTimestamp());
-      if (res && mFatalOnFailure) {
-        LOGP(fatal, "failed on uploading to {} / {}", mAPI.getURL(), wrp->getPath());
+void CCDBPopulator::checkCache(long delay)
+{
+  // check if some entries in cache are ripe enough to upload
+  auto nowMS = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count();
+  for (auto& pathCache : mOrdCache) { // loop over paths
+    if (delay < 0 && pathCache.second.size()) {
+      LOGP(important, "Uploading {} cached objects for path {}", pathCache.second.size(), pathCache.first);
+    }
+    for (auto it = pathCache.second.begin(); it != pathCache.second.end();) { // loop over objects of the path
+      if (nowMS - it->second.first > delay) {
+        doUpload(it->first, {it->second.second.data(), it->second.second.size()}, true);
+        it = pathCache.second.erase(it);
+      } else {
+        break;
       }
+    }
+  }
+}
 
-      // do we need to override previous object?
-      if (wrp->isAdjustableEOV() && !mAPI.isSnapshotMode()) {
-        o2::ccdb::adjustOverriddenEOV(mAPI, *wrp.get());
+void CCDBPopulator::doUpload(const CcdbObjectInfo& wrp, const gsl::span& pld, bool cached)
+{
+  std::string msg = fmt::format("Storing in ccdb {}{}/{} of size {} valid for {} : {}", cached ? "cached " : "", wrp.getPath(), wrp.getFileName(), pld.size(), wrp.getStartValidityTimestamp(), wrp.getEndValidityTimestamp());
+  auto uploadTS = o2::ccdb::getCurrentTimestamp();
+  logAsNeeded(uploadTS, wrp.getPath(), msg);
+  std::map metadata;
+  const auto* md = &wrp.getMetaData();
+  if (mRunNoFromDH > 0 && md->find(o2::base::NameConf::CCDBRunTag.data()) == md->end()) { // if valid run number is provided and it is not filled in the metadata, add it to the clone
+    metadata = *md;                                                                       // clone since the md from the message is const
+    metadata[o2::base::NameConf::CCDBRunTag.data()] = mRunNoStr;
+    md = &metadata;
+  }
+  int res = mAPI.storeAsBinaryFile(&pld[0], pld.size(), wrp.getFileName(), wrp.getObjectType(), wrp.getPath(), *md, wrp.getStartValidityTimestamp(), wrp.getEndValidityTimestamp());
+  if (res) {
+    if (mFatalOnFailure) {
+      LOGP(fatal, "failed on uploading to {} / {} for [{}:{}]", mAPI.getURL(), wrp.getPath(), wrp.getStartValidityTimestamp(), wrp.getEndValidityTimestamp());
+    } else {
+      LOGP(error, "failed on uploading to {} / {} for [{}:{}]", mAPI.getURL(), wrp.getPath(), wrp.getStartValidityTimestamp(), wrp.getEndValidityTimestamp());
+    }
+  }
+  // if requested, make sure that the new object can be queried
+  if (mValidateUpload || wrp.getValidateUpload()) {
+    constexpr long MAXDESYNC = 3;
+    auto headers = mAPI.retrieveHeaders(wrp.getPath(), {}, wrp.getStartValidityTimestamp() + (wrp.getEndValidityTimestamp() - wrp.getStartValidityTimestamp()) / 2);
+    if (headers.empty() ||
+        std::atol(headers["Created"].c_str()) < uploadTS - MAXDESYNC ||
+        std::atol(headers["Valid-From"].c_str()) != wrp.getStartValidityTimestamp() ||
+        std::atol(headers["Valid-Until"].c_str()) != wrp.getEndValidityTimestamp()) {
+      if (mFatalOnFailure) {
+        LOGP(fatal, "Failed to validate upload to {} / {} for [{}:{}]", mAPI.getURL(), wrp.getPath(), wrp.getStartValidityTimestamp(), wrp.getEndValidityTimestamp());
+      } else {
+        LOGP(error, "Failed to validate upload to {} / {} for [{}:{}]", mAPI.getURL(), wrp.getPath(), wrp.getStartValidityTimestamp(), wrp.getEndValidityTimestamp());
       }
+    } else {
+      LOGP(important, "Validated upload to {} / {} for [{}:{}]", mAPI.getURL(), wrp.getPath(), wrp.getStartValidityTimestamp(), wrp.getEndValidityTimestamp());
     }
   }
+  if (wrp.isAdjustableEOV() && !mAPI.isSnapshotMode()) {
+    o2::ccdb::adjustOverriddenEOV(mAPI, wrp);
+  }
+}
 
-  void endOfStream(o2::framework::EndOfStreamContext& ec) final
-  {
-    LOG(info) << "EndOfStream received";
+void CCDBPopulator::logAsNeeded(long nowMS, const std::string& path, std::string& msg)
+{
+  auto& lastLog = mThrottling[path];
+  if (lastLog.first + mThrottlingDelayMS < nowMS) {
+    if (lastLog.second) {
+      msg += fmt::format(" ({} uploads were logged as INFO)", lastLog.second);
+      lastLog.second = 0;
+    }
+    lastLog.first = nowMS;
+    LOG(important) << msg;
+  } else {
+    lastLog.second++;
+    LOG(info) << msg;
   }
+}
 
- private:
-  CcdbApi mAPI;
-  bool mFatalOnFailure = true;                             // produce fatal on failed upload
-  std::int64_t mSSpecMin = -1;                             // min subspec to accept
-  std::int64_t mSSpecMax = -1;                             // max subspec to accept
-  std::string mCCDBpath = "http://ccdb-test.cern.ch:8080"; // CCDB path
-};
+void CCDBPopulator::endOfStream(o2::framework::EndOfStreamContext& ec)
+{
+  if (mEnded) {
+    return;
+  }
+  mEnded = true;
+  LOG(info) << "EndOfStream received";
+  if (mOrderingLatencyMS > 0) {
+    checkCache(-mOrderingLatencyMS); // force
+  }
+}
+
+void CCDBPopulator::stop()
+{
+  if (mEnded) {
+    return;
+  }
+  mEnded = true;
+  LOG(info) << "Forced stop";
+  if (mOrderingLatencyMS > 0) {
+    checkCache(-mOrderingLatencyMS); // force
+  }
+}
 
 } // namespace calibration
 
@@ -133,7 +260,7 @@ namespace framework
 DataProcessorSpec getCCDBPopulatorDeviceSpec(const std::string& defCCDB, const std::string& nameExt)
 {
   using clbUtils = o2::calibration::Utils;
-  std::vector inputs = {{"clbPayload", "CLP"}, {"clbWrapper", "CLW"}};
+  std::vector inputs = {{"clbPayload", "CLP", Lifetime::Sporadic}, {"clbWrapper", "CLW", Lifetime::Sporadic}};
   std::string devName = "ccdb-populator";
   devName += nameExt;
   return DataProcessorSpec{
@@ -145,6 +272,9 @@ DataProcessorSpec getCCDBPopulatorDeviceSpec(const std::string& defCCDB, const s
       {"ccdb-path", VariantType::String, defCCDB, {"Path to CCDB"}},
       {"sspec-min", VariantType::Int64, -1L, {"min subspec to accept"}},
       {"sspec-max", VariantType::Int64, -1L, {"max subspec to accept"}},
+      {"ordering-latency", VariantType::Int, -1, {"if enabled (positive) bufferize object and upload it if no object with smaller SOV received in given waiting time (ms)"}},
+      {"throttling-delay", VariantType::Int64, 300000L, {"produce important type log at most once per this period in ms for each CCDB path"}},
+      {"validate-upload", VariantType::Bool, false, {"valider upload by querying its headers"}},
       {"fatal-on-failure", VariantType::Bool, false, {"do not produce fatal on failed upload"}}}};
 }
 
diff --git a/Detectors/Calibration/workflow/ccdb-populator-workflow.cxx b/Detectors/Calibration/workflow/ccdb-populator-workflow.cxx
index 5432712dfdac3..d03920b7a657f 100644
--- a/Detectors/Calibration/workflow/ccdb-populator-workflow.cxx
+++ b/Detectors/Calibration/workflow/ccdb-populator-workflow.cxx
@@ -33,7 +33,8 @@ void customize(std::vector& policies)
   // we customize the pipeline processors to consume data as it comes
   using CompletionPolicy = o2::framework::CompletionPolicy;
   using CompletionPolicyHelpers = o2::framework::CompletionPolicyHelpers;
-  policies.push_back(CompletionPolicyHelpers::defineByName("ccdb-populator.*", CompletionPolicy::CompletionOp::Consume));
+  auto& pol = policies.emplace_back(CompletionPolicyHelpers::defineByName("ccdb-populator.*", CompletionPolicy::CompletionOp::Consume));
+  pol.order = CompletionPolicy::CompletionOrder::Slot;
 }
 
 // ------------------------------------------------------------------
diff --git a/Detectors/Calibration/workflow/include/DetectorsCalibrationWorkflow/MeanVertexCalibratorSpec.h b/Detectors/Calibration/workflow/include/DetectorsCalibrationWorkflow/MeanVertexCalibratorSpec.h
index 3ee4c02bab3f2..9cfbe79484837 100644
--- a/Detectors/Calibration/workflow/include/DetectorsCalibrationWorkflow/MeanVertexCalibratorSpec.h
+++ b/Detectors/Calibration/workflow/include/DetectorsCalibrationWorkflow/MeanVertexCalibratorSpec.h
@@ -29,7 +29,7 @@ namespace calibration
 class MeanVertexCalibDevice : public Task
 {
  public:
-  MeanVertexCalibDevice(std::shared_ptr req) : mCCDBRequest(req) {}
+  MeanVertexCalibDevice(std::shared_ptr req, uint32_t dcsMVsubspec) : mCCDBRequest(req), mDCSSubSpec(dcsMVsubspec) {}
   void init(o2::framework::InitContext& ic) final;
   void run(o2::framework::ProcessingContext& pc) final;
   void endOfStream(o2::framework::EndOfStreamContext& ec) final;
@@ -37,7 +37,9 @@ class MeanVertexCalibDevice : public Task
 
  private:
   void sendOutput(DataAllocator& output);
-
+  int mFillNumber = 0;
+  int mRunNumber = 0;
+  uint32_t mDCSSubSpec = 0;
   std::unique_ptr mCalibrator;
   std::shared_ptr mCCDBRequest;
 };
@@ -46,7 +48,7 @@ class MeanVertexCalibDevice : public Task
 
 namespace framework
 {
-DataProcessorSpec getMeanVertexCalibDeviceSpec();
+DataProcessorSpec getMeanVertexCalibDeviceSpec(uint32_t dcsMVsubspec);
 
 } // namespace framework
 } // namespace o2
diff --git a/Detectors/Calibration/workflow/src/MeanVertexCalibratorSpec.cxx b/Detectors/Calibration/workflow/src/MeanVertexCalibratorSpec.cxx
index bfe0efc9e378c..1a64adcd9bf3c 100644
--- a/Detectors/Calibration/workflow/src/MeanVertexCalibratorSpec.cxx
+++ b/Detectors/Calibration/workflow/src/MeanVertexCalibratorSpec.cxx
@@ -14,9 +14,9 @@
 #include "Framework/ControlService.h"
 #include "Framework/ConfigParamRegistry.h"
 #include "Framework/Logger.h"
+#include "DetectorsCalibration/MeanVertexParams.h"
 #include "DetectorsCalibrationWorkflow/MeanVertexCalibratorSpec.h"
 #include "DetectorsCalibration/Utils.h"
-#include "DetectorsCalibration/MeanVertexParams.h"
 #include "CCDB/CcdbApi.h"
 #include "CCDB/CcdbObjectInfo.h"
 
@@ -28,23 +28,11 @@ namespace calibration
 {
 void MeanVertexCalibDevice::init(InitContext& ic)
 {
-
   o2::base::GRPGeomHelper::instance().setRequest(mCCDBRequest);
-  const o2::calibration::MeanVertexParams* params = &o2::calibration::MeanVertexParams::Instance();
-  int minEnt = params->minEntries;
-  int nbX = params->nbinsX;
-  float rangeX = params->rangeX;
-  int nbY = params->nbinsY;
-  float rangeY = params->rangeY;
-  int nbZ = params->nbinsZ;
-  float rangeZ = params->rangeZ;
-  int nSlots4SMA = params->nSlots4SMA;
-  auto slotL = params->tfPerSlot;
-  auto delay = params->maxTFdelay;
-  auto nPointsForSlope = params->nPointsForSlope;
-  mCalibrator = std::make_unique(minEnt, nbX, rangeX, nbY, rangeY, nbZ, rangeZ, nSlots4SMA, nPointsForSlope);
-  mCalibrator->setSlotLength(slotL);
-  mCalibrator->setMaxSlotsDelay(delay);
+  const auto& params = MeanVertexParams::Instance();
+  mCalibrator = std::make_unique();
+  mCalibrator->setSlotLength(params.tfPerSlot);
+  mCalibrator->setMaxSlotsDelay(float(params.maxTFdelay) / params.tfPerSlot);
   bool useVerboseMode = ic.options().get("use-verbose-mode");
   LOG(info) << " ************************* Verbose? " << useVerboseMode;
   if (useVerboseMode) {
@@ -57,13 +45,19 @@ void MeanVertexCalibDevice::init(InitContext& ic)
 void MeanVertexCalibDevice::run(o2::framework::ProcessingContext& pc)
 {
   o2::base::GRPGeomHelper::instance().checkUpdates(pc);
+  const auto& tinfo = pc.services().get();
+  if (tinfo.globalRunNumberChanged) { // new run is starting
+    mRunNumber = (tinfo.runNumber != -1 && tinfo.runNumber > 0) ? tinfo.runNumber : o2::base::GRPGeomHelper::instance().getGRPECS()->getRun();
+    mFillNumber = o2::base::GRPGeomHelper::instance().getGRPLHCIF()->getFillNumber();
+  }
+
   auto data = pc.inputs().get>("input");
   o2::base::TFIDInfoHelper::fillTFIDInfo(pc, mCalibrator->getCurrentTFInfo());
-  LOG(info) << "Processing TF " << mCalibrator->getCurrentTFInfo().tfCounter << " with " << data.size() << " vertices";
+  LOG(debug) << "Processing TF " << mCalibrator->getCurrentTFInfo().tfCounter << " with " << data.size() << " vertices";
   mCalibrator->process(data);
   sendOutput(pc.outputs());
   const auto& infoVec = mCalibrator->getMeanVertexObjectInfoVector();
-  LOG(info) << "Created " << infoVec.size() << " objects for TF " << mCalibrator->getCurrentTFInfo().tfCounter;
+  LOG(detail) << "Processed TF " << mCalibrator->getCurrentTFInfo().tfCounter << " with " << data.size() << " vertices, for which we created " << infoVec.size() << " objects for TF " << mCalibrator->getCurrentTFInfo().tfCounter;
 }
 
 //_________________________________________________________________
@@ -89,20 +83,38 @@ void MeanVertexCalibDevice::sendOutput(DataAllocator& output)
 
   // extract CCDB infos and calibration objects, convert it to TMemFile and send them to the output
   // TODO in principle, this routine is generic, can be moved to Utils.h
-
   using clbUtils = o2::calibration::Utils;
   const auto& payloadVec = mCalibrator->getMeanVertexObjectVector();
   auto& infoVec = mCalibrator->getMeanVertexObjectInfoVector(); // use non-const version as we update it
   assert(payloadVec.size() == infoVec.size());
 
+  if (mDCSSubSpec && mDCSSubSpec < payloadVec.size()) {
+    LOGP(alarm, "Minimum subspec {} of messages for DCS CCDB is below the maximum subspec {} for production CCDB, increase the former", mDCSSubSpec, payloadVec.size());
+  }
+  static std::vector dcsMVObj;
   for (uint32_t i = 0; i < payloadVec.size(); i++) {
-    auto& w = infoVec[i];
-    auto image = o2::ccdb::CcdbApi::createObjectImage(&payloadVec[i], &w);
-    LOG(info) << "Sending object " << w.getPath() << "/" << w.getFileName() << " of size " << image->size()
+    auto w = infoVec[i];
+    auto& mv = payloadVec[i];
+    auto image = o2::ccdb::CcdbApi::createObjectImage(&mv, &w);
+    LOG(info) << (MeanVertexParams::Instance().skipObjectSending ? "Skip " : "") << "sending object "
+              << w.getPath() << "/" << w.getFileName() << " of size " << image->size()
               << " bytes, valid for " << w.getStartValidityTimestamp() << " : " << w.getEndValidityTimestamp();
-
-    output.snapshot(Output{clbUtils::gDataOriginCDBPayload, "MEANVERTEX", i}, *image.get()); // vector
-    output.snapshot(Output{clbUtils::gDataOriginCDBWrapper, "MEANVERTEX", i}, w);            // root-serialized
+    if (!MeanVertexParams::Instance().skipObjectSending) {
+      if (mDCSSubSpec) { // create message for DCS CCDB
+        auto ts = (w.getStartValidityTimestamp() + w.getEndValidityTimestamp()) / 2;
+        o2::ccdb::CcdbObjectInfo dcsw("GLO/Calib/MeanVertexCSV", "csv", fmt::format("meanvertex_{}.csv", ts), {}, w.getStartValidityTimestamp(), w.getEndValidityTimestamp());
+
+        std::string csvMeanVertex = fmt::format("timestamp={},fillNumber={},runNumber={},x={:+.4e},y={:+.4e},z={:+.4e},sigmax={:+.4e},sigmay={:+.4e},sigmaz={:+.4e}",
+                                                ts, mFillNumber, mRunNumber, mv.getX(), mv.getY(), mv.getZ(), mv.getSigmaX(), mv.getSigmaY(), mv.getSigmaZ());
+        dcsMVObj.clear();
+        std::copy(csvMeanVertex.begin(), csvMeanVertex.end(), std::back_inserter(dcsMVObj));
+        output.snapshot(Output{clbUtils::gDataOriginCDBPayload, "MEANVERTEX_DCS", mDCSSubSpec + i}, dcsMVObj);
+        output.snapshot(Output{clbUtils::gDataOriginCDBWrapper, "MEANVERTEX_DCS", mDCSSubSpec + i}, dcsw);
+      }
+      w.setEndValidityTimestamp(w.getEndValidityTimestamp() + o2::ccdb::CcdbObjectInfo::MONTH);
+      output.snapshot(Output{clbUtils::gDataOriginCDBPayload, "MEANVERTEX", i}, *image.get()); // vector
+      output.snapshot(Output{clbUtils::gDataOriginCDBWrapper, "MEANVERTEX", i}, w);            // root-serialized
+    }
   }
   if (payloadVec.size()) {
     mCalibrator->initOutput(); // reset the outputs once they are already sent
@@ -113,7 +125,7 @@ void MeanVertexCalibDevice::sendOutput(DataAllocator& output)
 namespace framework
 {
 
-DataProcessorSpec getMeanVertexCalibDeviceSpec()
+DataProcessorSpec getMeanVertexCalibDeviceSpec(uint32_t dcsMVsubspec)
 {
 
   using device = o2::calibration::MeanVertexCalibDevice;
@@ -122,7 +134,7 @@ DataProcessorSpec getMeanVertexCalibDeviceSpec()
   inputs.emplace_back("input", "GLO", "PVTX");
   auto ccdbRequest = std::make_shared(true,                           // orbitResetTime
                                                                 true,                           // GRPECS=true
-                                                                false,                          // GRPLHCIF
+                                                                true,                           // GRPLHCIF
                                                                 false,                          // GRPMagField
                                                                 false,                          // askMatLUT
                                                                 o2::base::GRPGeomRequest::None, // geometry
@@ -131,12 +143,16 @@ DataProcessorSpec getMeanVertexCalibDeviceSpec()
   std::vector outputs;
   outputs.emplace_back(ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBPayload, "MEANVERTEX"}, Lifetime::Sporadic);
   outputs.emplace_back(ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBWrapper, "MEANVERTEX"}, Lifetime::Sporadic);
+  if (dcsMVsubspec) {
+    outputs.emplace_back(ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBPayload, "MEANVERTEX_DCS"}, Lifetime::Sporadic);
+    outputs.emplace_back(ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBWrapper, "MEANVERTEX_DCS"}, Lifetime::Sporadic);
+  }
 
   return DataProcessorSpec{
     "mean-vertex-calibration",
     inputs,
     outputs,
-    AlgorithmSpec{adaptFromTask(ccdbRequest)},
+    AlgorithmSpec{adaptFromTask(ccdbRequest, dcsMVsubspec)},
     Options{{"use-verbose-mode", VariantType::Bool, false, {"Use verbose mode"}}}};
 }
 
diff --git a/Detectors/Calibration/workflow/src/mean-vertex-calibration-workflow.cxx b/Detectors/Calibration/workflow/src/mean-vertex-calibration-workflow.cxx
index 337690ec5104d..a893f99a60552 100644
--- a/Detectors/Calibration/workflow/src/mean-vertex-calibration-workflow.cxx
+++ b/Detectors/Calibration/workflow/src/mean-vertex-calibration-workflow.cxx
@@ -20,6 +20,7 @@ void customize(std::vector& workflowOptions)
 {
   // option allowing to set parameters
   std::vector options{
+    {"meanvertex-dcs-subspec", VariantType::Int, 0x7fff, {"If >0, send text version of MeanVertex to this subspec"}},
     {"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}}};
   std::swap(workflowOptions, options);
 }
@@ -32,6 +33,6 @@ WorkflowSpec defineDataProcessing(ConfigContext const& configcontext)
 {
   WorkflowSpec specs;
   o2::conf::ConfigurableParam::updateFromString(configcontext.options().get("configKeyValues"));
-  specs.emplace_back(getMeanVertexCalibDeviceSpec());
+  specs.emplace_back(getMeanVertexCalibDeviceSpec(configcontext.options().get("meanvertex-dcs-subspec")));
   return specs;
 }
diff --git a/Detectors/DCS/include/DetectorsDCS/DataPointCompositeObject.h b/Detectors/DCS/include/DetectorsDCS/DataPointCompositeObject.h
index 3929dc52943e7..9aaf6e517336d 100644
--- a/Detectors/DCS/include/DetectorsDCS/DataPointCompositeObject.h
+++ b/Detectors/DCS/include/DetectorsDCS/DataPointCompositeObject.h
@@ -39,7 +39,7 @@ namespace dcs
      * @see ADAPRO::ADAPOS::DataPointIdentifier
      * @see ADAPRO::ADAPOS::DataPointValue
      */
-struct alignas(128) DataPointCompositeObject final {
+struct DataPointCompositeObject final {
   /**
          * The DataPointIdentifier object, which occupies the first 64 bytes of
          * the DataPointCompositeObject. This object contains the immutable
diff --git a/Detectors/DCS/include/DetectorsDCS/DataPointIdentifier.h b/Detectors/DCS/include/DetectorsDCS/DataPointIdentifier.h
index dc4abb3269d9d..79b7d7cf886c7 100644
--- a/Detectors/DCS/include/DetectorsDCS/DataPointIdentifier.h
+++ b/Detectors/DCS/include/DetectorsDCS/DataPointIdentifier.h
@@ -39,7 +39,7 @@ namespace dcs
      * DataPointIdentifier object is responsible for storing the alias and type
      * information of a data point.
      */
-class alignas(64) DataPointIdentifier final
+class DataPointIdentifier final
 {
   const uint64_t pt1;
   const uint64_t pt2;
diff --git a/Detectors/DCS/include/DetectorsDCS/DataPointValue.h b/Detectors/DCS/include/DetectorsDCS/DataPointValue.h
index 021e8bdf0beba..4c8686d16d6e4 100644
--- a/Detectors/DCS/include/DetectorsDCS/DataPointValue.h
+++ b/Detectors/DCS/include/DetectorsDCS/DataPointValue.h
@@ -45,7 +45,7 @@ namespace dcs
      * constructors and setters of this struct, that take an array containing
      * binary data as parameter, don't check the length of the given array.
      */
-struct alignas(64) DataPointValue final {
+struct DataPointValue final {
   /**
          * 
This flag is used for signaling that a DPVAL stream connection is
          * working, but there is no data to be sent.

diff --git a/Detectors/DCS/include/DetectorsDCS/DeliveryType.h b/Detectors/DCS/include/DetectorsDCS/DeliveryType.h
index 236d9eb084759..f0ee5573a9421 100644
--- a/Detectors/DCS/include/DetectorsDCS/DeliveryType.h
+++ b/Detectors/DCS/include/DetectorsDCS/DeliveryType.h
@@ -20,20 +20,12 @@
 #define O2_DCS_DELIVERY_TYPE
 
 #include 
-#include 
+#include 
 #include 
 #include "DetectorsDCS/GenericFunctions.h"
 
-namespace o2
-{
-namespace dcs
+namespace o2::dcs
 {
-/**
-     * This regular expression matches with strings representing payload types.
-     */
-static const std::regex REGEX_PT(
-  "^(Raw|DPVAL)/(Int|Uint|Float|Double|Bool|Char|String|Time|Binary)$");
-
 /**
      * 
DeliveryType is a piece of meta-information used for deducing types of
      * DPVAL payloads and DIM service description strings used with services
@@ -406,8 +398,8 @@ inline size_t dim_buffer_size(const DeliveryType type)
       throw std::domain_error("Illegal DeliveryType.");
   }
 }
-} // namespace dcs
+} // namespace o2::dcs
+
 
-} // namespace o2
 
 #endif /* O2_DCS_DELIVERY_TYPE */
diff --git a/Detectors/DCS/src/AliasExpander.cxx b/Detectors/DCS/src/AliasExpander.cxx
index b197d2fb78d67..41aaec8351f72 100644
--- a/Detectors/DCS/src/AliasExpander.cxx
+++ b/Detectors/DCS/src/AliasExpander.cxx
@@ -82,7 +82,7 @@ std::vector extractRange(std::string range)
   std::vector result;
 
   for (auto i = a; i <= b; i++) {
-    auto substituted = fmt::format(intFormat, i);
+    auto substituted = fmt::format(fmt::runtime(intFormat), i);
     result.push_back(substituted);
   }
   return result;
@@ -124,7 +124,7 @@ std::vector expandAlias(const std::string& pattern)
   std::vector result;
 
   for (auto r : range) {
-    auto substituted = fmt::format(newPattern, r);
+    auto substituted = fmt::format(fmt::runtime(newPattern), r);
     result.emplace_back(substituted);
   }
 
diff --git a/Detectors/DCS/src/DataPointCreator.cxx b/Detectors/DCS/src/DataPointCreator.cxx
index eca5440519fcc..0bfb5bcd7d387 100644
--- a/Detectors/DCS/src/DataPointCreator.cxx
+++ b/Detectors/DCS/src/DataPointCreator.cxx
@@ -37,10 +37,9 @@ DataPointCompositeObject createDataPointCompositeObject(const std::string& alias
 template <>
 DataPointCompositeObject createDataPointCompositeObject(const std::string& alias, float val, uint32_t seconds, uint16_t msec, uint16_t flags)
 {
-  float tmp[2];
-  tmp[0] = val;
-  tmp[1] = 0;
-  return createDPCOM(alias, reinterpret_cast(&tmp[0]), seconds, msec, flags, DeliveryType::DPVAL_FLOAT);
+  uint64_t tmp = 0;
+  memcpy(&tmp, &val, sizeof(val));
+  return createDPCOM(alias, &tmp, seconds, msec, flags, DeliveryType::DPVAL_FLOAT);
 }
 
 template <>
@@ -54,29 +53,38 @@ template <>
 DataPointCompositeObject createDataPointCompositeObject(const std::string& alias, uint32_t val, uint32_t seconds, uint16_t msec, uint16_t flags)
 {
   uint64_t tmp{val};
-  return createDPCOM(alias, reinterpret_cast(&tmp), seconds, msec, flags, DeliveryType::DPVAL_UINT);
+  return createDPCOM(alias, &tmp, seconds, msec, flags, DeliveryType::DPVAL_UINT);
+}
+
+template <>
+DataPointCompositeObject createDataPointCompositeObject(const std::string& alias, long long val, uint32_t seconds, uint16_t msec, uint16_t flags)
+{
+  uint64_t tmp{static_cast(val)};
+  return createDPCOM(alias, &tmp, seconds, msec, flags, DeliveryType::DPVAL_UINT);
 }
 
 template <>
 DataPointCompositeObject createDataPointCompositeObject(const std::string& alias, char val, uint32_t seconds, uint16_t msec, uint16_t flags)
 {
-  return createDPCOM(alias, reinterpret_cast(&val), seconds, msec, flags, DeliveryType::DPVAL_CHAR);
+  uint64_t tmp = 0;
+  memcpy(&tmp, &val, 1);
+  return createDPCOM(alias, &tmp, seconds, msec, flags, DeliveryType::DPVAL_CHAR);
 }
 
 template <>
 DataPointCompositeObject createDataPointCompositeObject(const std::string& alias, bool val, uint32_t seconds, uint16_t msec, uint16_t flags)
 {
   uint64_t tmp{val};
-  return createDPCOM(alias, reinterpret_cast(&tmp), seconds, msec, flags, DeliveryType::DPVAL_BOOL);
+  return createDPCOM(alias, &tmp, seconds, msec, flags, DeliveryType::DPVAL_BOOL);
 }
 
 template <>
 DataPointCompositeObject createDataPointCompositeObject(const std::string& alias, std::string val, uint32_t seconds, uint16_t msec, uint16_t flags)
 {
   constexpr int N{56};
-  char str[N];
-  strncpy(str, val.c_str(), N);
-  return createDPCOM(alias, reinterpret_cast(&str[0]), seconds, msec, flags, DeliveryType::DPVAL_STRING);
+  uint64_t tmp[N / sizeof(uint64_t)];
+  strncpy((char*)tmp, val.c_str(), N);
+  return createDPCOM(alias, tmp, seconds, msec, flags, DeliveryType::DPVAL_STRING);
 }
 
 } // namespace o2::dcs
diff --git a/Detectors/DCS/src/DataPointGenerator.cxx b/Detectors/DCS/src/DataPointGenerator.cxx
index 3e7ead2146687..15f79a709bba1 100644
--- a/Detectors/DCS/src/DataPointGenerator.cxx
+++ b/Detectors/DCS/src/DataPointGenerator.cxx
@@ -63,7 +63,7 @@ std::pair getDate(const std::string& refDate)
 namespace o2::dcs
 {
 
-//std::enable_if_t::value, bool> = true>
+// std::enable_if_t::value, bool> = true>
 
 template 
 std::vector
@@ -72,16 +72,15 @@ std::vector
 {
   std::vector dpcoms;
   static_assert(std::is_arithmetic::value, "T must be an arithmetic type");
-  typedef typename std::conditional::value,
-                                    std::uniform_int_distribution,
-                                    std::uniform_real_distribution>::type distType;
-
+  using distType = std::conditional_t::value,
+                                      std::uniform_int_distribution,
+                                      std::uniform_real_distribution>;
   std::random_device rd;
   std::mt19937 mt(rd());
   distType dist{minValue, maxValue};
   auto [seconds, msec] = getDate(refDate);
   for (auto alias : expandAliases(aliases)) {
-    auto value = dist(mt);
+    T value = dist(mt);
     dpcoms.emplace_back(o2::dcs::createDataPointCompositeObject(alias, value, seconds, msec));
   }
   return dpcoms;
@@ -106,11 +105,13 @@ template std::vector generateRandomDataPoints
 
 template std::vector generateRandomDataPoints(const std::vector& aliases, int32_t minValue, int32_t maxValue, std::string);
 
+template std::vector generateRandomDataPoints(const std::vector& aliases, long long minValue, long long maxValue, std::string);
+
 template std::vector generateRandomDataPoints(const std::vector& aliases, char minValue, char maxValue, std::string);
 
 /** Need a specific specialization for bool as got into trouble compiling uniform_int_distribution
-  * on some platform (e.g. CC7).
-  */
+ * on some platform (e.g. CC7).
+ */
 template <>
 std::vector generateRandomDataPoints(const std::vector& aliases, bool minValue, bool maxValue, std::string refDate)
 {
@@ -127,9 +128,9 @@ std::vector generateRandomDataPoints(co
 }
 
 /**
-  * Generate data points of type string, where each string is random, with
-  * a length between the length of the two input strings (minLength,maxLength)
-  */
+ * Generate data points of type string, where each string is random, with
+ * a length between the length of the two input strings (minLength,maxLength)
+ */
 template <>
 std::vector generateRandomDataPoints(const std::vector& aliases, std::string minLength, std::string maxLength, std::string refDate)
 {
@@ -139,7 +140,7 @@ std::vector generateRandomDataPoints dist{minLength.size(), maxLength.size()};
   auto [seconds, msec] = getDate(refDate);
   for (auto alias : expandAliases(aliases)) {
-    auto value = o2::dcs::random_string2(dist(mt));
+    std::string value = o2::dcs::random_string2(dist(mt));
     dpcoms.emplace_back(o2::dcs::createDataPointCompositeObject(alias, value, seconds, msec));
   }
   return dpcoms;
diff --git a/Detectors/DCS/src/StringUtils.cxx b/Detectors/DCS/src/StringUtils.cxx
index e66e7625c6836..f65f522fa426d 100644
--- a/Detectors/DCS/src/StringUtils.cxx
+++ b/Detectors/DCS/src/StringUtils.cxx
@@ -34,8 +34,8 @@ using namespace std;
 
 random_device dev;
 default_random_engine gen(dev());
-uniform_int_distribution uniform_dist(32, 126);
-geometric_distribution geom_dist(0.1);
+uniform_int_distribution uniform_dist(32, 126);
+geometric_distribution geom_dist(0.1);
 
 constexpr char ALPHABET[39]{
   'E', 'T', 'A', 'O', 'I', '_', 'N', 'S', 'H', 'R',
diff --git a/Detectors/DCS/testWorkflow/README.md b/Detectors/DCS/testWorkflow/README.md
index 424bd7e9d9b25..0f203365da84b 100644
--- a/Detectors/DCS/testWorkflow/README.md
+++ b/Detectors/DCS/testWorkflow/README.md
@@ -104,6 +104,7 @@ You can use it as e.g.:
 echo "blabla" > TOFfile.txt  # this is the file you want to send from the DCS to the config processor
 xterm -e "alienv load cppzmq/latest O2/latest; ./dcssend -f TOFfile.txt -o 5556 -a 5557"& # run the server emulator in separate terminal
 ```
+In case the process should end upon successful sending of the object, add `-q true`. This is helpful e.g. in case several objects should be sent from a script. Otherwise the object is sent over and over again.
 
 Then, in other terminal you can run your DCS config processor, as described above (make sure the ports of sender and receiver are consistent.
 In case of problems you can validate the receiving process using `dcsclient` test utility (emulates `o2-dcs-config-proxy ...` workflow by receiving the file from the `DCS server` and sending it an acknowledgment):
diff --git a/Detectors/DCS/testWorkflow/src/DCSDataReplaySpec.cxx b/Detectors/DCS/testWorkflow/src/DCSDataReplaySpec.cxx
index 05daf522f372a..8dc003fc176f3 100644
--- a/Detectors/DCS/testWorkflow/src/DCSDataReplaySpec.cxx
+++ b/Detectors/DCS/testWorkflow/src/DCSDataReplaySpec.cxx
@@ -46,6 +46,10 @@ class DCSDataReplayer : public o2::framework::Task
   char mAlias[50];
   uint64_t mMaxTF;
   uint64_t mTFs = 0;
+  int deltaTimeSendData = -1;
+  uint32_t startTime = -1;
+  uint32_t endTime = 0;
+  std::vector> dataIndicesPerTF;
   TTree mInputData;
   std::vector mDataPointHints;
   o2::header::DataDescription mDataDescription;
@@ -59,6 +63,7 @@ void DCSDataReplayer::init(o2::framework::InitContext& ic)
 {
   mMaxTF = ic.options().get("max-timeframes");
   mInputFileName = ic.options().get("input-file");
+  deltaTimeSendData = ic.options().get("delta-time-send-data");
   mInputData.ReadFile(mInputFileName.data(), "time/D:alias/C:value/D", ';');
   mInputData.SetBranchAddress("time", &mTime);
   mInputData.SetBranchAddress("value", &mValue);
@@ -73,22 +78,67 @@ void DCSDataReplayer::run(o2::framework::ProcessingContext& pc)
     LOG(info) << "Data generator reached TF " << tfid << ", stopping";
     pc.services().get().endOfStream();
     pc.services().get().readyToQuit(o2::framework::QuitRequest::Me);
+    return;
   }
 
   std::vector dpcoms;
+
   for (Long64_t iEntry = 0; iEntry < mInputData.GetEntries(); ++iEntry) {
-    mInputData.GetEntry(iEntry);
+    int entryTree = iEntry;
+
+    // load only releavant entries if requested
+    if (deltaTimeSendData > 0 && tfid > 2) {
+
+      if (tfid - 1 >= dataIndicesPerTF.size()) {
+        LOGP(warning, "TF ID {} is larger than the number of TFs in dataIndicesPerTF: {}", tfid, dataIndicesPerTF.size());
+        break;
+      }
+
+      if (iEntry >= dataIndicesPerTF[tfid - 1].size()) {
+        break;
+      } else {
+        entryTree = dataIndicesPerTF[tfid - 1][iEntry];
+      }
+    }
+
+    mInputData.GetEntry(entryTree);
     const auto ultime = uint64_t(std::round(mTime * 1000));
     const auto seconds = uint32_t(ultime / 1000);
     const auto msec = uint16_t(ultime % 1000);
-
-    dpcoms.emplace_back(o2::dcs::createDataPointCompositeObject(mAlias, float(mValue), seconds, msec));
+    if (deltaTimeSendData > 0) {
+      // send data in packages
+      if (tfid == 0) {
+        startTime = std::min(startTime, seconds);
+        endTime = std::max(endTime, seconds);
+        if (iEntry == mInputData.GetEntries() - 1) {
+          const int totalTFs = (endTime - startTime) / deltaTimeSendData + 1;
+          dataIndicesPerTF.resize(totalTFs);
+          LOGP(info, "Sending data from {} to {} with {} TFs", startTime, endTime, totalTFs);
+        }
+      } else {
+        if (tfid == 1) {
+          const int index = (seconds - startTime) / deltaTimeSendData;
+          dataIndicesPerTF[index].emplace_back(iEntry);
+        }
+        const uint64_t startTimeTF = startTime + (tfid - 1) * deltaTimeSendData;
+        const uint64_t endTimeTF = startTimeTF + deltaTimeSendData;
+        if (seconds >= startTimeTF && seconds < endTimeTF) {
+          dpcoms.emplace_back(o2::dcs::createDataPointCompositeObject(mAlias, float(mValue), seconds, msec));
+          // check if all data has been processed
+          if (seconds == endTime) {
+            mMaxTF = tfid;
+          }
+        }
+      }
+    } else {
+      dpcoms.emplace_back(o2::dcs::createDataPointCompositeObject(mAlias, float(mValue), seconds, msec));
+    }
   }
   // auto dpcoms = generate(mDataPointHints, fraction, tfid);
 
   LOG(info)
     << "***************** TF " << tfid << " has generated " << dpcoms.size() << " DPs";
-  pc.outputs().snapshot(Output{"DCS", mDataDescription, 0, Lifetime::Timeframe}, dpcoms);
+  pc.outputs().snapshot(Output{"DCS", mDataDescription, 0}, dpcoms);
   mTFs++;
 }
 } // namespace
@@ -113,6 +163,7 @@ o2::framework::DataProcessorSpec getDCSDataReplaySpec(std::vector(timeNow.time_since_epoch()).count(); // in ms
 
-  pc.outputs().snapshot(Output{"DCS", mDataDescription, 0, Lifetime::Timeframe}, dpcoms);
+  pc.outputs().snapshot(Output{"DCS", mDataDescription, 0}, dpcoms);
   mTFs++;
 }
 } // namespace
diff --git a/Detectors/DCS/testWorkflow/src/DCStoDPLconverter.h b/Detectors/DCS/testWorkflow/src/DCStoDPLconverter.h
index 6a1a0a3f04145..679a5b9d91f41 100644
--- a/Detectors/DCS/testWorkflow/src/DCStoDPLconverter.h
+++ b/Detectors/DCS/testWorkflow/src/DCStoDPLconverter.h
@@ -14,6 +14,7 @@
 
 #include "Framework/DataSpecUtils.h"
 #include "Framework/ExternalFairMQDeviceProxy.h"
+#include "Framework/RawDeviceService.h"
 #include 
 #include 
 #include "DetectorsDCS/DataPointIdentifier.h"
@@ -40,9 +41,7 @@ struct hash {
 };
 } // namespace std
 
-namespace o2
-{
-namespace dcs
+namespace o2::dcs
 {
 using DPID = o2::dcs::DataPointIdentifier;
 using DPVAL = o2::dcs::DataPointValue;
@@ -51,22 +50,26 @@ using DPCOM = o2::dcs::DataPointCompositeObject;
 /// A callback function to retrieve the FairMQChannel name to be used for sending
 /// messages of the specified OutputSpec
 
-o2f::InjectorFunction dcs2dpl(std::unordered_map& dpid2group, bool fbiFirst, bool verbose = false)
+o2f::InjectorFunction dcs2dpl(std::unordered_map>& dpid2group, bool fbiFirst, bool verbose = false, int FBIPerInterval = 1)
 {
 
-  return [dpid2group, fbiFirst, verbose](o2::framework::TimingInfo& tinfo, fair::mq::Device& device, fair::mq::Parts& parts, o2f::ChannelRetriever channelRetriever) {
+  return [dpid2group, fbiFirst, verbose, FBIPerInterval](o2::framework::TimingInfo& tinfo, framework::ServiceRegistryRef const& services, fair::mq::Parts& parts, o2f::ChannelRetriever channelRetriever, size_t newTimesliceId, bool& stop) -> bool {
+    auto *device = services.get().device();
     static std::unordered_map cache; // will keep only the latest measurement in the 1-second wide window for each DPID
+    static std::unordered_map sentToChannel;
     static auto timer = std::chrono::high_resolution_clock::now();
     static auto timer0 = std::chrono::high_resolution_clock::now();
     static bool seenFBI = false;
     static uint32_t localTFCounter = 0;
     static size_t nInp = 0, nInpFBI = 0;
     static size_t szInp = 0, szInpFBI = 0;
-    LOG(debug) << "In lambda function: ********* Size of unordered_map (--> number of defined groups) = " << dpid2group.size();
+    if (verbose) {
+      LOG(info) << "In lambda function: ********* Size of unordered_map (--> number of defined groups) = " << dpid2group.size();
+    }
     // check if we got FBI (Master) or delta (MasterDelta)
     if (!parts.Size()) {
       LOGP(warn, "Empty input recieved at timeslice {}", tinfo.timeslice);
-      return;
+      return false;
     }
     std::string firstName = std::string((char*)&(reinterpret_cast(parts.At(0)->GetData()))->id);
 
@@ -93,12 +96,23 @@ o2f::InjectorFunction dcs2dpl(std::unordered_map& dp
         szInpFBI += sz;
       }
       auto nDPCOM = sz / sizeof(DPCOM); // number of DPCOM in current part
+      LOGP(debug, "sz={} szof={} -> /={} %={} | {} {}", sz, sizeof(DPCOM), nDPCOM, sz % sizeof(DPCOM), sizeof(o2::dcs::DataPointIdentifier), sizeof(o2::dcs::DataPointValue));
       for (size_t j = 0; j < nDPCOM; j++) {
-        const auto& src = *(reinterpret_cast(parts.At(i)->GetData()) + j);
+        const auto* ptr = (reinterpret_cast(parts.At(i)->GetData()) + j);
+        DPCOM src;
+        memcpy(&src, ptr, sizeof(DPCOM));
         // do we want to check if this DP was requested ?
         auto mapEl = dpid2group.find(src.id);
         if (verbose) {
-          LOG(info) << "Received DP " << src.id << " (data = " << src.data << "), matched to output-> " << (mapEl == dpid2group.end() ? "none " : mapEl->second.as());
+          std::string dest;
+          if (mapEl == dpid2group.end()) {
+            dest = "none";
+          } else {
+            for (const auto& ds : mapEl->second) {
+              dest += fmt::format("{}, ", ds.as());
+            }
+          }
+          LOG(info) << "Received DP " << src.id << " (data = " << src.data << "), matched to output-> " << dest;
         }
         if (mapEl != dpid2group.end()) {
           cache[src.id] = src; // this is needed in case in the 1s window we get a new value for the same DP
@@ -118,6 +132,7 @@ o2f::InjectorFunction dcs2dpl(std::unordered_map& dp
     }
 
     std::chrono::duration> duration = timerNow - timer;
+    bool didSendMessages = false;
     if (duration.count() > 1 && (seenFBI || !fbiFirst)) { // did we accumulate for 1 sec and have we seen FBI if it was requested?
       std::unordered_map, std::hash> outputs;
       // in the cache we have the final values of the DPs that we should put in the output
@@ -125,7 +140,9 @@ o2f::InjectorFunction dcs2dpl(std::unordered_map& dp
       for (auto& it : cache) {
         auto mapEl = dpid2group.find(it.first);
         if (mapEl != dpid2group.end()) {
-          outputs[mapEl->second].push_back(it.second);
+          for (const auto& ds : mapEl->second) {
+            outputs[ds].push_back(it.second);
+          }
         }
       }
       std::uint64_t creation = std::chrono::time_point_cast(timerNow).time_since_epoch().count();
@@ -152,7 +169,7 @@ o2f::InjectorFunction dcs2dpl(std::unordered_map& dp
         hdr.payloadSize = it.second.size() * sizeof(DPCOM);
         hdr.firstTForbit = 0; // this should be irrelevant for DCS
         o2h::Stack headerStack{hdr, o2::framework::DataProcessingHeader{tinfo.timeslice, 1, creation}};
-        auto fmqFactory = device.GetChannel(channel).Transport();
+        auto fmqFactory = device->GetChannel(channel).Transport();
         auto hdMessage = fmqFactory->CreateMessage(headerStack.size(), fair::mq::Alignment{64});
         auto plMessage = fmqFactory->CreateMessage(hdr.payloadSize, fair::mq::Alignment{64});
         memcpy(hdMessage->GetData(), headerStack.data(), headerStack.size());
@@ -172,8 +189,12 @@ o2f::InjectorFunction dcs2dpl(std::unordered_map& dp
       }
       // push output of every route
       for (auto& msgIt : messagesPerRoute) {
-        LOG(info) << "Sending " << msgIt.second->Size() / 2 << " parts to channel " << msgIt.first;
-        o2f::sendOnChannel(device, *msgIt.second.get(), msgIt.first, tinfo.timeslice);
+        if (verbose) {
+          LOG(info) << "Sending " << msgIt.second->Size() / 2 << " parts to channel " << msgIt.first;
+        }
+        o2f::sendOnChannel(*device, *msgIt.second.get(), msgIt.first, tinfo.timeslice);
+        sentToChannel[msgIt.first]++;
+        didSendMessages |= msgIt.second->Size() > 0;
       }
       timer = timerNow;
       cache.clear();
@@ -181,14 +202,20 @@ o2f::InjectorFunction dcs2dpl(std::unordered_map& dp
         localTFCounter++;
       }
     }
-    if (isFBI) {
+    if (isFBI && ((nInpFBI % FBIPerInterval) == 0 || verbose)) {
       float runtime = 1e-3 * std::chrono::duration_cast(timerNow - timer0).count();
-      LOGP(info, "{} inputs ({} bytes) of which {} FBI ({} bytes) seen in {:.3f} s", nInp, fmt::group_digits(szInp), nInpFBI, fmt::group_digits(szInpFBI), runtime);
+      std::string sent = "Sent since last FBI report: ";
+      for (auto& m : sentToChannel) {
+        auto pos = m.first.find("_to_");
+        sent += fmt::format("{}:{} ", m.first.substr(pos != std::string::npos ? pos + 4 : 0), m.second);
+        m.second = 0;
+      }
+      LOGP(info, "{} inputs ({} bytes) of which {} FBI ({} bytes) seen in {:.3f} s | {}", nInp, fmt::group_digits(szInp), nInpFBI, fmt::group_digits(szInpFBI), runtime, sent);
     }
+    return didSendMessages;
   };
 }
 
-} // namespace dcs
 } // namespace o2
 
 #endif /* O2_DCS_TO_DPL_CONVERTER_H */
diff --git a/Detectors/DCS/testWorkflow/src/dcs-config-proxy.cxx b/Detectors/DCS/testWorkflow/src/dcs-config-proxy.cxx
index d118daf65f77b..b660093fb6029 100644
--- a/Detectors/DCS/testWorkflow/src/dcs-config-proxy.cxx
+++ b/Detectors/DCS/testWorkflow/src/dcs-config-proxy.cxx
@@ -17,6 +17,7 @@
 #include "Framework/DataProcessorSpec.h"
 #include "Framework/DataSpecUtils.h"
 #include "Framework/ControlService.h"
+#include "Framework/RawDeviceService.h"
 #include "Framework/Logger.h"
 #include "Framework/Lifetime.h"
 #include "Framework/ConfigParamSpec.h"
@@ -33,7 +34,7 @@
 using namespace o2::framework;
 using DetID = o2::detectors::DetID;
 
-std::array exceptionsDetID{"GRP"};
+std::array exceptionsDetID{"GRP", "AGD"};
 
 void sendAnswer(const std::string& what, const std::string& ack_chan, fair::mq::Device& device)
 {
@@ -66,19 +67,17 @@ auto getDataOriginFromFilename(const std::string& filename)
 
 InjectorFunction dcs2dpl(const std::string& acknowledge)
 {
-
-  auto timesliceId = std::make_shared(0);
-
-  return [acknowledge, timesliceId](TimingInfo&, fair::mq::Device& device, fair::mq::Parts& parts, ChannelRetriever channelRetriever) {
+  return [acknowledge](TimingInfo&, ServiceRegistryRef const& services, fair::mq::Parts& parts, ChannelRetriever channelRetriever, size_t newTimesliceId, bool&) -> bool {
+    auto *device = services.get().device();
     if (parts.Size() == 0) { // received at ^c, ignore
       LOG(info) << "ignoring empty message";
-      return;
+      return false;
     }
     // make sure just 2 messages received
     if (parts.Size() != 2) {
       LOG(error) << "received " << parts.Size() << " instead of 2 expected";
-      sendAnswer("error0: wrong number of messages", acknowledge, device);
-      return;
+      sendAnswer("error0: wrong number of messages", acknowledge, *device);
+      return false;
     }
     std::string filename{static_cast(parts.At(0)->GetData()), parts.At(0)->GetSize()};
     size_t filesize = parts.At(1)->GetSize();
@@ -86,44 +85,44 @@ InjectorFunction dcs2dpl(const std::string& acknowledge)
     o2::header::DataOrigin dataOrigin = getDataOriginFromFilename(filename);
     if (dataOrigin == o2::header::gDataOriginInvalid) {
       LOG(error) << "unknown detector for " << filename;
-      sendAnswer(fmt::format("{}:error1: unrecognized filename", filename), acknowledge, device);
-      return;
+      sendAnswer(fmt::format("{}:error1: unrecognized filename", filename), acknowledge, *device);
+      return false;
     }
 
     o2::header::DataHeader hdrF("DCS_CONFIG_FILE", dataOrigin, 0);
     o2::header::DataHeader hdrN("DCS_CONFIG_NAME", dataOrigin, 0);
     OutputSpec outsp{hdrF.dataOrigin, hdrF.dataDescription, hdrF.subSpecification};
-    auto channel = channelRetriever(outsp, *timesliceId);
+    auto channel = channelRetriever(outsp, newTimesliceId);
     if (channel.empty()) {
       LOG(error) << "No output channel found for OutputSpec " << outsp;
-      sendAnswer(fmt::format("{}:error2: no channel to send", filename), acknowledge, device);
-      return;
+      sendAnswer(fmt::format("{}:error2: no channel to send", filename), acknowledge, *device);
+      return false;
     }
 
-    hdrF.tfCounter = *timesliceId;
+    hdrF.tfCounter = newTimesliceId;
     hdrF.payloadSerializationMethod = o2::header::gSerializationMethodNone;
     hdrF.splitPayloadParts = 1;
     hdrF.splitPayloadIndex = 0;
     hdrF.payloadSize = filesize;
     hdrF.firstTForbit = 0; // this should be irrelevant for DCS
 
-    hdrN.tfCounter = *timesliceId;
+    hdrN.tfCounter = newTimesliceId;
     hdrN.payloadSerializationMethod = o2::header::gSerializationMethodNone;
     hdrN.splitPayloadParts = 1;
     hdrN.splitPayloadIndex = 0;
     hdrN.payloadSize = parts.At(0)->GetSize();
     hdrN.firstTForbit = 0; // this should be irrelevant for DCS
 
-    auto fmqFactory = device.GetChannel(channel).Transport();
+    auto fmqFactory = device->GetChannel(channel).Transport();
     std::uint64_t creation = std::chrono::time_point_cast(std::chrono::system_clock::now()).time_since_epoch().count();
 
-    o2::header::Stack headerStackF{hdrF, DataProcessingHeader{*timesliceId, 1, creation}};
+    o2::header::Stack headerStackF{hdrF, DataProcessingHeader{newTimesliceId, 1, creation}};
     auto hdMessageF = fmqFactory->CreateMessage(headerStackF.size(), fair::mq::Alignment{64});
     auto plMessageF = fmqFactory->CreateMessage(hdrF.payloadSize, fair::mq::Alignment{64});
     memcpy(hdMessageF->GetData(), headerStackF.data(), headerStackF.size());
     memcpy(plMessageF->GetData(), parts.At(1)->GetData(), hdrF.payloadSize);
 
-    o2::header::Stack headerStackN{hdrN, DataProcessingHeader{*timesliceId, 1, creation}};
+    o2::header::Stack headerStackN{hdrN, DataProcessingHeader{newTimesliceId, 1, creation}};
     auto hdMessageN = fmqFactory->CreateMessage(headerStackN.size(), fair::mq::Alignment{64});
     auto plMessageN = fmqFactory->CreateMessage(hdrN.payloadSize, fair::mq::Alignment{64});
     memcpy(hdMessageN->GetData(), headerStackN.data(), headerStackN.size());
@@ -132,16 +131,16 @@ InjectorFunction dcs2dpl(const std::string& acknowledge)
     fair::mq::Parts outPartsF;
     outPartsF.AddPart(std::move(hdMessageF));
     outPartsF.AddPart(std::move(plMessageF));
-    sendOnChannel(device, outPartsF, channel, (size_t)-1);
+    sendOnChannel(*device, outPartsF, channel, (size_t)-1);
 
     fair::mq::Parts outPartsN;
     outPartsN.AddPart(std::move(hdMessageN));
     outPartsN.AddPart(std::move(plMessageN));
-    sendOnChannel(device, outPartsN, channel, *timesliceId);
+    sendOnChannel(*device, outPartsN, channel, newTimesliceId);
 
-    sendAnswer(fmt::format("{}:ok", filename), acknowledge, device);
+    sendAnswer(fmt::format("{}:ok", filename), acknowledge, *device);
     LOG(info) << "Sent DPL message and acknowledgment for file " << filename;
-    (*timesliceId)++;
+    return true;
   };
 }
 
diff --git a/Detectors/DCS/testWorkflow/src/dcs-config-test-workflow.cxx b/Detectors/DCS/testWorkflow/src/dcs-config-test-workflow.cxx
index 68a7172dddd4d..d762037445d88 100644
--- a/Detectors/DCS/testWorkflow/src/dcs-config-test-workflow.cxx
+++ b/Detectors/DCS/testWorkflow/src/dcs-config-test-workflow.cxx
@@ -53,8 +53,8 @@ DataProcessorSpec getDCSConsumerSpec(DetID det)
   procName += det.getName();
   return DataProcessorSpec{
     procName,
-    Inputs{{"confFile", ConcreteDataTypeMatcher{det.getDataOrigin(), "DCS_CONFIG_FILE"}, Lifetime::Timeframe},
-           {"confFileName", ConcreteDataTypeMatcher{det.getDataOrigin(), "DCS_CONFIG_NAME"}, Lifetime::Timeframe}},
+    Inputs{{"confFile", ConcreteDataTypeMatcher{det.getDataOrigin(), "DCS_CONFIG_FILE"}, Lifetime::Sporadic},
+           {"confFileName", ConcreteDataTypeMatcher{det.getDataOrigin(), "DCS_CONFIG_NAME"}, Lifetime::Sporadic}},
     Outputs{},
     AlgorithmSpec{adaptFromTask()},
     Options{}};
diff --git a/Detectors/DCS/testWorkflow/src/dcs-proxy.cxx b/Detectors/DCS/testWorkflow/src/dcs-proxy.cxx
index e870c7cb48601..bfe91a946d13b 100644
--- a/Detectors/DCS/testWorkflow/src/dcs-proxy.cxx
+++ b/Detectors/DCS/testWorkflow/src/dcs-proxy.cxx
@@ -44,6 +44,7 @@ using CcdbManager = o2::ccdb::BasicCCDBManager;
 void customize(std::vector& workflowOptions)
 {
   workflowOptions.push_back(ConfigParamSpec{"verbose", VariantType::Bool, false, {"verbose output"}});
+  workflowOptions.push_back(ConfigParamSpec{"fbi-report-rate", VariantType::Int, 6, {"report pet N FBI received"}});
   workflowOptions.push_back(ConfigParamSpec{"test-mode", VariantType::Bool, false, {"test mode"}});
   workflowOptions.push_back(ConfigParamSpec{"may-send-delta-first", VariantType::Bool, false, {"if true, do not wait for FBI before sending 1st output"}});
   workflowOptions.push_back(ConfigParamSpec{"ccdb-url", VariantType::String, "http://ccdb-test.cern.ch:8080", {"url of CCDB to get the detectors DPs configuration"}});
@@ -59,22 +60,23 @@ WorkflowSpec defineDataProcessing(ConfigContext const& config)
   bool verbose = config.options().get("verbose");
   bool testMode = config.options().get("test-mode");
   bool fbiFirst = !config.options().get("may-send-delta-first");
+  int repRate = std::max(1, config.options().get("fbi-report-rate"));
   std::string detectorList = config.options().get("detector-list");
   o2::conf::ConfigurableParam::updateFromString(config.options().get("configKeyValues"));
   std::string url = config.options().get("ccdb-url");
 
-  std::unordered_map dpid2DataDesc;
+  std::unordered_map> dpid2DataDesc;
 
   if (testMode) {
     DPID dpidtmp;
     DPID::FILL(dpidtmp, "ADAPOS_LG/TEST_000100", DeliveryType::DPVAL_STRING);
-    dpid2DataDesc[dpidtmp] = "COMMON"; // i.e. this will go to {DCS/COMMON/0} OutputSpec
+    dpid2DataDesc[dpidtmp] = {"COMMON"}; // i.e. this will go to {DCS/COMMON/0} OutputSpec
     DPID::FILL(dpidtmp, "ADAPOS_LG/TEST_000110", DeliveryType::DPVAL_STRING);
-    dpid2DataDesc[dpidtmp] = "COMMON";
+    dpid2DataDesc[dpidtmp] = {"COMMON"};
     DPID::FILL(dpidtmp, "ADAPOS_LG/TEST_000200", DeliveryType::DPVAL_STRING);
-    dpid2DataDesc[dpidtmp] = "COMMON1";
+    dpid2DataDesc[dpidtmp] = {"COMMON1"};
     DPID::FILL(dpidtmp, "ADAPOS_LG/TEST_000240", DeliveryType::DPVAL_INT);
-    dpid2DataDesc[dpidtmp] = "COMMON1";
+    dpid2DataDesc[dpidtmp] = {"COMMON1"};
   }
 
   else {
@@ -93,7 +95,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const& config)
         for (auto& el : *dpid2Det) {
           o2::header::DataDescription tmpd;
           tmpd.runtimeInit(el.second.c_str(), el.second.size());
-          dpid2DataDesc[el.first] = tmpd;
+          dpid2DataDesc[el.first].push_back(tmpd);
         }
       }
     }
@@ -105,7 +107,9 @@ WorkflowSpec defineDataProcessing(ConfigContext const& config)
   // now collect all required outputs to define OutputSpecs for specifyExternalFairMQDeviceProxy
   std::unordered_map> outMap;
   for (auto itdp : dpid2DataDesc) {
-    outMap[itdp.second]++;
+    for (const auto& ds : itdp.second) {
+      outMap[ds]++;
+    }
   }
 
   Outputs dcsOutputs;
@@ -117,7 +121,8 @@ WorkflowSpec defineDataProcessing(ConfigContext const& config)
     "dcs-proxy",
     std::move(dcsOutputs),
     "type=pull,method=connect,address=tcp://aldcsadaposactor:60000,rateLogging=1,transport=zeromq",
-    dcs2dpl(dpid2DataDesc, fbiFirst, verbose));
+    dcs2dpl(dpid2DataDesc, fbiFirst, verbose, repRate));
+  dcsProxy.labels.emplace_back(DataProcessorLabel{"input-proxy"});
 
   WorkflowSpec workflow;
   workflow.emplace_back(dcsProxy);
diff --git a/Detectors/DCS/testWorkflow/src/dcssend.cpp b/Detectors/DCS/testWorkflow/src/dcssend.cpp
index e8abd1933a46c..33c194f491f75 100644
--- a/Detectors/DCS/testWorkflow/src/dcssend.cpp
+++ b/Detectors/DCS/testWorkflow/src/dcssend.cpp
@@ -35,6 +35,7 @@ int main(int argc, char** argv)
     add_option("file-port,o", bpo::value()->default_value(5556), "port to send the file");
     add_option("ack-port,a", bpo::value()->default_value(5557), "port to receive the acknowledgment");
     add_option("timeout,t", bpo::value()->default_value(5), "timeout for acknowledgment");
+    add_option("quit-on-ack,q", bpo::value()->default_value(false), "quit if acknowledgment is ok");
     opt_all.add(opt_general).add(opt_hidden);
     bpo::store(bpo::command_line_parser(argc, argv).options(opt_all).positional(opt_pos).run(), vm);
 
@@ -60,6 +61,8 @@ int main(int argc, char** argv)
 
   int recv_timeout = vm["timeout"].as();
 
+  const bool quitOnAck = vm["quit-on-ack"].as();
+
   collector.set(zmq::sockopt::rcvtimeo, recv_timeout * 1000);
 
   int pub_port = vm["file-port"].as(), col_port = vm["ack-port"].as();
@@ -112,6 +115,9 @@ int main(int argc, char** argv)
     ans.assign(ack.to_string());
     std::cout << ans << std::endl;
     // end of ack -------------------
+    if (quitOnAck && (ans.find("ok") == (ans.size() - 2))) {
+      break;
+    }
 
     trial++;
   }
diff --git a/Detectors/EMCAL/base/CMakeLists.txt b/Detectors/EMCAL/base/CMakeLists.txt
index 141c08d47f106..62a0cb6d9a25b 100644
--- a/Detectors/EMCAL/base/CMakeLists.txt
+++ b/Detectors/EMCAL/base/CMakeLists.txt
@@ -16,9 +16,10 @@ o2_add_library(EMCALBase
         src/RCUTrailer.cxx
         src/ClusterFactory.cxx
         src/TriggerMappingV2.cxx
+        src/NonlinearityHandler.cxx
         PUBLIC_LINK_LIBRARIES O2::CommonDataFormat O2::Headers Boost::serialization
         O2::MathUtils O2::DataFormatsEMCAL
-        O2::SimulationDataFormat ROOT::Physics)
+        O2::SimulationDataFormat)
 
 o2_target_root_dictionary(EMCALBase
         HEADERS include/EMCALBase/GeometryBase.h
@@ -29,6 +30,7 @@ o2_target_root_dictionary(EMCALBase
         include/EMCALBase/RCUTrailer.h
         include/EMCALBase/TriggerMappingV2.h
         include/EMCALBase/ClusterFactory.h
+        include/EMCALBase/NonlinearityHandler.h
 )
 
 o2_data_file(COPY files DESTINATION Detectors/EMC)
@@ -40,6 +42,13 @@ o2_add_test(Mapper
         LABELS emcal
         ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage)
 
+o2_add_test(Geometry
+        SOURCES test/testGeometry.cxx
+        PUBLIC_LINK_LIBRARIES O2::EMCALBase
+        COMPONENT_NAME emcal
+        LABELS emcal
+        ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage)
+
 o2_add_test(RCUTrailer
         SOURCES test/testRCUTrailer.cxx
         PUBLIC_LINK_LIBRARIES O2::EMCALBase
diff --git a/Detectors/EMCAL/base/include/EMCALBase/ClusterFactory.h b/Detectors/EMCAL/base/include/EMCALBase/ClusterFactory.h
index 683318029c6a8..0c3438042ca77 100644
--- a/Detectors/EMCAL/base/include/EMCALBase/ClusterFactory.h
+++ b/Detectors/EMCAL/base/include/EMCALBase/ClusterFactory.h
@@ -11,6 +11,8 @@
 #ifndef ALICEO2_EMCAL_CLUSTERFACTORY_H_
 #define ALICEO2_EMCAL_CLUSTERFACTORY_H_
 #include 
+#include 
+#include 
 #include 
 #include "Rtypes.h"
 #include "fmt/format.h"
@@ -18,6 +20,8 @@
 #include "DataFormatsEMCAL/Digit.h"
 #include "DataFormatsEMCAL/Cell.h"
 #include "DataFormatsEMCAL/AnalysisCluster.h"
+#include "DataFormatsEMCAL/CellLabel.h"
+#include "DataFormatsEMCAL/ClusterLabel.h"
 #include "EMCALBase/Geometry.h"
 #include "MathUtils/Cartesian.h"
 
@@ -187,7 +191,7 @@ class ClusterFactory
 
   ///
   /// Dummy constructor
-  ClusterFactory();
+  ClusterFactory() = default;
 
   ///
   /// \brief Constructor initializing the ClusterFactory
@@ -229,7 +233,7 @@ class ClusterFactory
 
   ///
   /// evaluates cluster parameters: position, shower shape, primaries ...
-  AnalysisCluster buildCluster(int index) const;
+  AnalysisCluster buildCluster(int index, o2::emcal::ClusterLabel* clusterLabel = nullptr) const;
 
   void SetECALogWeight(Float_t w) { mLogWeight = w; }
   float GetECALogWeight() const { return mLogWeight; }
@@ -266,7 +270,28 @@ class ClusterFactory
   /// \return the index of the cells with max enegry
   /// \return the maximum energy
   /// \return the total energy of the cluster
-  std::tuple getMaximalEnergyIndex(gsl::span inputsIndices) const;
+  /// \return if cluster is shared between super models
+  std::tuple getMaximalEnergyIndex(gsl::span inputsIndices) const;
+
+  /// \brief Look to cell neighbourhood and reject if it seems exotic
+  /// \param towerId: tower ID of cell with largest energy fraction in cluster
+  /// \param ecell: energy of the cell with largest energy fraction in cluster
+  /// \param exoticTime: time of the cell with largest energy fraction in cluster
+  /// \param fCross: exoticity parameter (1-E_cross/E_cell^max) will be caluclated for this check
+  /// \return bool true if cell is found exotic
+  bool isExoticCell(short towerId, float ecell, float const exoticTime, float& fCross) const;
+
+  /// \brief Calculate the energy in the cross around the energy of a given cell.
+  /// \param absID: controlled cell absolute ID number
+  /// \param energy: cluster or cell max energy, used for weight calculation
+  /// \param exoticTime time of the cell with largest energy fraction in cluster
+  /// \return the energy in the cross around the energy of a given cell
+  float getECross(short absID, float energy, float const exoticTime) const;
+
+  /// \param eCell: cluster cell energy
+  /// \param eCluster: cluster or cell max energy
+  /// \return weight of cell for shower shape calculation
+  float GetCellWeight(float eCell, float eCluster) const;
 
   ///
   /// Calculates the multiplicity of digits/cells with energy larger than level*energy
@@ -290,22 +315,46 @@ class ClusterFactory
   /// \param  key: = 0(gamma, default); !=  0(electron)
   Double_t tMaxInCm(const Double_t e = 0.0, const int key = 0) const;
 
+  bool getLookUpInit() const { return mLookUpInit; }
+
   bool getCoreRadius() const { return mCoreRadius; }
   void setCoreRadius(float radius) { mCoreRadius = radius; }
 
-  void setClustersContainer(gsl::span clusterContainer)
-  {
-    mClustersContainer = clusterContainer;
-  }
+  float getExoticCellFraction() const { return mExoticCellFraction; }
+  void setExoticCellFraction(float exoticCellFraction) { mExoticCellFraction = exoticCellFraction; }
+
+  float getExoticCellDiffTime() const { return mExoticCellDiffTime; }
+  void setExoticCellDiffTime(float exoticCellDiffTime) { mExoticCellDiffTime = exoticCellDiffTime; }
+
+  float getExoticCellMinAmplitude() const { return mExoticCellMinAmplitude; }
+  void setExoticCellMinAmplitude(float exoticCellMinAmplitude) { mExoticCellMinAmplitude = exoticCellMinAmplitude; }
+
+  float getExoticCellInCrossMinAmplitude() const { return mExoticCellInCrossMinAmplitude; }
+  void setExoticCellInCrossMinAmplitude(float exoticCellInCrossMinAmplitude) { mExoticCellInCrossMinAmplitude = exoticCellInCrossMinAmplitude; }
 
-  void setCellsContainer(gsl::span cellContainer)
+  bool getUseWeightExotic() const { return mUseWeightExotic; }
+  void setUseWeightExotic(float useWeightExotic) { mUseWeightExotic = useWeightExotic; }
+
+  void setContainer(gsl::span clusterContainer, gsl::span cellContainer, gsl::span indicesContainer, gsl::span cellLabelContainer = {})
   {
+    mClustersContainer = clusterContainer;
     mInputsContainer = cellContainer;
+    mCellsIndices = indicesContainer;
+    if (!getLookUpInit()) {
+      setLookUpTable();
+    }
+    if (!cellLabelContainer.empty()) {
+      mCellLabelContainer = cellLabelContainer;
+    }
   }
 
-  void setCellsIndicesContainer(gsl::span indicesContainer)
+  void setLookUpTable(void)
   {
-    mCellsIndices = indicesContainer;
+    mLoolUpTowerToIndex.fill(-1);
+    for (auto iCellIndex : mCellsIndices) {
+      mLoolUpTowerToIndex[mInputsContainer[iCellIndex].getTower()] = iCellIndex;
+    }
+    mLookUpInit = true;
   }
 
   int getNumberOfClusters() const
@@ -317,6 +366,21 @@ class ClusterFactory
   /// \param geometry EMCAL geometry
   void setGeometry(o2::emcal::Geometry* geometry) { mGeomPtr = geometry; }
 
+  /// \class UninitLookUpTableException
+  /// \brief Exception handling uninitialized look up table
+  class UninitLookUpTableException final : public std::exception
+  {
+   public:
+    /// \brief constructor
+    UninitLookUpTableException() = default;
+
+    /// \brief Destructor
+    ~UninitLookUpTableException() noexcept final = default;
+
+    /// \brief Access to error message of the exception
+    const char* what() const noexcept final { return "Lookup table not initialized, exotics evaluation not possible!"; }
+  };
+
  protected:
   ///
   /// This function calculates energy in the core,
@@ -337,6 +401,10 @@ class ClusterFactory
   /// in cell units
   void evalElipsAxis(gsl::span inputsIndices, AnalysisCluster& clusterAnalysis) const;
 
+  ///
+  /// Calculate the number of local maxima in the cluster
+  void evalNExMax(gsl::span inputsIndices, AnalysisCluster& clusterAnalysis) const;
+
   ///
   /// Time is set to the time of the digit with the maximum energy
   void evalTime(gsl::span inputsIndices, AnalysisCluster& clusterAnalysis) const;
@@ -357,18 +425,26 @@ class ClusterFactory
   float mLogWeight = 4.5; ///<  logarithmic weight for the cluster center of gravity calculation
 
   bool mJustCluster = kFALSE; ///< Flag to evaluates local to "tracking" c.s. transformation (B.P.).
-
-  mutable int mSuperModuleNumber = 0; ///<  number identifying supermodule containing cluster, reference is cell with maximum energy.
-  float mDistToBadTower = -1;         ///<  Distance to nearest bad tower
-  bool mSharedCluster = false;        ///<  States if cluster is shared by 2 SuperModules in same phi rack (0,1), (2,3) ... (10,11).
-
-  gsl::span mClustersContainer; ///< Container for all the clusters in the event
-  gsl::span mInputsContainer;            ///< Container for all the cells/digits in the event
-  gsl::span mCellsIndices;                     ///< Container for cells indices in the event
+  bool mLookUpInit = false;   ///< Flag to check if the mLoolUpTowerToIndex is currently set. Will be checked when needed and created if not set!
+
+  mutable int mSuperModuleNumber = 0;         ///<  number identifying supermodule containing cluster, reference is cell with maximum energy.
+  float mDistToBadTower = -1;                 ///<  Distance to nearest bad tower
+  bool mSharedCluster = false;                ///<  States if cluster is shared by 2 SuperModules in same phi rack (0,1), (2,3) ... (10,11).
+  float mExoticCellFraction = 0.97;           ///<  Good cell if fraction < 1-ecross/ecell
+  float mExoticCellDiffTime = 1e6;            ///<  If time of candidate to exotic and close cell is too different (in ns), it must be noisy, set amp to 0
+  float mExoticCellMinAmplitude = 4.;         ///<  Check for exotic only if amplitud is larger than this value
+  float mExoticCellInCrossMinAmplitude = 0.1; ///<  Minimum energy of cells in cross, if lower not considered in cross
+  bool mUseWeightExotic = false;              ///<  States if weights should be used for exotic cell cut
+
+  gsl::span mClustersContainer;    ///< Container for all the clusters in the event
+  gsl::span mInputsContainer;               ///< Container for all the cells/digits in the event
+  gsl::span mCellsIndices;                        ///< Container for cells indices in the event
+  std::array mLoolUpTowerToIndex;              ///< Lookup table to match tower id with cell index, needed for exotic check
+  gsl::span mCellLabelContainer; ///< Container for all the cell labels in the event
 
   ClassDefNV(ClusterFactory, 2);
 };
 
 } // namespace emcal
 } // namespace o2
-#endif // ALICEO2_EMCAL_CLUSTERFACTORY_H_
\ No newline at end of file
+#endif // ALICEO2_EMCAL_CLUSTERFACTORY_H_
diff --git a/Detectors/EMCAL/base/include/EMCALBase/Geometry.h b/Detectors/EMCAL/base/include/EMCALBase/Geometry.h
index 5be8095a4eb7a..d07f42689bf7a 100644
--- a/Detectors/EMCAL/base/include/EMCALBase/Geometry.h
+++ b/Detectors/EMCAL/base/include/EMCALBase/Geometry.h
@@ -12,14 +12,16 @@
 #ifndef ALICEO2_EMCAL_GEOMETRY_H_
 #define ALICEO2_EMCAL_GEOMETRY_H_
 
-#include 
+#include 
 #include 
+#include 
 #include 
 #include 
 
 #include 
+#include 
 #include 
-#include 
+#include 
 #include 
 #include 
 
@@ -57,7 +59,7 @@ class Geometry
   /// | EMCAL_COMPLETE12SMV1_DCAL             | Full EMCAL, 10 DCAL Supermodules (not used in practice)  |
   /// | EMCAL_COMPLETE12SMV1_DCAL_8SM         | Full EMCAL, 8 DCAL Supermodules (run2)                   |
   /// | EMCAL_COMPLETE12SMV1_DCAL_DEV         | Full EMCAL, DCAL development geometry (not used)         |
-  Geometry(const std::string_view name, const std::string_view mcname = "", const std::string_view mctitle = "");
+  explicit Geometry(const std::string_view name, const std::string_view mcname = "", const std::string_view mctitle = "");
 
   /// \brief Copy constructor.
   Geometry(const Geometry& geom);
@@ -70,6 +72,7 @@ class Geometry
 
   /// \brief Get geometry instance. It should have been set before.
   /// \return the pointer of the unique instance of the geometry
+  /// \throw GeometryNotInitializedException in case the geometry is not initialized
   static Geometry* GetInstance();
 
   /// \brief Get instance of the EMCAL geometry
@@ -426,6 +429,14 @@ class Geometry
   /// \return Position (0 - phi, 1 - eta) of the cell inside teh supermodule
   std::tuple GetCellPhiEtaIndexInSModule(int supermoduleID, int moduleID, int phiInModule, int etaInModule) const;
 
+  /// \brief Get topological row and column of cell in SM (same as for clusteriser with artifical gaps)
+  /// \param supermoduleID super module number
+  /// \param moduleID module number
+  /// \param phiInModule index in phi direction in module
+  /// \param etaInModule index in phi direction in module
+  /// \return tuple with (row, column) of the cell, which is global numbering scheme
+  std::tuple GetTopologicalRowColumn(int supermoduleID, int moduleID, int phiInModule, int etaInModule) const;
+
   /// \brief Adapt cell indices in supermodule to online indexing
   /// \param supermoduleID super module number of the channel/cell
   /// \param iphi row/phi cell index, modified for DCal
@@ -512,13 +523,21 @@ class Geometry
   /// \return col
   std::tuple getOnlineID(int towerID);
 
+  ///  \brief Check if 2 cells belong to the same T-Card
+  ///  \param absId1: Reference absId cell
+  ///  \param absId2: Cross checked cell absId
+  ///  \return true if belong to same TCard else false
+  ///  \return rowDiff: Distance in rows
+  ///  \return colDiff: Distance in columns
+  std::tuple areAbsIDsFromSameTCard(int absId1, int absId2) const;
+
   /// \brief Temporary link assignment (till final link assignment is known -
   /// \brief eventually taken from CCDB)
   /// \brief Current mapping can be found under https://alice.its.cern.ch/jira/browse/EMCAL-660
   /// \param ddlID DDL ID
   /// \return CRORC ID
   /// \return CRORC Link
-  std::tuple getLinkAssignment(int ddlID) const { return std::make_tuple(mCRORCID[ddlID / 2], mCRORCLink[ddlID]); };
+  std::tuple getLinkAssignment(int ddlID) const { return std::make_tuple(mCRORCID[ddlID], mCRORCLink[ddlID]); };
 
   std::vector GetEMCSystem() const { return mEMCSMSystem; } // EMC System, SM type list
   // Local Coordinates of SM
@@ -563,6 +582,11 @@ class Geometry
   ///
   void SetMisalMatrix(const TGeoHMatrix* m, Int_t smod) const;
 
+  ///
+  /// Method to set shift-rotational matrixes from CCDB
+  ///
+  void SetMisalMatrixFromCcdb(const char* path = "Users/m/mhemmer/EMCAL/Config/GeometryAligned", int timestamp = 10000) const;
+
   ///
   /// Transform clusters cell position into global with alternative method, taking into account the depth calculation.
   /// Input are:
@@ -689,8 +713,8 @@ class Geometry
 
   Float_t mSteelFrontThick; ///< Thickness of the front stell face of the support box - 9-sep-04; obsolete?
 
-  std::array mCRORCID = {110, 112, 110, 112, 110, 112, 111, 113, 111, 113, 111, 113, 114, 116, 114, 116, 115, 117, 115, 117};                         // CRORC ID w.r.t SM
-  std::array mCRORCLink = {0, 1, 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 0, 1, 0, 1, 2, 3, 2, 3, 4, -1, 4, 5, 0, 1, 0, 1, 2, 3, 2, 3, 0, 1, 0, 1, 2, 3, 2, -1}; // CRORC limk w.r.t FEE ID
+  std::array mCRORCID = {110, 110, 112, 112, 110, 110, 112, 112, 110, 110, 112, 112, 111, 111, 113, 113, 111, 111, 113, 113, 111, 111, 113, 113, 114, 114, 116, 116, 114, 114, 116, 116, 115, 115, 117, 117, 115, 115, 117, 117, -1, -1, -1, -1, 111, 117}; // CRORC ID w.r.t SM
+  std::array mCRORCLink = {0, 1, 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 0, 1, 0, 1, 2, 3, 2, 3, 4, -1, 4, 5, 0, 1, 0, 1, 2, 3, 2, 3, 0, 1, 0, 1, 2, 3, 2, -1, -1, -1, -1, -1, 5, 3};                                                                                 // CRORC limk w.r.t FEE ID
 
   mutable const TGeoHMatrix* SMODULEMATRIX[EMCAL_MODULES];      ///< Orientations of EMCAL super modules
   std::vector> mCellIndexLookup; ///< Lookup table for cell indices
diff --git a/Detectors/EMCAL/base/include/EMCALBase/GeometryBase.h b/Detectors/EMCAL/base/include/EMCALBase/GeometryBase.h
index 5cba99a3567c0..3fda26cbbfbc6 100644
--- a/Detectors/EMCAL/base/include/EMCALBase/GeometryBase.h
+++ b/Detectors/EMCAL/base/include/EMCALBase/GeometryBase.h
@@ -33,6 +33,23 @@ enum AcceptanceType_t { EMCAL_ACCEPTANCE = 1,
 
 const std::string DEFAULT_GEOMETRY = "EMCAL_COMPLETE12SMV1_DCAL_8SM";
 
+/// \class GeometryNotInitializedException
+/// \brief Error handling access to non-initialized geometry
+/// \ingroup EMCALbase
+class GeometryNotInitializedException final : public std::exception
+{
+ public:
+  /// \brief Constructor
+  GeometryNotInitializedException() = default;
+
+  /// \brief Destructor
+  ~GeometryNotInitializedException() noexcept final = default;
+
+  /// \brief Access to error message
+  /// \return Error message
+  const char* what() const noexcept { return "Geometry not initialized"; }
+};
+
 /// \class InvalidModuleException
 /// \brief Error Handling when an invalid module ID (outside the limits) is called
 /// \ingroup EMCALbase
diff --git a/Detectors/EMCAL/base/include/EMCALBase/Hit.h b/Detectors/EMCAL/base/include/EMCALBase/Hit.h
index 640001a64872d..345a0918c3831 100644
--- a/Detectors/EMCAL/base/include/EMCALBase/Hit.h
+++ b/Detectors/EMCAL/base/include/EMCALBase/Hit.h
@@ -22,6 +22,8 @@ namespace emcal
 /// \class Hit
 /// \brief EMCAL simulation hit information
 /// \ingroup EMCALbase
+/// \author Markus Fasel , Oak Ridge National Laboratory
+/// \since August 31st, 2017
 class Hit : public o2::BasicXYZEHit
 {
  public:
@@ -63,11 +65,6 @@ class Hit : public o2::BasicXYZEHit
   /// \return This point with the summed energy loss
   Hit& operator+=(const Hit& rhs);
 
-  /// \brief Creates a new point base on this point but adding the energy loss of the right hand side
-  /// \param rhs Hit to add to
-  /// \return New EMAL point base on this point
-  Hit operator+(const Hit& rhs) const;
-
   /// \brief Destructor
   ~Hit() = default;
 
@@ -99,6 +96,16 @@ class Hit : public o2::BasicXYZEHit
   ClassDefNV(Hit, 1);
 };
 
+/// \brief Creates a new point base on this point but adding the energy loss of the right hand side
+/// \param lhs Left hand side of the sum
+/// \param rhs Right hand side of the sum
+/// \return New EMAL point base on this point
+Hit operator+(const Hit& lhs, const Hit& rhs);
+
+/// \brief Output stream operator for EMCAL hits
+/// \param stream Stream to write on
+/// \param point Hit to be printed
+/// \return Stream after printing
 std::ostream& operator<<(std::ostream& stream, const Hit& point);
 } // namespace emcal
 } // namespace o2
diff --git a/Detectors/EMCAL/base/include/EMCALBase/Mapper.h b/Detectors/EMCAL/base/include/EMCALBase/Mapper.h
index b8b51b5d54cbb..6c7aaf329fcc3 100644
--- a/Detectors/EMCAL/base/include/EMCALBase/Mapper.h
+++ b/Detectors/EMCAL/base/include/EMCALBase/Mapper.h
@@ -88,8 +88,11 @@ class Mapper
     /// \param address Hardware address raising the exception
     AddressNotFoundException(int address) : exception(),
                                             mAddress(address),
-                                            mMessage("Hardware address " + std::to_string(address) + " not found")
+                                            mMessage()
     {
+      std::stringstream msgbuilder;
+      msgbuilder << "Hardware address " << address << "(0x" << std::hex << address << std::dec << ") not found";
+      mMessage = msgbuilder.str();
     }
 
     /// \brief Destructor
@@ -238,13 +241,13 @@ class Mapper
   /// \return Channel params corresponding to the hardware address
   /// \throw InitStatusException in case the mapping was not properly initialized
   /// \throw AddressNotFoundException in case of invalid hardware address
-  ChannelID getChannelID(int hardawareaddress) const;
+  ChannelID getChannelID(unsigned int hardawareaddress) const;
 
   /// \brief Get channel row for a given hardware address
   /// \return Row corresponding to the hardware address
   /// \throw InitStatusException in case the mapping was not properly initialized
   /// \throw AddressNotFoundException in case of invalid hardware address
-  int getRow(int hardawareaddress) const
+  uint8_t getRow(unsigned int hardawareaddress) const
   {
     return getChannelID(hardawareaddress).mRow;
   }
@@ -253,7 +256,7 @@ class Mapper
   /// \return Column corresponding to the hardware address
   /// \throw InitStatusException in case the mapping was not properly initialized
   /// \throw AddressNotFoundException in case of invalid hardware address
-  int getColumn(int hardawareaddress) const
+  uint8_t getColumn(unsigned int hardawareaddress) const
   {
     return getChannelID(hardawareaddress).mColumn;
   }
@@ -262,7 +265,7 @@ class Mapper
   /// \return Channel type corresponding to the hardware address
   /// \throw InitStatusException in case the mapping was not properly initialized
   /// \throw AddressNotFoundException in case of invalid hardware address
-  ChannelType_t getChannelType(int hardawareaddress) const
+  ChannelType_t getChannelType(unsigned int hardawareaddress) const
   {
     return getChannelID(hardawareaddress).mChannelType;
   }
@@ -273,7 +276,7 @@ class Mapper
   /// \param channeltype type of the channel
   /// \return Harware address of the channel
   /// \throw InitStatusException in case the mapping was not properly initialized
-  int getHardwareAddress(int row, int col, ChannelType_t channeltype) const;
+  unsigned int getHardwareAddress(uint8_t row, uint8_t col, ChannelType_t channeltype) const;
 
   /// \brief Initialize with new
   /// \param inputfile Name of the input file from which to read the mapping
@@ -290,9 +293,9 @@ class Mapper
   /// \throw ChannelTypeException in case hardware address with unkknown channel types are found
   void init(const std::string_view inputfile);
 
-  std::unordered_map mMapping;                         ///< Mapping between hardware address and col / row /caloflag
-  std::unordered_map mInverseMapping; ///< Inverse Mapping of channel type to hardware address
-  bool mInitStatus = false;                                            ///< Initialization status
+  std::unordered_map mMapping;                         ///< Mapping between hardware address and col / row /caloflag
+  std::unordered_map mInverseMapping; ///< Inverse Mapping of channel type to hardware address
+  bool mInitStatus = false;                                                     ///< Initialization status
 
   ClassDefNV(Mapper, 1);
 };
diff --git a/Detectors/EMCAL/base/include/EMCALBase/NonlinearityHandler.h b/Detectors/EMCAL/base/include/EMCALBase/NonlinearityHandler.h
new file mode 100644
index 0000000000000..b3dd3bd111835
--- /dev/null
+++ b/Detectors/EMCAL/base/include/EMCALBase/NonlinearityHandler.h
@@ -0,0 +1,296 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#ifndef ALICEO2_EMCAL_NONLINEARITYHANDLER__H
+#define ALICEO2_EMCAL_NONLINEARITYHANDLER__H
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include "DataFormatsEMCAL/AnalysisCluster.h"
+
+namespace o2::emcal
+{
+
+/// \class NonlinearityHandler
+/// \brief Nonlinearity functions for energy correction
+/// \ingroup EMCALbase
+/// \author Markus Fasel , Oak Ridge National Laboratory
+/// \since Feb 27, 2023
+///
+/// Calculating a corrected cluster energy based on the raw cluster energy.
+/// Several parameterisations are provided. The function is selected during
+/// construction of the object. The corrected cluster energy is obtained via
+/// the function getCorrectedClusterEnergy.
+///
+/// The correction for the shaper sturation must be applied at cell energy level.
+/// Only one parameterisation for the shaper nonlinearity exists, for which the
+/// parameterisation does not depend on the type of the cluster nonlinearity. The
+/// function evaluateShaperCorrectionCellEnergy is static and can therefore be applied
+/// without a cluster nonlinearity parameterisation.
+///
+/// based on nonlinearity implementation in AliEMCALRecoUtils
+class NonlinearityHandler
+{
+ public:
+  /// \class UninitException
+  /// \brief Handling missing initialisation of the NonlinearityHanlder
+  class UninitException : public std::exception
+  {
+   public:
+    /// \brief Constructor
+    UninitException() = default;
+
+    /// \brief Destructor
+    ~UninitException() noexcept final = default;
+
+    /// \brief Provide error message
+    /// \return Error message
+    const char* what() const noexcept final
+    {
+      return "Nonlinearity handler not initialized";
+    }
+  };
+
+  /// \enum NonlinType_t
+  /// \brief Types of nonlinearity functions available
+  enum class NonlinType_t {
+    DATA_TESTBEAM_SHAPER,         ///< Data, testbeam nonlin for shaper correction, with energy rescaling
+    DATA_TESTBEAM_SHAPER_WOSCALE, ///< Data, testbeam nonlin for shaper correction, without energy rescaling
+    DATA_TESTBEAM_CORRECTED,      ///< Data, inital testbeam nonlin without shaper correction
+    DATA_TESTBEAM_CORRECTED_V2,   ///< Data, inital testbeam nonlin without shaper correction, version 2
+    DATA_TESTBEAM_CORRECTED_V3,   ///< Data, inital testbeam nonlin without shaper correction, version 3
+    DATA_TESTBEAM_CORRECTED_V4,   ///< Data, inital testbeam nonlin without shaper correction, version 4
+    MC_TESTBEAM_FINAL,            ///< MC, function corresponding to data testbeam nonlin for shaper correction, with energy rescaling
+    MC_PI0,                       ///< MC, function corresponding to inital testbeam nonlin without shaper correction
+    MC_PI0_V2,                    ///< MC, function corresponding to inital testbeam nonlin without shaper correction, version 2
+    MC_PI0_V3,                    ///< MC, function corresponding to inital testbeam nonlin without shaper correction, version 3
+    MC_PI0_V5,                    ///< MC, function corresponding to inital testbeam nonlin without shaper correction, version 5
+    MC_PI0_V6,                    ///< MC, function corresponding to inital testbeam nonlin without shaper correction, version 6
+    UNKNOWN                       ///< No function set
+  };
+
+  /// \brief Get type of a nonlinearity function from its name
+  /// \param name Name of the nonlinearity function
+  /// \return Nonlinearity function type
+  static NonlinType_t getNonlinType(const std::string_view name);
+
+  /// \brief Get name of the nonlinearity function from the function type
+  /// \param nonlin Type of the nonlinearity function
+  /// \return Name of the nonlinearity function
+  static const char* getNonlinName(NonlinType_t nonlin);
+
+  /// \brief Dummy constructor
+  ///
+  /// Non-linearity type not set, will fail when trying to evaluate
+  /// for a certain energy. Use constructor with nonlinearity function
+  /// specified instead. Only intended for constructing standard containers.
+  NonlinearityHandler() = default;
+
+  /// \brief Constructor, defining nonlinearity function
+  /// \param nonlintype Type of t, he nonlinearity function
+  ///
+  /// Initializing all parameters and settings of the nonlinearity function.
+  /// Nonlinearity correction at cluster level can be obtained using
+  /// objects constructed by this.
+  NonlinearityHandler(NonlinType_t nonlintype);
+
+  /// \brief Destructor
+  ~NonlinearityHandler() = default;
+
+  /// \brief Set type of nonlinearity function
+  /// \param nonlintype Type of nonlinearity function
+  ///
+  /// Updating also function parameters
+  void setNonlinType(NonlinType_t nonlintype)
+  {
+    mNonlinearyFunction = nonlintype;
+    initParams();
+  }
+
+  /// \brief Get corrected cluster energy for the selected nonlinearity parameterization
+  /// \param energy Raw cluster energy
+  /// \return Corrected cluster energy
+  /// \throw UninitException in case the NonlinearityHandler is not configured
+  double getCorrectedClusterEnergy(double energy) const;
+
+  /// \brief Get corrected cluster energy for the selected nonlinearity parameterization
+  /// \param energy Raw cluster energy
+  /// \return Corrected cluster energy
+  /// \throw UninitException in case the NonlinearityHandler is not configured
+  double getCorrectedClusterEnergy(const AnalysisCluster& cluster) const { return getCorrectedClusterEnergy(cluster.E()); }
+
+  /// \brief Get corrected energy at cell level for the shaper saturation at high energy
+  /// \param energy Raw cell energy
+  /// \param ecalibHG Finetuning of the high-gain energy scale
+  static double evaluateShaperCorrectionCellEnergy(double energy, double ecalibHG = 1);
+
+  /// \brief Print information about the nonlinearity function
+  /// \param stream Stream to print the information on
+  void printStream(std::ostream& stream) const;
+
+ private:
+  NonlinType_t mNonlinearyFunction = NonlinType_t::UNKNOWN; ///< Nonlinearity function
+  bool mApplyScaleCorrection = false;                       ///< Scale correction
+  std::array mNonlinearityParam;                ///< Storage for params used in the function evaluation
+
+  /// \brief Classical model, before testbeam reanalysis, data and most MC parameterisations
+  /// \param energy Raw cluster energy
+  /// \return Corrected energy
+  double evaluateTestbeamCorrected(double energy) const;
+
+  /// \brief New model after testbeam reanalysis, data and MC
+  /// \param energy Raw cluster energy
+  /// \return Corrected energy
+  double evaluateTestbeamShaper(double energy) const;
+
+  /// \brief Model for nonlin based on MC pi0 analysis, initial version
+  /// \param energy Raw cluster energy
+  /// \return Corrected energy
+  double evaluatePi0MC(double energy) const;
+
+  /// \brief Model for nonlin based on MC pi0 analysis, version 2
+  /// \param energy Raw cluster energy
+  /// \return Corrected energy
+  double evaluatePi0MCv2(double energy) const;
+
+  /// \brief Initialise params of the nonlinearity function
+  ///
+  /// Initialising all params corresponding to the fits of the various
+  /// nonlinearity functions.
+  void initParams();
+
+  ClassDefNV(NonlinearityHandler, 1)
+};
+
+/// \class NonlinearityFactory
+/// \brief Creator and handler class of nonlinearity parameterizations
+/// \ingroup EMCALbase
+/// \author Markus Fasel , Oak Ridge National Laboratory
+/// \since Feb 27, 2023
+///
+/// Factory and manager of nonlinearty handlers. The class acts as singleton class.
+/// NonlinearityHandlers can be constructed via the function getNonlinearity
+/// either using either the symbolic form or using the name as a string representation.
+/// In the second case an exception is thrown in case the request is done for a funciton
+/// name which doesn't exist.
+class NonlinearityFactory
+{
+ public:
+  /// \class FunctionNotFoundExcpetion
+  /// \brief Handling request of non-exisiting nonlinearity functions
+  class FunctionNotFoundExcpetion : public std::exception
+  {
+   public:
+    /// \brief Constructor
+    /// \param Name of the nonlinearity function raising the exception
+    FunctionNotFoundExcpetion(const std::string_view name) : mName(name), mMessage()
+    {
+      mMessage = "Nonlinearity funciton " + mName + " not found";
+    }
+
+    /// \brief Destructor
+    ~FunctionNotFoundExcpetion() noexcept final = default;
+
+    /// \brief Provide error message
+    /// \return Error message
+    const char* what() const noexcept final
+    {
+      return mMessage.data();
+    }
+
+    /// \brief Getting the name of the function raising the exception
+    /// \return Name of the function raising the exception
+    const std::string_view getNonlinName() const noexcept { return mName.data(); }
+
+   private:
+    std::string mName;    ///< Name of the requested function
+    std::string mMessage; ///< Error message
+  };
+
+  /// \class NonlinInitError
+  /// \brief Handling errors of initialisation of a certain nonlinearity function
+  class NonlinInitError : public std::exception
+  {
+   public:
+    /// \brief Constructor
+    NonlinInitError() {}
+
+    /// \brief Destructor
+    ~NonlinInitError() noexcept final = default;
+
+    /// \brief Provide error message
+    /// \return Error message
+    const char* what() const noexcept final
+    {
+      return "Failed constructing nonlinearity handler";
+    }
+  };
+
+  /// \brief Get instance of the nonlinearity factory
+  /// \return Factory instance
+  static NonlinearityFactory& getInstance()
+  {
+    static NonlinearityFactory currentInstance;
+    return currentInstance;
+  }
+
+  /// \brief Get nonlinearity handler for the given nonlinearty type
+  /// \return Nonlineary handler for the given nonlinearity function
+  /// \throw NonlinInitError in case the object could not be constructed
+  ///
+  /// Internally caching existing nonlinearty handlers. Only constructing
+  /// handler in case the handler is not yet existing.
+  NonlinearityHandler& getNonlinearity(NonlinearityHandler::NonlinType_t nonlintype);
+
+  /// \brief Get nonlinearity handler for the given nonlinearty name
+  /// \return Nonlineary handler for the given nonlinearity function based on its name
+  /// \throw FunctionNotFoundExcpetion in case the name of the function is unknown
+  /// \throw NonlinInitError in case the object could not be constructed
+  ///
+  /// Internally caching existing nonlinearty handlers. Only constructing
+  /// handler in case the handler is not yet existing.
+  NonlinearityHandler& getNonlinearity(const std::string_view nonlintype);
+
+ private:
+  /// \brief Constructor
+  ///
+  /// Initialising lookup of function types for function names
+  NonlinearityFactory() { initNonlinNames(); }
+
+  /// \brief Destructor
+  ~NonlinearityFactory() = default;
+
+  /// \brief Initialise lookup table with string representation of nonlinearity functions
+  void initNonlinNames();
+
+  /// \brief Find type of nonlinearity function from a function name (string representation)
+  /// \param nonlinName Name of the nonlinearity function
+  /// \return Type of the nonlinearity function
+  /// \throw FunctionNotFoundExcpetion in case no function type can be found for the given name
+  NonlinearityHandler::NonlinType_t getNonlinType(const std::string_view nonlinName) const;
+
+  std::unordered_map mHandlers; ///< Map with nonlinearity handlers for given nonlinearity functions
+  std::unordered_map mNonlinNames;      ///< Lookup table with nonlinearity types for given nonlinearity names
+
+  ClassDefNV(NonlinearityFactory, 1)
+};
+
+/// \brief Output streaming operator for the NonlinearityHander
+/// \param in Stream to print on
+/// \param handler NonlinearityHander to be displayed
+/// \return Stream after printing
+std::ostream& operator<<(std::ostream& in, const NonlinearityHandler& handler);
+
+} // namespace o2::emcal
+
+#endif // ALICEO2_EMCAL_NONLINEARITYHANDLER__H
\ No newline at end of file
diff --git a/Detectors/EMCAL/base/include/EMCALBase/RCUTrailer.h b/Detectors/EMCAL/base/include/EMCALBase/RCUTrailer.h
index 8723a7a68264b..0fcb87f20f2ff 100644
--- a/Detectors/EMCAL/base/include/EMCALBase/RCUTrailer.h
+++ b/Detectors/EMCAL/base/include/EMCALBase/RCUTrailer.h
@@ -153,6 +153,10 @@ class RCUTrailer
   /// \return Size of the payload as number of 32 bit workds
   uint32_t getPayloadSize() const { return mPayloadSize; }
 
+  /// \brief Get number of corrupted trailer words (undefined trailer word code)
+  /// \return Number of trailer word corruptions
+  uint32_t getTrailerWordCorruptions() const { return mWordCorruptions; }
+
   /// \brief Get the firmware version
   /// \return Firmware version
   uint8_t getFirmwareVersion() const { return mFirmwareVersion; }
@@ -384,6 +388,11 @@ class RCUTrailer
   /// are assigned based on the trailer word marker.
   static RCUTrailer constructFromPayloadWords(const gsl::span payloadwords);
 
+  /// \brief Check whether the word is a valid last trailer word
+  /// \param trailerword Word to be checked
+  /// \return True if the word is a valid last trailer word, false if there are inconsistencies
+  static bool checkLastTrailerWord(uint32_t trailerword);
+
  private:
   /// \struct AltroConfig
   /// \brief Bit field configuration of the ALTRO config registers
@@ -438,6 +447,7 @@ class RCUTrailer
   uint8_t mFirmwareVersion = 0;         ///< RCU firmware version
   uint32_t mTrailerSize = 0;            ///< Size of the trailer (in number of 32 bit words)
   uint32_t mPayloadSize = 0;            ///< Size of the payload (in nunber of 32 bit words)
+  uint32_t mWordCorruptions = 0;        ///< Number of trailer word corruptions (decoding only)
   uint32_t mFECERRA = 0;                ///< contains errors related to ALTROBUS transactions
   uint32_t mFECERRB = 0;                ///< contains errors related to ALTROBUS transactions
   ErrorCounters mErrorCounter = {0, 0}; ///< Error counter registers
diff --git a/Detectors/EMCAL/base/include/EMCALBase/ShishKebabTrd1Module.h b/Detectors/EMCAL/base/include/EMCALBase/ShishKebabTrd1Module.h
index 2e96f1038fef8..43f7df3a5757c 100644
--- a/Detectors/EMCAL/base/include/EMCALBase/ShishKebabTrd1Module.h
+++ b/Detectors/EMCAL/base/include/EMCALBase/ShishKebabTrd1Module.h
@@ -19,7 +19,7 @@
 #include 
 #include 
 
-#include 
+#include 
 
 namespace o2
 {
diff --git a/Detectors/EMCAL/base/include/EMCALBase/TriggerMappingV2.h b/Detectors/EMCAL/base/include/EMCALBase/TriggerMappingV2.h
index 3b56c1794d3e4..24d8c1686f005 100644
--- a/Detectors/EMCAL/base/include/EMCALBase/TriggerMappingV2.h
+++ b/Detectors/EMCAL/base/include/EMCALBase/TriggerMappingV2.h
@@ -49,6 +49,7 @@ class TriggerMappingV2
   static constexpr unsigned int FASTORSPHI = (5 * FASTORSPHISM) + (1 * FASTORSPHISM / 3)              /*EMCAL*/
                                              + (3 * FASTORSPHISM) + (1 * FASTORSPHISM / 3) /*DCAL */; ///< Number of FastOR/EMCALs in Phi
   static constexpr unsigned int ALLFASTORS = FASTORSETA * FASTORSPHI;                                 ///< Number of FastOR/EMCALs
+  static constexpr unsigned int PATCHESINTRU = 77;
 
   //********************************************
   // Index types
diff --git a/Detectors/EMCAL/base/src/ClusterFactory.cxx b/Detectors/EMCAL/base/src/ClusterFactory.cxx
index d1667591437c5..1752e5c0e98ee 100644
--- a/Detectors/EMCAL/base/src/ClusterFactory.cxx
+++ b/Detectors/EMCAL/base/src/ClusterFactory.cxx
@@ -18,6 +18,8 @@
 #include "DataFormatsEMCAL/Cell.h"
 #include "DataFormatsEMCAL/AnalysisCluster.h"
 #include "DataFormatsEMCAL/Constants.h"
+#include "DataFormatsEMCAL/CellLabel.h"
+#include "DataFormatsEMCAL/ClusterLabel.h"
 #include "EMCALBase/Geometry.h"
 #include "MathUtils/Cartesian.h"
 
@@ -25,20 +27,10 @@
 
 using namespace o2::emcal;
 
-template 
-ClusterFactory::ClusterFactory()
-{
-  mGeomPtr = o2::emcal::Geometry::GetInstance();
-}
-
 template 
 ClusterFactory::ClusterFactory(gsl::span clustersContainer, gsl::span inputsContainer, gsl::span cellsIndices)
 {
-  mGeomPtr = o2::emcal::Geometry::GetInstance();
-
-  setClustersContainer(clustersContainer);
-  setCellsContainer(inputsContainer);
-  setCellsIndicesContainer(cellsIndices);
+  setContainer(clustersContainer, inputsContainer, cellsIndices);
 }
 
 template 
@@ -47,13 +39,15 @@ void ClusterFactory::reset()
   mClustersContainer = gsl::span();
   mInputsContainer = gsl::span();
   mCellsIndices = gsl::span();
+  mLookUpInit = false;
+  mCellLabelContainer = gsl::span();
 }
 
 ///
 /// evaluates cluster parameters: position, shower shape, primaries ...
 //____________________________________________________________________________
 template 
-o2::emcal::AnalysisCluster ClusterFactory::buildCluster(int clusterIndex) const
+o2::emcal::AnalysisCluster ClusterFactory::buildCluster(int clusterIndex, o2::emcal::ClusterLabel* clusterLabel) const
 {
   if (clusterIndex >= mClustersContainer.size()) {
     throw ClusterRangeException(clusterIndex, mClustersContainer.size());
@@ -73,20 +67,47 @@ o2::emcal::AnalysisCluster ClusterFactory::buildCluster(int clusterIn
   // First calculate the index of input with maximum amplitude and get
   // the supermodule number where it sits.
 
-  auto [inputIndMax, inputEnergyMax, cellAmp] = getMaximalEnergyIndex(inputsIndices);
+  auto [inputIndMax, inputEnergyMax, cellAmp, shared] = getMaximalEnergyIndex(inputsIndices);
+
+  short towerId = mInputsContainer[inputIndMax].getTower();
+
+  float exoticTime = mInputsContainer[inputIndMax].getTimeStamp();
+
+  float fCross = 0.;
+
+  try {
+    clusterAnalysis.setIsExotic(isExoticCell(towerId, inputEnergyMax, exoticTime, fCross));
+    clusterAnalysis.setFCross(fCross);
+  } catch (UninitLookUpTableException& e) {
+    LOG(error) << e.what();
+  }
 
   clusterAnalysis.setIndMaxInput(inputIndMax);
 
   clusterAnalysis.setE(cellAmp);
 
-  mSuperModuleNumber = mGeomPtr->GetSuperModuleNumber(mInputsContainer[inputIndMax].getTower());
+  mSuperModuleNumber = mGeomPtr->GetSuperModuleNumber(towerId);
 
   clusterAnalysis.setNCells(inputsIndices.size());
 
   std::vector cellsIdices;
 
+  bool addClusterLabels = ((clusterLabel != nullptr) && (mCellLabelContainer.size() > 0));
   for (auto cellIndex : inputsIndices) {
     cellsIdices.push_back(cellIndex);
+    if (addClusterLabels) {
+      for (size_t iLabel = 0; iLabel < mCellLabelContainer[cellIndex].GetLabelSize(); iLabel++) {
+        if (mCellLabelContainer[cellIndex].GetAmplitudeFraction(iLabel) <= 0.f) {
+          continue; // skip 0 entries
+        }
+        clusterLabel->addValue(mCellLabelContainer[cellIndex].GetLabel(iLabel),
+                               mCellLabelContainer[cellIndex].GetAmplitudeFraction(iLabel) * mInputsContainer[cellIndex].getEnergy());
+      }
+    }
+  }
+  if (addClusterLabels) {
+    clusterLabel->orderLabels();
+    clusterLabel->normalize(cellAmp);
   }
 
   clusterAnalysis.setCellsIndices(cellsIdices);
@@ -99,17 +120,20 @@ o2::emcal::AnalysisCluster ClusterFactory::buildCluster(int clusterIn
   evalElipsAxis(inputsIndices, clusterAnalysis);
   evalDispersion(inputsIndices, clusterAnalysis);
 
+  // evaluate number of local maxima
+  evalNExMax(inputsIndices, clusterAnalysis);
+
   evalCoreEnergy(inputsIndices, clusterAnalysis);
   evalTime(inputsIndices, clusterAnalysis);
 
   // TODO to be added at a later stage
-  //evalPrimaries(inputsIndices, clusterAnalysis);
-  //evalParents(inputsIndices, clusterAnalysis);
+  // evalPrimaries(inputsIndices, clusterAnalysis);
+  // evalParents(inputsIndices, clusterAnalysis);
 
   // TODO to be added at a later stage
   // Called last because it sets the global position of the cluster?
   // Do not call it when recalculating clusters out of standard reconstruction
-  //if (!mJustCluster)
+  // if (!mJustCluster)
   //  evalLocal2TrackingCSTransform();
 
   return clusterAnalysis;
@@ -215,7 +239,7 @@ void ClusterFactory::evalLocalPosition(gsl::span inputsInd
       continue;
     }
 
-    //Temporal patch, due to mapping problem, need to swap "y" in one of the 2 SM, although no effect in position calculation. GCB 05/2010
+    // Temporal patch, due to mapping problem, need to swap "y" in one of the 2 SM, although no effect in position calculation. GCB 05/2010
     if (mSharedCluster && mSuperModuleNumber != mGeomPtr->GetSuperModuleNumber(mInputsContainer[iInput].getTower())) {
       xyzi[1] *= -1;
     }
@@ -235,7 +259,7 @@ void ClusterFactory::evalLocalPosition(gsl::span inputsInd
         clRmsXYZ[i] += (w * xyzi[i] * xyzi[i]);
       }
     } // w > 0
-  }   // dig loop
+  } // dig loop
 
   //  cout << " wtot " << wtot << endl;
 
@@ -372,7 +396,7 @@ void ClusterFactory::evalLocalPositionFit(double deff, double mLogWei
         clRmsXYZ[i] += (w * xyzi[i] * xyzi[i]);
       }
     }
-  } //loop
+  } // loop
 
   //  cout << " wtot " << wtot << endl;
 
@@ -407,7 +431,7 @@ void ClusterFactory::evalLocalPositionFit(double deff, double mLogWei
     // May be put to global level or seperate method
     double ycorr = clXYZ[1] * (1. + phiSlope);
 
-    //printf(" y %f : ycorr %f : slope %f \n", clXYZ[1], ycorr, phiSlope);
+    // printf(" y %f : ycorr %f : slope %f \n", clXYZ[1], ycorr, phiSlope);
     clXYZ[1] = ycorr;
   }
 
@@ -468,6 +492,64 @@ void ClusterFactory::evalCoreEnergy(gsl::span inputsIndice
   clusterAnalysis.setCoreEnergy(coreEnergy);
 }
 
+///
+/// Calculate the number of local maxima in the cluster
+//____________________________________________________________________________
+template 
+void ClusterFactory::evalNExMax(gsl::span inputsIndices, AnalysisCluster& clusterAnalysis) const
+{
+  // Pre-compute cell indices and energies for all cells in cluster to avoid multiple expensive geometry lookups
+  const size_t n = inputsIndices.size();
+  std::vector rows;
+  std::vector columns;
+  std::vector energies;
+
+  rows.reserve(n);
+  columns.reserve(n);
+  energies.reserve(n);
+
+  for (auto iInput : inputsIndices) {
+    auto [nSupMod, nModule, nIphi, nIeta] = mGeomPtr->GetCellIndex(mInputsContainer[iInput].getTower());
+
+    // get a nice topological indexing that is done in exactly the same way as used by the clusterizer
+    // this way we can handle the shared cluster cases correctly
+    const auto [row, column] = mGeomPtr->GetTopologicalRowColumn(nSupMod, nModule, nIphi, nIeta);
+
+    rows.push_back(row);
+    columns.push_back(column);
+    energies.push_back(mInputsContainer[iInput].getEnergy());
+  }
+
+  // Now find local maxima using pre-computed data
+  int nExMax = 0;
+  for (size_t i = 0; i < n; i++) {
+    // this cell is assumed to be local maximum unless we find a higher energy cell in the neighborhood
+    bool isExMax = true;
+
+    // loop over all other cells in cluster
+    for (size_t j = 0; j < n; j++) {
+      if (i == j) {
+        continue;
+      }
+
+      // adjacent cell is any cell with adjacent phi or eta index
+      if (std::abs(rows[i] - rows[j]) <= 1 &&
+          std::abs(columns[i] - columns[j]) <= 1) {
+
+        // if there is a cell with higher energy than the current cell, it is not a local maximum
+        if (energies[j] > energies[i]) {
+          isExMax = false;
+          break;
+        }
+      }
+    }
+    if (isExMax) {
+      nExMax++;
+    }
+  }
+  clusterAnalysis.setNExMax(nExMax);
+}
+
 ///
 /// Calculates the axis of the shower ellipsoid in eta and phi
 /// in cell units
@@ -533,7 +615,7 @@ void ClusterFactory::evalElipsAxis(gsl::span inputsIndices
 
     lambda[1] = 0.5 * (dxx + dzz) - TMath::Sqrt(0.25 * (dxx - dzz) * (dxx - dzz) + dxz * dxz);
 
-    if (lambda[1] > 0) { //To avoid exception if numerical errors lead to negative lambda.
+    if (lambda[1] > 0) { // To avoid exception if numerical errors lead to negative lambda.
       lambda[1] = TMath::Sqrt(lambda[1]);
     } else {
       lambda[1] = 0.;
@@ -551,24 +633,190 @@ void ClusterFactory::evalElipsAxis(gsl::span inputsIndices
 /// Finds the maximum energy in the cluster and computes the Summed amplitude of digits/cells
 //____________________________________________________________________________
 template 
-std::tuple ClusterFactory::getMaximalEnergyIndex(gsl::span inputsIndices) const
+std::tuple ClusterFactory::getMaximalEnergyIndex(gsl::span inputsIndices) const
 {
 
   float energy = 0.;
   int mid = 0;
   float cellAmp = 0;
+  int iSupMod0 = -1;
+  bool shared = false;
   for (auto iInput : inputsIndices) {
     if (iInput >= mInputsContainer.size()) {
       throw CellIndexRangeException(iInput, mInputsContainer.size());
     }
     cellAmp += mInputsContainer[iInput].getEnergy();
+    if (iSupMod0 == -1) {
+      iSupMod0 = mGeomPtr->GetSuperModuleNumber(mInputsContainer[iInput].getTower());
+    } else if (iSupMod0 != mGeomPtr->GetSuperModuleNumber(mInputsContainer[iInput].getTower())) {
+      shared = true;
+    }
     if (mInputsContainer[iInput].getEnergy() > energy) {
       energy = mInputsContainer[iInput].getEnergy();
       mid = iInput;
     }
   } // loop on cluster inputs
 
-  return std::make_tuple(mid, energy, cellAmp);
+  return std::make_tuple(mid, energy, cellAmp, shared);
+}
+
+///
+/// Look to cell neighbourhood and reject if it seems exotic
+//____________________________________________________________________________
+template 
+bool ClusterFactory::isExoticCell(short towerId, float ecell, float const exoticTime, float& fCross) const
+{
+  if (ecell < mExoticCellMinAmplitude) {
+    return false; // do not reject low energy cells
+  }
+
+  // if the look up table is not set yet (mostly due to a reset call) then set it up now.
+  if (!getLookUpInit()) {
+    throw UninitLookUpTableException();
+  }
+
+  float eCross = getECross(towerId, ecell, exoticTime);
+  fCross = 1.f - eCross / ecell;
+
+  if (fCross > mExoticCellFraction) {
+    LOG(debug) << "EXOTIC CELL id " << towerId << ", eCell " << ecell << ", eCross " << eCross << ", 1-eCross/eCell " << 1 - eCross / ecell;
+    return true;
+  }
+
+  return false;
+}
+
+///
+///  Calculate the energy in the cross around the energy of a given cell.
+//____________________________________________________________________________
+template 
+float ClusterFactory::getECross(short towerId, float energy, float const exoticTime) const
+{
+  auto [iSM, iMod, iIphi, iIeta] = mGeomPtr->GetCellIndex(towerId);
+  auto [iphi, ieta] = mGeomPtr->GetCellPhiEtaIndexInSModule(iSM, iMod, iIphi, iIeta);
+
+  // Get close cells index, energy and time, not in corners
+
+  short towerId1 = -1;
+  short towerId2 = -1;
+
+  if (iphi < o2::emcal::EMCAL_ROWS - 1) {
+    try {
+      towerId1 = mGeomPtr->GetAbsCellIdFromCellIndexes(iSM, iphi + 1, ieta);
+    } catch (InvalidCellIDException& e) {
+      towerId1 = -1 * e.getCellID();
+    }
+  }
+  if (iphi > 0) {
+    try {
+      towerId2 = mGeomPtr->GetAbsCellIdFromCellIndexes(iSM, iphi - 1, ieta);
+    } catch (InvalidCellIDException& e) {
+      towerId2 = -1 * e.getCellID();
+    }
+  }
+
+  // In case of cell in eta = 0 border, depending on SM shift the cross cell index
+
+  short towerId3 = -1;
+  short towerId4 = -1;
+
+  if (ieta == o2::emcal::EMCAL_COLS - 1 && !(iSM % 2)) {
+    try {
+      towerId3 = mGeomPtr->GetAbsCellIdFromCellIndexes(iSM + 1, iphi, 0);
+    } catch (InvalidCellIDException& e) {
+      towerId3 = -1 * e.getCellID();
+    }
+    try {
+      towerId4 = mGeomPtr->GetAbsCellIdFromCellIndexes(iSM, iphi, ieta - 1);
+    } catch (InvalidCellIDException& e) {
+      towerId4 = -1 * e.getCellID();
+    }
+  } else if (ieta == 0 && iSM % 2) {
+    try {
+      towerId3 = mGeomPtr->GetAbsCellIdFromCellIndexes(iSM, iphi, ieta + 1);
+    } catch (InvalidCellIDException& e) {
+      towerId3 = -1 * e.getCellID();
+    }
+    try {
+      towerId4 = mGeomPtr->GetAbsCellIdFromCellIndexes(iSM - 1, iphi, o2::emcal::EMCAL_COLS - 1);
+    } catch (InvalidCellIDException& e) {
+      towerId4 = -1 * e.getCellID();
+    }
+  } else {
+    if (ieta < o2::emcal::EMCAL_COLS - 1) {
+      try {
+        towerId3 = mGeomPtr->GetAbsCellIdFromCellIndexes(iSM, iphi, ieta + 1);
+      } catch (InvalidCellIDException& e) {
+        towerId3 = -1 * e.getCellID();
+      }
+    }
+    if (ieta > 0) {
+      try {
+        towerId4 = mGeomPtr->GetAbsCellIdFromCellIndexes(iSM, iphi, ieta - 1);
+      } catch (InvalidCellIDException& e) {
+        towerId4 = -1 * e.getCellID();
+      }
+    }
+  }
+
+  LOG(debug) << "iSM " << iSM << ", towerId " << towerId << ", a " << towerId1 << ", b " << towerId2 << ", c " << towerId3 << ", e " << towerId3;
+
+  short index1 = (towerId1 > -1) ? mLoolUpTowerToIndex.at(towerId1) : -1;
+  short index2 = (towerId2 > -1) ? mLoolUpTowerToIndex.at(towerId2) : -1;
+  short index3 = (towerId3 > -1) ? mLoolUpTowerToIndex.at(towerId3) : -1;
+  short index4 = (towerId4 > -1) ? mLoolUpTowerToIndex.at(towerId4) : -1;
+
+  std::array, 4> cellData = {
+    {{(index1 > -1) ? mInputsContainer[index1].getEnergy() : 0., (index1 > -1) ? mInputsContainer[index1].getTimeStamp() : 0.},
+     {(index2 > -1) ? mInputsContainer[index2].getEnergy() : 0., (index2 > -1) ? mInputsContainer[index2].getTimeStamp() : 0.},
+     {(index3 > -1) ? mInputsContainer[index3].getEnergy() : 0., (index3 > -1) ? mInputsContainer[index3].getTimeStamp() : 0.},
+     {(index4 > -1) ? mInputsContainer[index4].getEnergy() : 0., (index4 > -1) ? mInputsContainer[index4].getTimeStamp() : 0.}}};
+
+  for (auto& cell : cellData) {
+    if (std::abs(exoticTime - cell.second) > mExoticCellDiffTime) {
+      cell.first = 0;
+    }
+  }
+
+  float w1 = 1, w2 = 1, w3 = 1, w4 = 1;
+  if (mUseWeightExotic) {
+    w1 = GetCellWeight(cellData[0].first, energy);
+    w2 = GetCellWeight(cellData[1].first, energy);
+    w3 = GetCellWeight(cellData[2].first, energy);
+    w4 = GetCellWeight(cellData[3].first, energy);
+  }
+
+  if (cellData[0].first < mExoticCellInCrossMinAmplitude || w1 <= 0) {
+    cellData[0].first = 0;
+  }
+  if (cellData[1].first < mExoticCellInCrossMinAmplitude || w2 <= 0) {
+    cellData[1].first = 0;
+  }
+  if (cellData[2].first < mExoticCellInCrossMinAmplitude || w3 <= 0) {
+    cellData[2].first = 0;
+  }
+  if (cellData[3].first < mExoticCellInCrossMinAmplitude || w4 <= 0) {
+    cellData[3].first = 0;
+  }
+
+  return cellData[0].first + cellData[1].first + cellData[2].first + cellData[3].first;
+}
+
+///
+/// return weight of cell for shower shape calculation
+//____________________________________________________________________________
+template 
+float ClusterFactory::GetCellWeight(float eCell, float eCluster) const
+{
+  if (eCell > 0 && eCluster > 0) {
+    if (mLogWeight > 0) {
+      return std::max(0.f, mLogWeight + std::log(eCell / eCluster));
+    } else {
+      return std::log(eCluster / eCell);
+    }
+  } else {
+    return 0.;
+  }
 }
 
 ///
diff --git a/Detectors/EMCAL/base/src/EMCALBaseLinkDef.h b/Detectors/EMCAL/base/src/EMCALBaseLinkDef.h
index 6f79dd4419912..6b4b18baa861a 100644
--- a/Detectors/EMCAL/base/src/EMCALBaseLinkDef.h
+++ b/Detectors/EMCAL/base/src/EMCALBaseLinkDef.h
@@ -19,6 +19,8 @@
 #pragma link C++ class o2::emcal::Geometry + ;
 #pragma link C++ class o2::emcal::Mapper + ;
 #pragma link C++ class o2::emcal::MappingHandler + ;
+#pragma link C++ class o2::emcal::NonlinearityHandler + ;
+#pragma link C++ class o2::emcal::NonlinearityFactory + ;
 #pragma link C++ class o2::emcal::TriggerMappingV2 + ;
 #pragma link C++ class o2::emcal::RCUTrailer + ;
 
diff --git a/Detectors/EMCAL/base/src/Geometry.cxx b/Detectors/EMCAL/base/src/Geometry.cxx
index ec3f90011afdf..3707e22f2da57 100644
--- a/Detectors/EMCAL/base/src/Geometry.cxx
+++ b/Detectors/EMCAL/base/src/Geometry.cxx
@@ -8,17 +8,40 @@
 // In applying this license CERN does not waive the privileges and immunities
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
+#include "EMCALBase/Geometry.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
 #include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
 
 #include 
 #include 
 #include 
-#include 
-
-#include 
 
-#include "EMCALBase/Geometry.h"
+#include "DataFormatsEMCAL/Constants.h"
+#include "EMCALBase/GeometryBase.h"
+#include "CCDB/CcdbApi.h"
 #include "EMCALBase/ShishKebabTrd1Module.h"
+#include "GPUROOTCartesianFwd.h"
 
 #include 
 
@@ -190,6 +213,9 @@ Geometry::~Geometry()
 Geometry* Geometry::GetInstance()
 {
   Geometry* rv = static_cast(sGeom);
+  if (!rv) {
+    throw GeometryNotInitializedException();
+  }
   return rv;
 }
 
@@ -306,6 +332,9 @@ void Geometry::DefineSamplingFraction(const std::string_view mcname, const std::
   }
 
   Float_t samplingFactorTranportModel = 1.;
+  // Note: The sampling factors are chosen so that results from the simulation
+  // engines correspond well with testbeam data
+
   if (contains(mcname, "Geant3")) {
     samplingFactorTranportModel = 1.; // 0.988 // Do nothing
   } else if (contains(mcname, "Fluka")) {
@@ -315,6 +344,9 @@ void Geometry::DefineSamplingFraction(const std::string_view mcname, const std::
     LOG(info) << "Selected physics list: " << physicslist;
     // sampling factors for different Geant4 physics list
     // GEANT4 10.7 -> EMCAL-784
+
+    // set a default (there may be many physics list strings)
+    samplingFactorTranportModel = 0.81;
     if (physicslist == "FTFP_BERT_EMV+optical") {
       samplingFactorTranportModel = 0.821;
     } else if (physicslist == "FTFP_BERT_EMV+optical+biasing") {
@@ -1023,7 +1055,7 @@ std::tuple Geometry::CalculateCellIndex(Int_t absId) const
 
   Int_t nModule = tmp / mNCellsInModule;
   tmp = tmp % mNCellsInModule;
-  Int_t nIphi = tmp / mNPHIdiv, nIeta = tmp % mNPHIdiv;
+  Int_t nIphi = tmp / mNPHIdiv, nIeta = tmp % mNETAdiv;
   return std::make_tuple(nSupMod, nModule, nIphi, nIeta);
 }
 
@@ -1071,6 +1103,30 @@ std::tuple Geometry::GetCellPhiEtaIndexInSModule(int supermoduleID, in
   return std::make_tuple(phiInSupermodule, etaInSupermodule);
 }
 
+std::tuple Geometry::GetTopologicalRowColumn(int supermoduleID, int moduleID, int phiInModule, int etaInModule) const
+{
+  auto [iphi, ieta] = GetCellPhiEtaIndexInSModule(supermoduleID, moduleID, phiInModule, etaInModule);
+  int row = iphi;
+  int column = ieta;
+
+  // Add shifts wrt. supermodule and type of calorimeter
+  // NOTE:
+  // * Rows (phi) are arranged that one space is left empty between supermodules in phi
+  //   This is due to the physical gap that forbids clustering
+  // * For DCAL, there is an additional empty column between two supermodules in eta
+  //   Again, this is to account for the gap in DCAL
+
+  row += supermoduleID / 2 * (24 + 1);
+  // In DCAL, leave a gap between two SMs with same phi
+  if (!IsDCALSM(supermoduleID)) { // EMCAL
+    column += supermoduleID % 2 * 48;
+  } else {
+    column += supermoduleID % 2 * (48 + 1);
+  }
+
+  return std::make_tuple(static_cast(row), static_cast(column));
+}
+
 std::tuple Geometry::ShiftOnlineToOfflineCellIndexes(Int_t supermoduleID, Int_t iphi, Int_t ieta) const
 {
   if (supermoduleID == 13 || supermoduleID == 15 || supermoduleID == 17) {
@@ -1548,6 +1604,7 @@ const TGeoHMatrix* Geometry::GetMatrixForSuperModule(Int_t smod) const
 
   if (!SMODULEMATRIX[smod]) {
     if (gGeoManager) {
+      LOG(info) << "Loading EMCAL misalignment matrix for SM " << smod << " from GeoManager.";
       SetMisalMatrix(GetMatrixForSuperModuleFromGeoManager(smod), smod);
     } else {
       LOG(fatal) << "Cannot find EMCAL misalignment matrices! Recover them either: \n"
@@ -1753,6 +1810,25 @@ void Geometry::SetMisalMatrix(const TGeoHMatrix* m, Int_t smod) const
   }
 }
 
+void Geometry::SetMisalMatrixFromCcdb(const char* path, int timestamp) const
+{
+  LOG(info) << "Using CCDB to obtain EMCal alignment.";
+  o2::ccdb::CcdbApi api;
+  std::map metadata; // can be empty
+  api.init("http://alice-ccdb.cern.ch");
+  TObjArray* matrices = api.retrieveFromTFileAny(path, metadata, timestamp);
+
+  for (int iSM = 0; iSM < mNumberOfSuperModules; ++iSM) {
+    TGeoHMatrix* mat = reinterpret_cast(matrices->At(iSM));
+    if (mat) {
+
+      SetMisalMatrix(mat, iSM);
+    } else {
+      LOG(info) << "Could not obtain Alignment Matrix for SM " << iSM;
+    }
+  }
+}
+
 Bool_t Geometry::IsDCALSM(Int_t iSupMod) const
 {
   if (mEMCSMSystem[iSupMod] == DCAL_STANDARD || mEMCSMSystem[iSupMod] == DCAL_EXT) {
@@ -1825,3 +1901,44 @@ std::tuple Geometry::getOnlineID(int towerID)
 
   return std::make_tuple(supermoduleID * 2 + ddlInSupermoudel, row, col);
 }
+
+std::tuple Geometry::areAbsIDsFromSameTCard(int absId1, int absId2) const
+{
+
+  int rowDiff = -100;
+  int colDiff = -100;
+
+  if (absId1 == absId2) {
+    return {false, rowDiff, colDiff};
+  }
+
+  // Check if in same SM, if not for sure not same TCard
+  const int sm1 = GetSuperModuleNumber(absId1);
+  const int sm2 = GetSuperModuleNumber(absId2);
+  if (sm1 != sm2) {
+    return {false, rowDiff, colDiff};
+  }
+
+  // Get the column and row of each absId
+  const auto [_, iTower1, iIphi1, iIeta1] = GetCellIndex(absId1);
+  const auto [row1, col1] = GetCellPhiEtaIndexInSModule(sm1, iTower1, iIphi1, iIeta1);
+
+  const auto [__, iTower2, iIphi2, iIeta2] = GetCellIndex(absId2);
+  const auto [row2, col2] = GetCellPhiEtaIndexInSModule(sm2, iTower2, iIphi2, iIeta2);
+
+  // Define corner of TCard for absId1
+  const int tcardRow0 = row1 - row1 % 8;
+  const int tcardCol0 = col1 - col1 % 2;
+
+  // Difference of absId2 from corner of absId1's TCard
+  const int rowOffset = row2 - tcardRow0;
+  const int colOffset = col2 - tcardCol0;
+
+  // Differences between the two cells directly
+  rowDiff = row1 - row2;
+  colDiff = col1 - col2;
+
+  const bool sameTCard = (rowOffset >= 0 && rowOffset < 8 &&
+                          colOffset >= 0 && colOffset < 2);
+  return {sameTCard, rowDiff, colDiff};
+}
diff --git a/Detectors/EMCAL/base/src/Hit.cxx b/Detectors/EMCAL/base/src/Hit.cxx
index 39edba54e6b12..898d86dc2e2c4 100644
--- a/Detectors/EMCAL/base/src/Hit.cxx
+++ b/Detectors/EMCAL/base/src/Hit.cxx
@@ -32,7 +32,7 @@ Bool_t Hit::operator<(const Hit& rhs) const
 
 Bool_t Hit::operator==(const Hit& rhs) const
 {
-  return (GetDetectorID() == GetDetectorID()) && (GetTrackID() == rhs.GetTrackID());
+  return (GetDetectorID() == rhs.GetDetectorID()) && (GetTrackID() == rhs.GetTrackID());
 }
 
 Hit& Hit::operator+=(const Hit& rhs)
@@ -41,14 +41,14 @@ Hit& Hit::operator+=(const Hit& rhs)
   return *this;
 }
 
-Hit Hit::operator+(const Hit& rhs) const
+Hit o2::emcal::operator+(const Hit& lhs, const Hit& rhs)
 {
-  Hit result(*this);
+  Hit result(lhs);
   result.SetEnergyLoss(result.GetEnergyLoss() + rhs.GetEnergyLoss());
-  return *this;
+  return result;
 }
 
-std::ostream& operator<<(std::ostream& stream, const Hit& p)
+std::ostream& o2::emcal::operator<<(std::ostream& stream, const Hit& p)
 {
   p.PrintStream(stream);
   return stream;
diff --git a/Detectors/EMCAL/base/src/Mapper.cxx b/Detectors/EMCAL/base/src/Mapper.cxx
index 42f98820ebd58..529fa74cebce9 100644
--- a/Detectors/EMCAL/base/src/Mapper.cxx
+++ b/Detectors/EMCAL/base/src/Mapper.cxx
@@ -66,14 +66,14 @@ void Mapper::init(const std::string_view filename)
 
     auto chantype = o2::emcal::intToChannelType(caloflag);
 
-    mMapping.insert(std::pair(address, {uint8_t(row), uint8_t(col), chantype}));
-    mInverseMapping.insert(std::pair({uint8_t(row), uint8_t(col), chantype}, address));
+    mMapping.insert(std::pair(static_cast(address), {uint8_t(row), uint8_t(col), chantype}));
+    mInverseMapping.insert(std::pair({uint8_t(row), uint8_t(col), chantype}, static_cast(address)));
   }
 
   mInitStatus = true;
 }
 
-Mapper::ChannelID Mapper::getChannelID(int hardawareaddress) const
+Mapper::ChannelID Mapper::getChannelID(unsigned int hardawareaddress) const
 {
   if (!mInitStatus) {
     throw InitStatusException();
@@ -85,12 +85,12 @@ Mapper::ChannelID Mapper::getChannelID(int hardawareaddress) const
   return res->second;
 }
 
-int Mapper::getHardwareAddress(int row, int col, ChannelType_t channeltype) const
+unsigned int Mapper::getHardwareAddress(uint8_t row, uint8_t col, ChannelType_t channeltype) const
 {
   if (!mInitStatus) {
     throw InitStatusException();
   }
-  ChannelID channelToFind{uint8_t(row), uint8_t(col), channeltype};
+  ChannelID channelToFind{row, col, channeltype};
   auto found = mInverseMapping.find(channelToFind);
   if (found == mInverseMapping.end()) {
     throw ChannelNotFoundException(channelToFind);
@@ -101,9 +101,9 @@ int Mapper::getHardwareAddress(int row, int col, ChannelType_t channeltype) cons
 MappingHandler::MappingHandler()
 {
   const std::array SIDES = {{'A', 'C'}};
-  const int NDDL = 2;
-  for (int iside = 0; iside < 2; iside++) {
-    for (int iddl = 0; iddl < NDDL; iddl++) {
+  const unsigned int NDDL = 2;
+  for (unsigned int iside = 0; iside < 2; iside++) {
+    for (unsigned int iddl = 0; iddl < NDDL; iddl++) {
       mMappings[iside * NDDL + iddl].setMapping(Form("%s/share/Detectors/EMC/files/RCU%d%c.data", gSystem->Getenv("O2_ROOT"), iddl, SIDES[iside]));
     }
   }
@@ -114,10 +114,15 @@ Mapper& MappingHandler::getMappingForDDL(unsigned int ddl)
   if (ddl >= 40) {
     throw MappingHandler::DDLInvalid(ddl);
   }
-  const int NDDLSM = 2, NSIDES = 2;
-  int ddlInSM = ddl % NDDLSM,
-      sideID = (ddl / NDDLSM) % NSIDES;
-  return mMappings[sideID * NDDLSM + ddlInSM];
+  const unsigned int NDDLSM = 2, NSIDES = 2;
+  unsigned int ddlInSM = ddl % NDDLSM,
+               sideID = (ddl / NDDLSM) % NSIDES;
+  unsigned int mappingIndex = sideID * NDDLSM + ddlInSM;
+  if (mappingIndex < 0 || mappingIndex >= mMappings.size()) {
+    std::cout << "Access to invalid mapping position for ddl " << ddl << std::endl;
+    throw MappingHandler::DDLInvalid(ddl);
+  }
+  return mMappings[mappingIndex];
 }
 
 int MappingHandler::getFEEForChannelInDDL(unsigned int ddl, unsigned int channelFEC, unsigned int branch)
diff --git a/Detectors/EMCAL/base/src/NonlinearityHandler.cxx b/Detectors/EMCAL/base/src/NonlinearityHandler.cxx
new file mode 100644
index 0000000000000..026c86a08c63f
--- /dev/null
+++ b/Detectors/EMCAL/base/src/NonlinearityHandler.cxx
@@ -0,0 +1,359 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#include 
+#include 
+#include 
+#include  // for TMath::Pi() - to be removed once we switch to c++20
+#include "EMCALBase/NonlinearityHandler.h"
+
+using namespace o2::emcal;
+
+NonlinearityHandler::NonlinearityHandler(NonlinType_t nonlintype) : mNonlinearyFunction(nonlintype)
+{
+  initParams();
+}
+
+void NonlinearityHandler::initParams()
+{
+  std::fill(mNonlinearityParam.begin(), mNonlinearityParam.end(), 0);
+  switch (mNonlinearyFunction) {
+    case NonlinType_t::MC_TESTBEAM_FINAL:
+      mNonlinearityParam[0] = 1.09357;
+      mNonlinearityParam[1] = 0.0192266;
+      mNonlinearityParam[2] = 0.291993;
+      mNonlinearityParam[3] = 370.927;
+      mNonlinearityParam[4] = 694.656;
+      break;
+    case NonlinType_t::MC_PI0:
+      mNonlinearityParam[0] = 1.014;
+      mNonlinearityParam[1] = -0.03329;
+      mNonlinearityParam[2] = -0.3853;
+      mNonlinearityParam[3] = 0.5423;
+      mNonlinearityParam[4] = -0.4335;
+      break;
+    case NonlinType_t::MC_PI0_V2:
+      mNonlinearityParam[0] = 3.11111e-02;
+      mNonlinearityParam[1] = -5.71666e-02;
+      mNonlinearityParam[2] = 5.67995e-01;
+      break;
+    case NonlinType_t::MC_PI0_V3:
+      mNonlinearityParam[0] = 9.81039e-01;
+      mNonlinearityParam[1] = 1.13508e-01;
+      mNonlinearityParam[2] = 1.00173e+00;
+      mNonlinearityParam[3] = 9.67998e-02;
+      mNonlinearityParam[4] = 2.19381e+02;
+      mNonlinearityParam[5] = 6.31604e+01;
+      mNonlinearityParam[6] = 1;
+      break;
+    case NonlinType_t::MC_PI0_V5:
+      mNonlinearityParam[0] = 1.0;
+      mNonlinearityParam[1] = 6.64778e-02;
+      mNonlinearityParam[2] = 1.570;
+      mNonlinearityParam[3] = 9.67998e-02;
+      mNonlinearityParam[4] = 2.19381e+02;
+      mNonlinearityParam[5] = 6.31604e+01;
+      mNonlinearityParam[6] = 1.01286;
+      break;
+    case NonlinType_t::MC_PI0_V6:
+      mNonlinearityParam[0] = 1.0;
+      mNonlinearityParam[1] = 0.0797873;
+      mNonlinearityParam[2] = 1.68322;
+      mNonlinearityParam[3] = 0.0806098;
+      mNonlinearityParam[4] = 244.586;
+      mNonlinearityParam[5] = 116.938;
+      mNonlinearityParam[6] = 1.00437;
+      break;
+    case NonlinType_t::DATA_TESTBEAM_CORRECTED:
+      mNonlinearityParam[0] = 0.99078;
+      mNonlinearityParam[1] = 0.161499;
+      mNonlinearityParam[2] = 0.655166;
+      mNonlinearityParam[3] = 0.134101;
+      mNonlinearityParam[4] = 163.282;
+      mNonlinearityParam[5] = 23.6904;
+      mNonlinearityParam[6] = 0.978;
+      break;
+    case NonlinType_t::DATA_TESTBEAM_CORRECTED_V2:
+      // Parameters until November 2015, use now kBeamTestCorrectedv3
+      mNonlinearityParam[0] = 0.983504;
+      mNonlinearityParam[1] = 0.210106;
+      mNonlinearityParam[2] = 0.897274;
+      mNonlinearityParam[3] = 0.0829064;
+      mNonlinearityParam[4] = 152.299;
+      mNonlinearityParam[5] = 31.5028;
+      mNonlinearityParam[6] = 0.968;
+      break;
+    case NonlinType_t::DATA_TESTBEAM_CORRECTED_V3:
+      // New parametrization of kBeamTestCorrected
+      // excluding point at 0.5 GeV from Beam Test Data
+      // https://indico.cern.ch/event/438805/contribution/1/attachments/1145354/1641875/emcalPi027August2015.pdf
+
+      mNonlinearityParam[0] = 0.976941;
+      mNonlinearityParam[1] = 0.162310;
+      mNonlinearityParam[2] = 1.08689;
+      mNonlinearityParam[3] = 0.0819592;
+      mNonlinearityParam[4] = 152.338;
+      mNonlinearityParam[5] = 30.9594;
+      mNonlinearityParam[6] = 0.9615;
+      break;
+
+    case NonlinType_t::DATA_TESTBEAM_CORRECTED_V4:
+      // New parametrization of kBeamTestCorrected,
+      // fitting new points for E>100 GeV.
+      // I should have same performance as v3 in the low energies
+      // See EMCal meeting 21/09/2018 slides
+      // https://indico.cern.ch/event/759154/contributions/3148448/attachments/1721042/2778585/nonLinearityUpdate.pdf
+      //  and jira ticket EMCAL-190
+
+      mNonlinearityParam[0] = 0.9892;
+      mNonlinearityParam[1] = 0.1976;
+      mNonlinearityParam[2] = 0.865;
+      mNonlinearityParam[3] = 0.06775;
+      mNonlinearityParam[4] = 156.6;
+      mNonlinearityParam[5] = 47.18;
+      mNonlinearityParam[6] = 0.97;
+      break;
+
+    case NonlinType_t::DATA_TESTBEAM_SHAPER:
+    case NonlinType_t::DATA_TESTBEAM_SHAPER_WOSCALE:
+      mNonlinearityParam[0] = 1.91897;
+      mNonlinearityParam[1] = 0.0264988;
+      mNonlinearityParam[2] = 0.965663;
+      mNonlinearityParam[3] = -187.501;
+      mNonlinearityParam[4] = 2762.51;
+      break;
+
+    default:
+      break;
+  }
+  if (mNonlinearyFunction == NonlinType_t::DATA_TESTBEAM_SHAPER) {
+    mApplyScaleCorrection = true;
+  }
+}
+double NonlinearityHandler::getCorrectedClusterEnergy(double energy) const
+{
+  double correctedEnergy = energy;
+  switch (mNonlinearyFunction) {
+    case NonlinType_t::MC_PI0:
+      correctedEnergy = evaluatePi0MC(energy);
+      break;
+    case NonlinType_t::MC_PI0_V2:
+      correctedEnergy = evaluatePi0MCv2(energy);
+    case NonlinType_t::MC_PI0_V3:
+    case NonlinType_t::MC_PI0_V5:
+    case NonlinType_t::MC_PI0_V6:
+    case NonlinType_t::DATA_TESTBEAM_CORRECTED:
+    case NonlinType_t::DATA_TESTBEAM_CORRECTED_V2:
+    case NonlinType_t::DATA_TESTBEAM_CORRECTED_V3:
+    case NonlinType_t::DATA_TESTBEAM_CORRECTED_V4:
+      correctedEnergy = evaluateTestbeamCorrected(energy);
+      break;
+    case NonlinType_t::DATA_TESTBEAM_SHAPER:
+    case NonlinType_t::DATA_TESTBEAM_SHAPER_WOSCALE:
+    case NonlinType_t::MC_TESTBEAM_FINAL:
+      correctedEnergy = evaluateTestbeamShaper(energy);
+      break;
+    default:
+      throw UninitException();
+  }
+  if (mApplyScaleCorrection) {
+    correctedEnergy /= 1.0505;
+  }
+  return correctedEnergy;
+}
+
+double NonlinearityHandler::evaluateTestbeamShaper(double energy) const
+{
+  return energy / (1.00 * (mNonlinearityParam[0] + mNonlinearityParam[1] * std::log(energy)) / (1 + (mNonlinearityParam[2] * std::exp((energy - mNonlinearityParam[3]) / mNonlinearityParam[4]))));
+}
+
+double NonlinearityHandler::evaluateTestbeamCorrected(double energy) const
+{
+  return energy * mNonlinearityParam[6] / (mNonlinearityParam[0] * (1. / (1. + mNonlinearityParam[1] * std::exp(-energy / mNonlinearityParam[2])) * 1. / (1. + mNonlinearityParam[3] * std::exp((energy - mNonlinearityParam[4]) / mNonlinearityParam[5]))));
+}
+
+double NonlinearityHandler::evaluatePi0MC(double energy) const
+{
+  return energy * (mNonlinearityParam[0] * std::exp(-mNonlinearityParam[1] / energy)) +
+         ((mNonlinearityParam[2] / (mNonlinearityParam[3] * 2. * TMath::Pi()) *
+           std::exp(-(energy - mNonlinearityParam[4]) * (energy - mNonlinearityParam[4]) / (2. * mNonlinearityParam[3] * mNonlinearityParam[3]))));
+}
+
+double NonlinearityHandler::evaluatePi0MCv2(double energy) const
+{
+  return energy * mNonlinearityParam[0] / TMath::Power(energy + mNonlinearityParam[1], mNonlinearityParam[2]) + 1;
+}
+
+double NonlinearityHandler::evaluateShaperCorrectionCellEnergy(double energy, double ecalibHG)
+{
+  if (energy < 40) {
+    return energy * 16.3 / 16;
+  }
+  constexpr std::array par = {{1, 29.8279, 0.607704, 0.00164896, -2.28595e-06, -8.54664e-10, 5.50191e-12, -3.28098e-15}};
+  double x = par[0] * energy / ecalibHG / 16 / 0.0162;
+
+  double res = par[1];
+  res += par[2] * x;
+  res += par[3] * x * x;
+  res += par[4] * x * x * x;
+  res += par[5] * x * x * x * x;
+  res += par[6] * x * x * x * x * x;
+  res += par[7] * x * x * x * x * x * x;
+
+  return ecalibHG * 16.3 * res * 0.0162;
+}
+
+void NonlinearityHandler::printStream(std::ostream& stream) const
+{
+  stream << "Nonlinearity function: " << getNonlinName(mNonlinearyFunction)
+         << "(Parameters:";
+  bool first = true;
+  for (auto& param : mNonlinearityParam) {
+    if (first) {
+      first = false;
+    } else {
+      stream << ",";
+    }
+    stream << " " << param;
+  }
+  stream << ")";
+}
+
+NonlinearityHandler::NonlinType_t NonlinearityHandler::getNonlinType(const std::string_view name)
+{
+  using NLType = NonlinearityHandler::NonlinType_t;
+  if (name == "MC_Pi0") {
+    return NLType::MC_PI0;
+  }
+  if (name == "MC_Pi0_v2") {
+    return NLType::MC_PI0_V2;
+  }
+  if (name == "MC_Pi0_v3") {
+    return NLType::MC_PI0_V3;
+  }
+  if (name == "MC_Pi0_v5") {
+    return NLType::MC_PI0_V5;
+  }
+  if (name == "MC_Pi0_v6") {
+    return NLType::MC_PI0_V6;
+  }
+  if (name == "MC_TestbeamFinal") {
+    return NLType::MC_TESTBEAM_FINAL;
+  }
+  if (name == "DATA_BeamTestCorrected") {
+    return NLType::DATA_TESTBEAM_CORRECTED;
+  }
+  if (name == "DATA_BeamTestCorrected_v2") {
+    return NLType::DATA_TESTBEAM_CORRECTED_V2;
+  }
+  if (name == "DATA_BeamTestCorrected_v3") {
+    return NLType::DATA_TESTBEAM_CORRECTED_V3;
+  }
+  if (name == "DATA_BeamTestCorrected_v4") {
+    return NLType::DATA_TESTBEAM_CORRECTED_V4;
+  }
+  if (name == "DATA_TestbeamFinal") {
+    return NLType::DATA_TESTBEAM_SHAPER;
+  }
+  if (name == "DATA_TestbeamFinal_NoScale") {
+    return NLType::DATA_TESTBEAM_SHAPER_WOSCALE;
+  }
+  return NLType::UNKNOWN;
+}
+
+const char* NonlinearityHandler::getNonlinName(NonlinearityHandler::NonlinType_t nonlin)
+{
+  using NLType = NonlinearityHandler::NonlinType_t;
+  switch (nonlin) {
+    case NLType::MC_PI0:
+      return "MC_Pi0";
+    case NLType::MC_PI0_V2:
+      return "MC_Pi0_v2";
+    case NLType::MC_PI0_V3:
+      return "MC_Pi0_v3";
+    case NLType::MC_PI0_V5:
+      return "MC_Pi0_v5";
+    case NLType::MC_PI0_V6:
+      return "MC_Pi0_v6";
+    case NLType::MC_TESTBEAM_FINAL:
+      return "MC_TestbeamFinal";
+    case NLType::DATA_TESTBEAM_CORRECTED:
+      return "DATA_BeamTestCorrected";
+    case NLType::DATA_TESTBEAM_CORRECTED_V2:
+      return "DATA_BeamTestCorrected_v2";
+    case NLType::DATA_TESTBEAM_CORRECTED_V3:
+      return "DATA_BeamTestCorrected_v3";
+    case NLType::DATA_TESTBEAM_CORRECTED_V4:
+      return "DATA_BeamTestCorrected_v4";
+    case NLType::DATA_TESTBEAM_SHAPER:
+      return "DATA_TestbeamFinal";
+    case NLType::DATA_TESTBEAM_SHAPER_WOSCALE:
+      return "DATA_TestbeamFinal_NoScale";
+    case NLType::UNKNOWN:
+      return "Unknown";
+    default:
+      return "";
+  }
+}
+
+NonlinearityHandler& NonlinearityFactory::getNonlinearity(NonlinearityHandler::NonlinType_t nonlintype)
+{
+  auto found = mHandlers.find(nonlintype);
+  if (found != mHandlers.end()) {
+    return found->second;
+  }
+  auto [insert_result, insert_status] = mHandlers.try_emplace(nonlintype, NonlinearityHandler(nonlintype));
+  if (insert_status) {
+    return insert_result->second;
+  }
+  throw NonlinInitError();
+}
+
+NonlinearityHandler& NonlinearityFactory::getNonlinearity(const std::string_view nonname)
+{
+  return getNonlinearity(getNonlinType(nonname));
+}
+
+NonlinearityHandler::NonlinType_t NonlinearityFactory::getNonlinType(const std::string_view nonlinName) const
+{
+  auto found = mNonlinNames.find(static_cast(nonlinName));
+  if (found != mNonlinNames.end()) {
+    return found->second;
+  }
+  throw NonlinearityFactory::FunctionNotFoundExcpetion(nonlinName);
+}
+
+void NonlinearityFactory::initNonlinNames()
+{
+  using NLType = NonlinearityHandler::NonlinType_t;
+  constexpr std::array nonlintypes = {{NLType::MC_PI0,
+                                                   NLType::MC_PI0_V2,
+                                                   NLType::MC_PI0_V3,
+                                                   NLType::MC_PI0_V5,
+                                                   NLType::MC_PI0_V6,
+                                                   NLType::MC_TESTBEAM_FINAL,
+                                                   NLType::DATA_TESTBEAM_CORRECTED,
+                                                   NLType::DATA_TESTBEAM_CORRECTED_V2,
+                                                   NLType::DATA_TESTBEAM_CORRECTED_V3,
+                                                   NLType::DATA_TESTBEAM_CORRECTED_V4,
+                                                   NLType::DATA_TESTBEAM_SHAPER,
+                                                   NLType::DATA_TESTBEAM_SHAPER_WOSCALE
+
+  }};
+  for (auto nonlin : nonlintypes) {
+    mNonlinNames[NonlinearityHandler::getNonlinName(nonlin)] = nonlin;
+  }
+}
+
+std::ostream& o2::emcal::operator<<(std::ostream& in, const NonlinearityHandler& handler)
+{
+  handler.printStream(in);
+  return in;
+}
\ No newline at end of file
diff --git a/Detectors/EMCAL/base/src/RCUTrailer.cxx b/Detectors/EMCAL/base/src/RCUTrailer.cxx
index 770f2857ac86a..24aac7b5bb25a 100644
--- a/Detectors/EMCAL/base/src/RCUTrailer.cxx
+++ b/Detectors/EMCAL/base/src/RCUTrailer.cxx
@@ -14,6 +14,7 @@
 #include 
 #include "CommonConstants/LHCConstants.h"
 #include "EMCALBase/RCUTrailer.h"
+#include 
 
 using namespace o2::emcal;
 
@@ -34,6 +35,29 @@ void RCUTrailer::reset()
   mIsInitialized = false;
 }
 
+bool RCUTrailer::checkLastTrailerWord(uint32_t trailerword)
+{
+  const int MIN_FWVERSION = 2;
+  const int MAX_FWVERSION = 2;
+  if ((trailerword >> 30) != 3) {
+    return false;
+  }
+  auto firmwarevesion = (trailerword >> 16) & 0xFF;
+  auto trailerSize = (trailerword & 0x7F);
+  if (firmwarevesion < MIN_FWVERSION || firmwarevesion > MAX_FWVERSION) {
+    return false;
+  }
+  if (trailerSize < 2) {
+    return false;
+  }
+  if (firmwarevesion == 2) {
+    if (trailerSize < 9) {
+      return false;
+    }
+  }
+  return true;
+}
+
 void RCUTrailer::constructFromRawPayload(const gsl::span payloadwords)
 {
   reset();
@@ -62,7 +86,7 @@ void RCUTrailer::constructFromRawPayload(const gsl::span payload
     foundTrailerWords++;
     int parCode = (word >> 26) & 0xF;
     int parData = word & 0x3FFFFFF;
-    // std::cout << "Found trailer word 0x" << std::hex << word << "(Par code: " << std::dec << parCode << ", Par data: 0x" << std::hex << parData << std::dec << ")";
+    // std::cout << "Found trailer word 0x" << std::hex << word << "(Par code: " << std::dec << parCode << ", Par data: 0x" << std::hex << parData << std::dec << ")" << std::endl;
     switch (parCode) {
       case 1:
         // ERR_REG1
@@ -94,7 +118,8 @@ void RCUTrailer::constructFromRawPayload(const gsl::span payload
         mAltroConfig.mWord2 = parData & 0x1FFFFFF;
         break;
       default:
-        std::cerr << "Undefined parameter code " << parCode << ", ignore it !\n";
+        LOG(warning) << "RCU trailer: Undefined parameter code " << parCode << " in word " << index << " (0x" << std::hex << word << std::dec << "), ignoring word";
+        mWordCorruptions++;
         break;
     }
   }
@@ -147,7 +172,7 @@ void RCUTrailer::setTimeSamplePhaseNS(uint64_t triggertime, uint64_t timesample)
       sample = 2;
       break;
     default:
-      throw Error(Error::ErrorType_t::SAMPLINGFREQ_INVALID, fmt::format("invalid time sample: {:f}", timesample).data());
+      throw Error(Error::ErrorType_t::SAMPLINGFREQ_INVALID, fmt::format(fmt::runtime("invalid time sample: {:f}"), timesample).data());
   };
   mAltroConfig.mSampleTime = sample;
   // calculate L1 phase
@@ -226,8 +251,8 @@ void RCUTrailer::printStream(std::ostream& stream) const
          << "Sparse readout:                            " << (isSparseReadout() ? "yes" : "no") << "\n"
          << "AltroCFG1:                                 0x" << std::hex << mAltroConfig.mWord1 << "\n"
          << "AltroCFG2:                                 0x" << std::hex << mAltroConfig.mWord2 << "\n"
-         << "Sampling time:                             " << timesample << " ns\n"
-         << "L1 Phase:                                  " << l1phase << " ns\n"
+         << "Sampling time:                             " << std::dec << timesample << " ns\n"
+         << "L1 Phase:                                  " << std::dec << l1phase << " ns (" << mAltroConfig.mL1Phase << ")\n"
          << std::dec << std::fixed;
   if (errors.size()) {
     stream << "Errors: \n"
@@ -250,4 +275,4 @@ std::ostream& o2::emcal::operator<<(std::ostream& stream, const o2::emcal::RCUTr
 {
   trailer.printStream(stream);
   return stream;
-}
\ No newline at end of file
+}
diff --git a/Detectors/EMCAL/base/src/TriggerMappingV2.cxx b/Detectors/EMCAL/base/src/TriggerMappingV2.cxx
index 5d5ee46dc6f2f..e60d7b31e5305 100644
--- a/Detectors/EMCAL/base/src/TriggerMappingV2.cxx
+++ b/Detectors/EMCAL/base/src/TriggerMappingV2.cxx
@@ -390,7 +390,7 @@ TriggerMappingV2::IndexTRU TriggerMappingV2::getTRUIndexFromOnlineHardareAddree(
 
   unsigned short branch = (hardwareAddress >> 11) & 0x1; // 0/1
 
-  IndexTRU truIndex = ((ddlID << 1) | branch) - 1; // 0..2
+  IndexTRU truIndex = (((ddlID % 2) << 1) | branch) - 1; // 0..2
 
   truIndex = (supermoduleID % 2) ? 2 - truIndex : truIndex;
 
diff --git a/Detectors/EMCAL/base/test/testGeometry.cxx b/Detectors/EMCAL/base/test/testGeometry.cxx
new file mode 100644
index 0000000000000..f297410b6959f
--- /dev/null
+++ b/Detectors/EMCAL/base/test/testGeometry.cxx
@@ -0,0 +1,66 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#define BOOST_TEST_MODULE Test EMCAL Base
+#define BOOST_TEST_MAIN
+#define BOOST_TEST_DYN_LINK
+#include 
+#include 
+#include "EMCALBase/Geometry.h"
+#include 
+#include 
+
+std::tuple GetRefCellIndex(int CellId);
+
+/// \macro Test implementation of the EMCAL Geometry
+///
+/// Test coverage:
+/// - GetCellIndex (get #sm, #mod, phi index and eta index): all cells (0-17663)
+/// - Invalid CellId: exception test for cell -1 and 17664
+BOOST_AUTO_TEST_CASE(Geometry_test)
+{
+  auto testgeometry = o2::emcal::Geometry::GetInstanceFromRunNumber(300000);
+
+  // Check GetCellIndex function for all valid cells by comparing to GetRefCellIndex function
+  for (int iCell = 0; iCell < 17664; iCell++) {
+    auto [smod, mod, iphi, ieta] = testgeometry->GetCellIndex(iCell);
+    auto [smod_ref, mod_ref, iphi_ref, ieta_ref] = GetRefCellIndex(iCell);
+    BOOST_CHECK_EQUAL(smod, smod_ref);
+    BOOST_CHECK_EQUAL(mod, mod_ref);
+    BOOST_CHECK_EQUAL(iphi, iphi_ref);
+    BOOST_CHECK_EQUAL(ieta, ieta_ref);
+  } // And then check the exeptions of -1 and 17664
+  BOOST_CHECK_EXCEPTION(testgeometry->GetCellIndex(-1), o2::emcal::InvalidCellIDException, [](o2::emcal::InvalidCellIDException const& mCellID) { return -1; });
+  BOOST_CHECK_EXCEPTION(testgeometry->GetCellIndex(17664), o2::emcal::InvalidCellIDException, [](o2::emcal::InvalidCellIDException const& mCellID) { return 17664; });
+}
+
+std::tuple GetRefCellIndex(int CellId)
+{
+  // Four cells per module:
+  int ieta = CellId % 2;                                   // cells 0 and 2 (in each module) have eta index 0
+  int iphi = (CellId % 4 == 2 || CellId % 4 == 3) ? 1 : 0; // cells 0 and 1 (in each module) have phi index 0
+
+  int smod = 0, mod = 0;                          // Super module number and module number
+  if (CellId >= 0 && CellId < 11520) {            // The first 10 super modules are full modules
+    smod = CellId / 1152;                         // Their number is their cell number divided by the cells per sm (rounded down)
+    mod = (CellId % 1152) / 4;                    // And the module is the cell number within the sm (%) divided by four (four cells in one module)
+  } else if (CellId >= 11520 && CellId < 12288) { // First two one thirds
+    smod = 10 + (CellId - 11520) / 384;           // +10 to account for the - 11520
+    mod = ((CellId - 11520) % 384) / 4;           // -11520 to subtract all cells in full super modules
+  } else if (CellId >= 12288 && CellId < 16896) { // Six two third modules
+    smod = 12 + (CellId - 12288) / 768;
+    mod = ((CellId - 12288) % 768) / 4;
+  } else if (CellId >= 16896 && CellId < 17664) { // Second two one third modules
+    smod = 18 + (CellId - 16896) / 384;
+    mod = ((CellId - 16896) % 384) / 4;
+  }
+
+  return std::make_tuple(smod, mod, iphi, ieta);
+}
\ No newline at end of file
diff --git a/Detectors/EMCAL/calib/CMakeLists.txt b/Detectors/EMCAL/calib/CMakeLists.txt
index 0ff2431c74a27..a8ca8d9505318 100644
--- a/Detectors/EMCAL/calib/CMakeLists.txt
+++ b/Detectors/EMCAL/calib/CMakeLists.txt
@@ -10,152 +10,165 @@
 # or submit itself to any jurisdiction.
 
 o2_add_library(EMCALCalib
-               SOURCES src/BadChannelMap.cxx
-                       src/TimeCalibrationParams.cxx
-                       src/TimeCalibParamL1Phase.cxx
-                       src/TempCalibrationParams.cxx
-                       src/TempCalibParamSM.cxx
-                       src/GainCalibrationFactors.cxx
-                       src/TriggerTRUDCS.cxx
-                       src/TriggerSTUDCS.cxx
-                       src/TriggerSTUErrorCounter.cxx
-                       src/TriggerDCS.cxx
-                       src/FeeDCS.cxx
-                       src/CalibDB.cxx
-                       src/ElmbMeasurement.cxx
-                       src/EMCALChannelScaleFactors.cxx
-               PUBLIC_LINK_LIBRARIES O2::CCDB O2::EMCALBase)
+        SOURCES src/CalibContainerErrors.cxx
+        src/BadChannelMap.cxx
+        src/TimeCalibrationParams.cxx
+        src/TimeCalibrationSlewingParams.cxx
+        src/TimeCalibParamL1Phase.cxx
+        src/TempCalibrationParams.cxx
+        src/TempCalibParamSM.cxx
+        src/GainCalibrationFactors.cxx
+        src/Pedestal.cxx
+        src/TriggerTRUDCS.cxx
+        src/TriggerSTUDCS.cxx
+        src/TriggerSTUErrorCounter.cxx
+        src/TriggerDCS.cxx
+        src/FeeDCS.cxx
+        src/CalibDB.cxx
+        src/ElmbMeasurement.cxx
+        src/EMCALChannelScaleFactors.cxx
+        src/CellRecalibrator.cxx
+        PUBLIC_LINK_LIBRARIES O2::CCDB O2::EMCALBase)
 
 o2_target_root_dictionary(EMCALCalib
-                          HEADERS include/EMCALCalib/BadChannelMap.h
-                                  include/EMCALCalib/TimeCalibrationParams.h
-                                  include/EMCALCalib/TimeCalibParamL1Phase.h
-                                  include/EMCALCalib/TempCalibrationParams.h
-                                  include/EMCALCalib/TempCalibParamSM.h
-                                  include/EMCALCalib/GainCalibrationFactors.h
-                                  include/EMCALCalib/TriggerTRUDCS.h
-                                  include/EMCALCalib/TriggerSTUDCS.h
-                                  include/EMCALCalib/TriggerSTUErrorCounter.h
-                                  include/EMCALCalib/TriggerDCS.h
-                                  include/EMCALCalib/FeeDCS.h
-                                  include/EMCALCalib/CalibDB.h
-                                  include/EMCALCalib/ElmbData.h
-                                  include/EMCALCalib/ElmbMeasurement.h
-                                  include/EMCALCalib/EMCALChannelScaleFactors.h
-                          LINKDEF src/EMCALCalibLinkDef.h)
+        HEADERS include/EMCALCalib/BadChannelMap.h
+        include/EMCALCalib/TimeCalibrationParams.h
+        include/EMCALCalib/TimeCalibrationSlewingParams.h
+        include/EMCALCalib/TimeCalibParamL1Phase.h
+        include/EMCALCalib/TempCalibrationParams.h
+        include/EMCALCalib/TempCalibParamSM.h
+        include/EMCALCalib/GainCalibrationFactors.h
+        include/EMCALCalib/Pedestal.h
+        include/EMCALCalib/TriggerTRUDCS.h
+        include/EMCALCalib/TriggerSTUDCS.h
+        include/EMCALCalib/TriggerSTUErrorCounter.h
+        include/EMCALCalib/TriggerDCS.h
+        include/EMCALCalib/FeeDCS.h
+        include/EMCALCalib/CalibDB.h
+        include/EMCALCalib/ElmbData.h
+        include/EMCALCalib/ElmbMeasurement.h
+        include/EMCALCalib/EMCALChannelScaleFactors.h
+        include/EMCALCalib/CellRecalibrator.h
+        LINKDEF src/EMCALCalibLinkDef.h)
 
 o2_add_test(BadChannelMap
-            SOURCES test/testBadChannelMap.cxx
-            PUBLIC_LINK_LIBRARIES O2::EMCALCalib
-            COMPONENT_NAME emcal
-            LABELS emcal)
+        SOURCES test/testBadChannelMap.cxx
+        PUBLIC_LINK_LIBRARIES O2::EMCALCalib
+        COMPONENT_NAME emcal
+        LABELS emcal)
 
 o2_add_test(TimeCalibrationParams
-            SOURCES test/testTimeCalibration.cxx
-            PUBLIC_LINK_LIBRARIES O2::EMCALCalib
-            COMPONENT_NAME emcal
-            LABELS emcal
-            ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage)
+        SOURCES test/testTimeCalibration.cxx
+        PUBLIC_LINK_LIBRARIES O2::EMCALCalib
+        COMPONENT_NAME emcal
+        LABELS emcal
+        ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage)
 
 o2_add_test(TimeCalibParamL1Phase
-            SOURCES test/testTimeL1PhaseCalib.cxx
-            PUBLIC_LINK_LIBRARIES O2::EMCALCalib
-            COMPONENT_NAME emcal
-            LABELS emcal
-            ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage)
+        SOURCES test/testTimeL1PhaseCalib.cxx
+        PUBLIC_LINK_LIBRARIES O2::EMCALCalib
+        COMPONENT_NAME emcal
+        LABELS emcal
+        ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage)
 
 o2_add_test(TempCalibrationParams
-            SOURCES test/testTempCalibration.cxx
-            PUBLIC_LINK_LIBRARIES O2::EMCALCalib
-            COMPONENT_NAME emcal
-            LABELS emcal
-            ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage)
+        SOURCES test/testTempCalibration.cxx
+        PUBLIC_LINK_LIBRARIES O2::EMCALCalib
+        COMPONENT_NAME emcal
+        LABELS emcal
+        ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage)
 
 o2_add_test(TempCalibParamSM
-            SOURCES test/testTempCalibrationSM.cxx
-            PUBLIC_LINK_LIBRARIES O2::EMCALCalib
-            COMPONENT_NAME emcal
-            LABELS emcal
-            ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage)
+        SOURCES test/testTempCalibrationSM.cxx
+        PUBLIC_LINK_LIBRARIES O2::EMCALCalib
+        COMPONENT_NAME emcal
+        LABELS emcal
+        ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage)
 
 o2_add_test(GainCalibrationFactors
-            SOURCES test/testGainCalibration.cxx
-            PUBLIC_LINK_LIBRARIES O2::EMCALCalib
-            COMPONENT_NAME emcal
-            LABELS emcal
-            ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage)
+        SOURCES test/testGainCalibration.cxx
+        PUBLIC_LINK_LIBRARIES O2::EMCALCalib
+        COMPONENT_NAME emcal
+        LABELS emcal
+        ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage)
+
+o2_add_test(Pedestal
+        SOURCES test/testPedestal.cxx
+        PUBLIC_LINK_LIBRARIES O2::EMCALCalib
+        COMPONENT_NAME emcal
+        LABELS emcal
+        ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage)
 
 o2_add_test(TriggerTRUDCS
-            SOURCES test/testTriggerTRUDCS.cxx
-            PUBLIC_LINK_LIBRARIES O2::EMCALCalib
-            COMPONENT_NAME emcal
-            LABELS emcal)
+        SOURCES test/testTriggerTRUDCS.cxx
+        PUBLIC_LINK_LIBRARIES O2::EMCALCalib
+        COMPONENT_NAME emcal
+        LABELS emcal)
 
 o2_add_test(TriggerSTUDCS
-            SOURCES test/testTriggerSTUDCS.cxx
-            PUBLIC_LINK_LIBRARIES O2::EMCALCalib
-            COMPONENT_NAME emcal
-            LABELS emcal)
+        SOURCES test/testTriggerSTUDCS.cxx
+        PUBLIC_LINK_LIBRARIES O2::EMCALCalib
+        COMPONENT_NAME emcal
+        LABELS emcal)
 
 o2_add_test(TriggerSTUErrorCounter
-            SOURCES test/testTriggerSTUErrorCounter.cxx
-            PUBLIC_LINK_LIBRARIES O2::EMCALCalib
-            COMPONENT_NAME emcal
-            LABELS emcal)
+        SOURCES test/testTriggerSTUErrorCounter.cxx
+        PUBLIC_LINK_LIBRARIES O2::EMCALCalib
+        COMPONENT_NAME emcal
+        LABELS emcal)
 
 o2_add_test(TriggerDCS
-            SOURCES test/testTriggerDCS.cxx
-            PUBLIC_LINK_LIBRARIES O2::EMCALCalib
-            COMPONENT_NAME emcal
-            LABELS emcal)
+        SOURCES test/testTriggerDCS.cxx
+        PUBLIC_LINK_LIBRARIES O2::EMCALCalib
+        COMPONENT_NAME emcal
+        LABELS emcal)
 
 o2_add_test_root_macro(macros/BadChannelMap_CCDBApitest.C
-            PUBLIC_LINK_LIBRARIES O2::EMCALCalib
-            LABELS emcal COMPILE_ONLY)
+        PUBLIC_LINK_LIBRARIES O2::EMCALCalib
+        LABELS emcal COMPILE_ONLY)
 
 o2_add_test_root_macro(macros/BadChannelMap_CalibDBtest.C
-            PUBLIC_LINK_LIBRARIES O2::EMCALCalib
-            LABELS emcal COMPILE_ONLY)
+        PUBLIC_LINK_LIBRARIES O2::EMCALCalib
+        LABELS emcal COMPILE_ONLY)
 
 o2_add_test_root_macro(macros/TimeCalibrationParams_CCDBApiTest.C
-            PUBLIC_LINK_LIBRARIES O2::EMCALCalib
-            LABELS emcal COMPILE_ONLY)
+        PUBLIC_LINK_LIBRARIES O2::EMCALCalib
+        LABELS emcal COMPILE_ONLY)
 
 o2_add_test_root_macro(macros/TimeCalibrationParams_CalibDBTest.C
-            PUBLIC_LINK_LIBRARIES O2::EMCALCalib
-            LABELS emcal COMPILE_ONLY)
+        PUBLIC_LINK_LIBRARIES O2::EMCALCalib
+        LABELS emcal COMPILE_ONLY)
 
 o2_add_test_root_macro(macros/TimeCalibParamsL1Phase_CCDBApiTest.C
-            PUBLIC_LINK_LIBRARIES O2::EMCALCalib
-            LABELS emcal COMPILE_ONLY)
+        PUBLIC_LINK_LIBRARIES O2::EMCALCalib
+        LABELS emcal COMPILE_ONLY)
 
 o2_add_test_root_macro(macros/TimeCalibParamsL1Phase_CalibDBTest.C
-            PUBLIC_LINK_LIBRARIES O2::EMCALCalib
-            LABELS emcal COMPILE_ONLY)
+        PUBLIC_LINK_LIBRARIES O2::EMCALCalib
+        LABELS emcal COMPILE_ONLY)
 
 o2_add_test_root_macro(macros/TempCalibrationParams_CCDBApiTest.C
-            PUBLIC_LINK_LIBRARIES O2::EMCALCalib
-            LABELS emcal COMPILE_ONLY)
+        PUBLIC_LINK_LIBRARIES O2::EMCALCalib
+        LABELS emcal COMPILE_ONLY)
 
 o2_add_test_root_macro(macros/TempCalibrationParams_CalibDBTest.C
-            PUBLIC_LINK_LIBRARIES O2::EMCALCalib
-            LABELS emcal COMPILE_ONLY)
+        PUBLIC_LINK_LIBRARIES O2::EMCALCalib
+        LABELS emcal COMPILE_ONLY)
 
 o2_add_test_root_macro(macros/TempCalibParamSM_CCDBApiTest.C
-            PUBLIC_LINK_LIBRARIES O2::EMCALCalib
-            LABELS emcal COMPILE_ONLY)
+        PUBLIC_LINK_LIBRARIES O2::EMCALCalib
+        LABELS emcal COMPILE_ONLY)
 
 o2_add_test_root_macro(macros/TempCalibParamSM_CalibDBTest.C
-            PUBLIC_LINK_LIBRARIES O2::EMCALCalib
-            LABELS emcal COMPILE_ONLY)
+        PUBLIC_LINK_LIBRARIES O2::EMCALCalib
+        LABELS emcal COMPILE_ONLY)
 
 o2_add_test_root_macro(macros/GainCalibrationFactors_CCDBApiTest.C
-            PUBLIC_LINK_LIBRARIES O2::EMCALCalib
-            LABELS emcal COMPILE_ONLY)
+        PUBLIC_LINK_LIBRARIES O2::EMCALCalib
+        LABELS emcal COMPILE_ONLY)
 
 o2_add_test_root_macro(macros/GainCalibrationFactors_CalibDBTest.C
-            PUBLIC_LINK_LIBRARIES O2::EMCALCalib
-            LABELS emcal COMPILE_ONLY)
+        PUBLIC_LINK_LIBRARIES O2::EMCALCalib
+        LABELS emcal COMPILE_ONLY)
 
 o2_data_file(COPY files DESTINATION Detectors/EMC)
-
diff --git a/Detectors/EMCAL/calib/include/EMCALCalib/BadChannelMap.h b/Detectors/EMCAL/calib/include/EMCALCalib/BadChannelMap.h
index d2b728dbe0dfa..043297b95c206 100644
--- a/Detectors/EMCAL/calib/include/EMCALCalib/BadChannelMap.h
+++ b/Detectors/EMCAL/calib/include/EMCALCalib/BadChannelMap.h
@@ -106,6 +106,7 @@ class BadChannelMap
   /// \brief Add bad cell to the container
   /// \param channelID Absolute ID of the bad channel
   /// \param mask type of the bad channel
+  /// \throw CalibContainerIndexException in case the cell ID exceeds the range of cells in EMCAL
   ///
   /// Adding new bad channel to the container. In case a cell
   /// with the same ID is already present in the container,
@@ -119,6 +120,7 @@ class BadChannelMap
   /// \brief Get the status of a certain cell
   /// \param channelID channel for which to obtain the channel status
   /// \return Mask status of the cell (GOOD_CELL if not registered)
+  /// \throw CalibContainerIndexException in case the cell ID exceeds the range of cells in EMCAL
   ///
   /// Provide the mask status of a cell. In case the cell is registered
   /// in the container the mask status registered is returned, otherwise
diff --git a/Detectors/EMCAL/calib/include/EMCALCalib/CalibContainerErrors.h b/Detectors/EMCAL/calib/include/EMCALCalib/CalibContainerErrors.h
new file mode 100644
index 0000000000000..4d1830207d1a2
--- /dev/null
+++ b/Detectors/EMCAL/calib/include/EMCALCalib/CalibContainerErrors.h
@@ -0,0 +1,62 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#ifndef ALICEO2_EMCAL_CALIBCONTAINERERORS_H
+#define ALICEO2_EMCAL_CALIBCONTAINERERORS_H
+
+#include 
+#include 
+#include 
+
+namespace o2
+{
+
+namespace emcal
+{
+
+/// \class CalibContainerIndexException
+/// \brief Error handling for invalid index in calibration request
+/// \ingroup EMCALCalib
+/// \author Markus Fasel , Oak Ridge National Laboratory
+/// \since Sept 15, 2022
+class CalibContainerIndexException : public std::exception
+{
+ public:
+  /// \brief Constructor
+  /// \param index Index in container raising the exception
+  CalibContainerIndexException(unsigned int index);
+
+  /// \brief Destructor
+  ~CalibContainerIndexException() noexcept final = default;
+
+  /// \brief Access to error message of the exception
+  /// \return Error message
+  const char* what() const noexcept final { return mErrorMessage.data(); }
+
+  /// \brief Access to index raising the exception
+  /// \return Index raising the exception
+  unsigned int getIndex() const noexcept { return mIndex; }
+
+ private:
+  unsigned int mIndex;       ///< Index raising the error message
+  std::string mErrorMessage; ///< Buffer for error message
+};
+
+/// \brief Output stream operator for CalibContainerIndexException
+/// \param stream Stream where the error message should be displayed
+/// \param obj Exception object to be streamed
+/// \return Stream after the message
+std::ostream& operator<<(std::ostream& stream, const CalibContainerIndexException& obj);
+
+} // namespace emcal
+
+} // namespace o2
+
+#endif // !ALICEO2_EMCAL_CALIBCONTAINERERORS_H
\ No newline at end of file
diff --git a/Detectors/EMCAL/calib/include/EMCALCalib/CalibDB.h b/Detectors/EMCAL/calib/include/EMCALCalib/CalibDB.h
index 01fb64884edce..d4fc0329c0bf5 100644
--- a/Detectors/EMCAL/calib/include/EMCALCalib/CalibDB.h
+++ b/Detectors/EMCAL/calib/include/EMCALCalib/CalibDB.h
@@ -8,12 +8,17 @@
 // In applying this license CERN does not waive the privileges and immunities
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
+
+#ifndef ALICEO2_EMCAL_CALIBDB
+#define ALICEO2_EMCAL_CALIBDB
+
 #include 
 #include 
 #include 
 #include "Rtypes.h"
 #include "RStringView.h"
 #include "CCDB/CcdbApi.h"
+#include "CCDB/BasicCCDBManager.h"
 
 namespace o2
 {
@@ -27,8 +32,10 @@ class TempCalibParamSM;
 class TimeCalibrationParams;
 class TimeCalibParamL1Phase;
 class GainCalibrationFactors;
+class EMCALChannelScaleFactors;
 class FeeDCS;
 class ElmbData;
+class Pedestal;
 
 /// \class CalibDB
 /// \brief Interface to calibration data from CCDB for EMCAL
@@ -43,6 +50,7 @@ class ElmbData;
 /// - Time calibration
 /// - Gain calibration
 /// - Temperature calibration
+/// - Pedestals
 /// Users only need to specify the CCDB server, the timestamp and
 /// (optionally) additional meta data. Handling of the CCDB path
 /// and type conversions is done internally - users deal directly
@@ -58,6 +66,9 @@ class ElmbData;
 /// Users must handle the exceptions.
 class CalibDB
 {
+
+  using CcdbManager = o2::ccdb::BasicCCDBManager;
+
  public:
   /// \class ObjectNotFoundException
   /// \brief Handling errors due to objects not found in the CCDB
@@ -257,6 +268,13 @@ class CalibDB
   /// \throw TypeMismatchException if object is present but type is different (CCDB corrupted)
   GainCalibrationFactors* readGainCalibFactors(ULong_t timestamp, const std::map& metadata);
 
+  /// \brief Find scale factors used for bad channel calibration in the CCDB for given timestamp
+  /// \param timestamp Timestamp used in query (there is only one entry in the CCDB)
+  /// \param metadata Additional metadata to be used in the query
+  /// \throw ObjectNotFoundException if object is not found for the given timestamp
+  /// \throw TypeMismatchException if object is present but type is different (CCDB corrupted)
+  EMCALChannelScaleFactors* readChannelScaleFactors(ULong_t timestamp, const std::map& metadata);
+
   /// \brief Store FEE DCS data in the CCDB
   /// \param dcs FEE DCS data to be stored
   /// \param metadata Additional metadata that can be used in the query
@@ -285,6 +303,20 @@ class CalibDB
   /// \throw TypeMismatchException if object is present but type is different (CCDB corrupted)
   ElmbData* readTemperatureSensorData(ULong_t timestamp, const std::map& metadata);
 
+  /// \brief Store pedestal data in the CCDB
+  /// \param pedestals Pedestal data to be stored
+  /// \param metadata Additional metadata that can be used in the query
+  /// \param timestart Start of the time range of the validity of the object
+  /// \param timeend End of the time range of the validity of the object
+  void storePedestalData(Pedestal* pedestals, const std::map& metadata, ULong_t timestart, ULong_t timeend);
+
+  /// \brief Find pedestal data in the CCDB for given timestamp
+  /// \param timestamp Timestamp used in query
+  /// \param metadata Additional metadata to be used in the query
+  /// \throw ObjectNotFoundException if object is not found for the given timestamp
+  /// \throw TypeMismatchException if object is present but type is different (CCDB corrupted)
+  Pedestal* readPedestalData(ULong_t timestamp, const std::map& metadata);
+
   /// \brief Set new CCDB server URL
   /// \param server Name of the CCDB server to be used in queries
   ///
@@ -329,6 +361,14 @@ class CalibDB
   /// \return Path of the Temperature Sensor data in the CCDB
   static const char* getCDBPathTemperatureSensor() { return "EMC/Calib/Temperature"; }
 
+  /// \brief Get CCDB path for the scale factors used in the bad channel calibration
+  /// \return Path of the scale factors used in the bad channel calibration in the CCDB
+  static const char* getCDBPathChannelScaleFactors() { return "EMC/Config/ChannelScaleFactors"; }
+
+  /// \brief Get CCDB path for the pedestal data
+  /// \return Path of the pedestal data
+  static const char* getCDBPathChannelPedestals() { return "EMC/Calib/Pedestal"; }
+
  private:
   /// \brief Initialize CCDB server (when new object is created or the server URL changes)
   void
@@ -338,8 +378,10 @@ class CalibDB
   std::string mCCDBServer = "emcccdb-test.cern.ch"; ///< Name of the CCDB server
   Bool_t mInit = false;                             ///< Init status (needed for lazy evaluation of the CcdbApi init)
 
-  ClassDefNV(CalibDB, 1);
+  ClassDefNV(CalibDB, 2);
 };
 } // namespace emcal
 
 } // namespace o2
+
+#endif
\ No newline at end of file
diff --git a/Detectors/EMCAL/calib/include/EMCALCalib/CellRecalibrator.h b/Detectors/EMCAL/calib/include/EMCALCalib/CellRecalibrator.h
new file mode 100644
index 0000000000000..ea8a0445bbe5e
--- /dev/null
+++ b/Detectors/EMCAL/calib/include/EMCALCalib/CellRecalibrator.h
@@ -0,0 +1,211 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#ifndef ALCEO2_EMCAL_CELLRECALIBRATOR_H
+#define ALCEO2_EMCAL_CELLRECALIBRATOR_H
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "Rtypes.h"
+
+#include 
+#include 
+#include 
+#include 
+
+namespace o2
+{
+
+namespace emcal
+{
+
+/// \class CellRecalibrator
+/// \brief Tool for recalibration at cell level
+/// \ingroup EMCALcalib
+/// \author Markus Fasel  Oak Ridge National Laboratory
+/// \since Oct 12, 2022
+///
+/// Applying cell-level calibrations
+/// - bad channel removal
+/// - time shift
+/// - gain calibration
+///
+/// Attention: All calibrations for which calibration objects
+/// are provided are applied. Check for active calibration is
+/// therefore related to the presence of the corresponding
+/// calibration object.
+///
+/// Input can be a single cell (getCalibratedCell) or a
+/// collection of cells (getCalibratedCells). In case of
+/// single cell the result is optional since the cell can
+/// be rejected by the bad channel mask. Only cells of type
+/// high gain or low gain can be calibrated, in case cells
+/// of other types (LEDMON/TRU) are passed to getCalibratedCell
+/// an exception will be thrown.
+///
+/// The calibrator supports all cells of the CellInterface
+/// concept.
+class CellRecalibrator
+{
+ public:
+  /// \class CellTypeException
+  /// \brief Handling of invalid cell types in calibration
+  class CellTypeException final : public std::exception
+  {
+   public:
+    /// \brief Constructor
+    CellTypeException() = default;
+
+    /// \brief Destructor
+    ~CellTypeException() noexcept final = default;
+
+    /// \brief Get error message of the exception
+    /// \return Error message
+    [[nodiscard]] char const* what() const noexcept final
+    {
+      return "Only possible to calibrate cells of type high gain or low gain";
+    }
+  };
+
+  /// \brief Constructor
+  CellRecalibrator() = default;
+
+  /// \brief Destructor
+  ~CellRecalibrator() = default;
+
+  /// \brief Set the bad channel map
+  /// \param bcm Bad channel map to be applied
+  void setBadChannelMap(const BadChannelMap* bcm) { mBadChannelMap = bcm; }
+
+  /// \brief Set the time calibration params
+  /// \param tcp Time calibration params to be applied
+  void setTimeCalibration(const TimeCalibrationParams* tcp) { mTimeCalibration = tcp; }
+
+  /// \brief Set the gain calibration params
+  /// \param gcf Gain calibration factors to be applied
+  void setGainCalibration(const GainCalibrationFactors* gcf) { mGainCalibration = gcf; }
+
+  /// \brief Check if the bad channel calibration is enabled
+  /// \return True if the bad channel calibration is active (object available), false otherwise
+  bool hasBadChannelMap() const { return mBadChannelMap != nullptr; }
+
+  /// \brief Check if the time calibration is enabled
+  /// \return True if the time calibration is active (object available), false otherwise
+  bool hasTimeCalibration() const { return mTimeCalibration != nullptr; }
+
+  /// \brief Check if the energy calibration is enabled
+  /// \return True if the energy calibration is active (object available), false otherwise
+  bool hasGainCalibration() const { return mGainCalibration != nullptr; }
+
+  /// \brief Get bad channel map currently used in the calibrator
+  /// \return Current bad channel map (nullptr if not set)
+  const BadChannelMap* getBadChannelMap() const { return mBadChannelMap; }
+
+  /// \brief Get time calibration parameters currently used in the calibrator
+  /// \return Current time calibration parameters (nullptr if not set)
+  const TimeCalibrationParams* getTimeCalibration() const { return mTimeCalibration; }
+
+  /// \brief Get gain calibration factors currently used in the calibrator
+  /// \return Current gain calibration factors (nullptr if not set)
+  const GainCalibrationFactors* getGainCalibration() const { return mGainCalibration; }
+
+  /// \brief Calibrate single cell
+  ///
+  /// Applying all calibrations provided based on presence of calibration objects. Only
+  /// cells of type high-gain or low-gain can be calibrated. Since the cell can be rejected
+  /// by the bad channel calibration the return type is std::optional, where an empty optional
+  /// reflects rejected cells.
+  ///
+  /// \param inputcell Cell to calibrate
+  /// \return Calibrated cell (empty optional if rejected by the bad channel map)
+  /// \throw CellTypeException in case the inputcell is neither of type high gain nor low gain
+  template 
+  std::optional getCalibratedCell(const T& inputcell) const
+  {
+    if (!(inputcell.getHighGain() || inputcell.getLowGain())) {
+      throw CellTypeException();
+    }
+    if (hasBadChannelMap()) {
+      if (mBadChannelMap->getChannelStatus(inputcell.getTower()) != BadChannelMap::MaskType_t::GOOD_CELL) {
+        return std::optional();
+      }
+    }
+
+    float calibratedEnergy = inputcell.getEnergy();
+    float calibratedTime = inputcell.getTimeStamp();
+
+    if (hasTimeCalibration()) {
+      calibratedTime -= mTimeCalibration->getTimeCalibParam(inputcell.getTower(), inputcell.getLowGain());
+    }
+
+    if (hasGainCalibration()) {
+      calibratedEnergy *= mGainCalibration->getGainCalibFactors(inputcell.getTower());
+    }
+    return std::make_optional(inputcell.getTower(), calibratedEnergy, calibratedTime, inputcell.getHighGain() ? ChannelType_t::HIGH_GAIN : ChannelType_t::LOW_GAIN);
+  }
+
+  /// \brief Get list of calibrated cells based on a cell input collection
+  ///
+  /// Applying calibrations to all cells in the input collections and return
+  /// a vector of accepted and calibrated cells. Cells not of type high gain
+  /// or low gain are discarded. All calibrations for which calibration objects
+  /// are available are applied.
+  ///
+  /// \param inputcells Collection of input cells
+  /// \return Tuple with vector of calibrated cells and vector of kept indices.
+  template 
+  std::tuple, std::vector> getCalibratedCells(const gsl::span inputcells)
+  {
+    std::vector result;
+    std::vector indices;
+    int currentindex = 0;
+    for (const auto& cellToCalibrate : inputcells) {
+      if (!(cellToCalibrate.getHighGain() || cellToCalibrate.getLowGain())) {
+        currentindex++;
+        continue;
+      }
+      auto calibrated = getCalibratedCell(cellToCalibrate);
+      if (calibrated) {
+        result.push_back(calibrated.value());
+        indices.emplace_back(currentindex);
+      }
+      currentindex++;
+    }
+    return std::make_tuple(result, indices);
+  }
+
+  /// \brief Print settings to the stream
+  /// \param stream Stream to print on
+  void printStream(std::ostream& stream) const;
+
+ private:
+  const BadChannelMap* mBadChannelMap = nullptr;            ///< Bad channel map
+  const TimeCalibrationParams* mTimeCalibration = nullptr;  ///< Time calibration parameters
+  const GainCalibrationFactors* mGainCalibration = nullptr; ///< Gain calibration parameters
+
+  ClassDefNV(CellRecalibrator, 1);
+};
+
+/// \brief Output stream operator for cell-level calibrator
+/// \param in Stream to print on
+/// \param calib CellRecalibrator object to be printed
+/// \return Stream after printing
+std::ostream& operator<<(std::ostream& in, const CellRecalibrator& calib);
+
+} // namespace emcal
+
+} // namespace o2
+
+#endif // !ALCEO2_EMCAL_CELLRECALIBRATOR_H
diff --git a/Detectors/EMCAL/calib/include/EMCALCalib/EMCALChannelScaleFactors.h b/Detectors/EMCAL/calib/include/EMCALCalib/EMCALChannelScaleFactors.h
index 81a5c126a46b7..b954ff96c07ad 100644
--- a/Detectors/EMCAL/calib/include/EMCALCalib/EMCALChannelScaleFactors.h
+++ b/Detectors/EMCAL/calib/include/EMCALCalib/EMCALChannelScaleFactors.h
@@ -33,10 +33,10 @@ class InvalidEnergyIntervalException final : public std::exception
  public:
   /// \brief Constructor
   /// \param E Cell energy requested
-  InvalidEnergyIntervalException(float E, int cellID) : std::exception(),
-                                                        mE(E),
-                                                        mCellID(cellID),
-                                                        mMessage("Invalid energy " + std::to_string(mE) + "for cell " + std::to_string(mCellID) + "]")
+  InvalidEnergyIntervalException(float E, unsigned int cellID) : std::exception(),
+                                                                 mE(E),
+                                                                 mCellID(cellID),
+                                                                 mMessage("Invalid energy " + std::to_string(mE) + "for cell " + std::to_string(mCellID) + "]")
   {
   }
 
@@ -49,7 +49,7 @@ class InvalidEnergyIntervalException final : public std::exception
 
  private:
   float mE;             ///< Cell energy requested
-  int mCellID;          ///< Cell ID
+  unsigned int mCellID; ///< Cell ID
   std::string mMessage; ///< Message to be printed
 };
 
@@ -136,13 +136,16 @@ class EMCALChannelScaleFactors
   /// \param E_min Minimum energy of the interval
   /// \param E_max Maximum energy of the interval
   /// \param scale Scale factor for number of hits in bad channel calibration
-  void insertVal(const int cellID, float E_min, float E_max, float scale);
+  /// \throw CalibContainerIndexException in case the cell ID exceeds the range of cells in EMCAL
+  void insertVal(unsigned int cellID, float E_min, float E_max, float scale);
 
   /// Get the scale factor for a given cell and energy
   /// \param cell The cell number
   /// \param E The energy
   /// \return The scale factor
-  float getScaleVal(const int cellID, float E) const;
+  /// \throw CalibContainerIndexException in case the cell ID exceeds the range of cells in EMCAL
+  /// \throw InvalidEnergyIntervalException in case no energy interval is found for the requested energy value and cell ID
+  float getScaleVal(unsigned int cellID, float E) const;
 
  private:
   static constexpr int NCells = 17664;                               ///< Number of cells in the EMCal
diff --git a/Detectors/EMCAL/calib/include/EMCALCalib/GainCalibrationFactors.h b/Detectors/EMCAL/calib/include/EMCALCalib/GainCalibrationFactors.h
index 1f048dc9738c0..a5271a5807705 100644
--- a/Detectors/EMCAL/calib/include/EMCALCalib/GainCalibrationFactors.h
+++ b/Detectors/EMCAL/calib/include/EMCALCalib/GainCalibrationFactors.h
@@ -9,6 +9,21 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 
+#ifndef GAINCALIBRATIONFACTORS_H_
+#define GAINCALIBRATIONFACTORS_H_
+
+#include 
+#include 
+#include 
+
+class TH1;
+
+namespace o2
+{
+
+namespace emcal
+{
+
 /// \class GainCalibrationFactors
 /// \brief CCDB container for the gain calibration factors
 /// \ingroup EMCALcalib
@@ -27,22 +42,6 @@
 /// This will return the gain calibration factor for a certain channel.
 /// ~~~
 ///
-
-#ifndef GAINCALIBRATIONFACTORS_H_
-#define GAINCALIBRATIONFACTORS_H_
-
-#include 
-#include 
-#include 
-
-class TH1;
-
-namespace o2
-{
-
-namespace emcal
-{
-
 class GainCalibrationFactors
 {
  public:
@@ -54,11 +53,13 @@ class GainCalibrationFactors
 
   /// \brief Comparison of two gain calibration factors containers
   /// \return true if the two list of gain calibration factors are the same, false otherwise
+  /// \throw CalibContainerIndexException in case the cell ID exceeds the range of cells in EMCAL
   bool operator==(const GainCalibrationFactors& other) const;
 
   /// \brief Add gain calibration factors to the container
   /// \param iCell is the cell index
   /// \param gainFactor is the gain calibration factor
+  /// \throw CalibContainerIndexException in case the cell ID exceeds the range of cells in EMCAL
   void addGainCalibFactor(unsigned short iCell, float gainFactor);
 
   /// \brief Get the gain calibration factor for a certain cell
diff --git a/Detectors/EMCAL/calib/include/EMCALCalib/Pedestal.h b/Detectors/EMCAL/calib/include/EMCALCalib/Pedestal.h
new file mode 100644
index 0000000000000..7a8983409d995
--- /dev/null
+++ b/Detectors/EMCAL/calib/include/EMCALCalib/Pedestal.h
@@ -0,0 +1,96 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#ifndef ALICEO2_EMCAL_PEDESTAL_H
+#define ALICEO2_EMCAL_PEDESTAL_H
+
+#include 
+#include 
+#include 
+
+class TH1;
+class TH2;
+
+namespace o2
+{
+
+namespace emcal
+{
+
+/// \class Pedestal
+/// \brief CCDB container for pedestal values
+/// \ingroup EMCALcalib
+/// \author Markus Fasel , Oak Ridge National Laboratory
+/// \since July 16th, 2019
+///
+/// Pedestal values can be added for each channel by
+/// ~~~.{cxx}
+/// o2::emcal::Pedestal ped;
+/// ped.addPedestalValue(23, 3, false);
+/// ~~~
+/// For the High Gain cells the last parameter should be set to false, for low
+/// gain it should be set to true.
+///
+/// One can read the pedestal values by calling
+/// ~~~.{cxx}
+/// auto param = ped.getPedestalValue(23, false);
+/// ~~~
+/// This will return the pedestal value for a certain HG cell.
+/// For low gain cells you have to set the last parameter false
+class Pedestal
+{
+ public:
+  /// \brief Constructor
+  Pedestal() = default;
+
+  /// \brief Destructor
+  ~Pedestal() = default;
+
+  /// \brief Comparison of two pedestal containers
+  /// \return true if the two list of pedestal values are the same, false otherwise
+  bool operator==(const Pedestal& other) const;
+
+  /// \brief Add pedestal to the container
+  /// \param cellID Absolute ID of cell
+  /// \param isLowGain Cell type is low gain cell
+  /// \param pedestal Pedestal value
+  /// \throw CalibContainerIndexException in case the cell ID exceeds the range of cells in EMCAL
+  void addPedestalValue(unsigned short cellID, short pedestal, bool isLowGain, bool isLEDMON);
+
+  /// \brief Get the time calibration coefficient for a certain cell
+  /// \param cellID Absolute ID of cell
+  /// \param isLowGain Cell type is low gain cell
+  /// \return Pedestal value of the cell
+  /// \throw CalibContainerIndexException in case the cell ID exceeds the range of cells in EMCAL
+  short getPedestalValue(unsigned short cellID, bool isLowGain, bool isLEDMON) const;
+
+  /// \brief Convert the pedestal container to a histogram
+  /// \param isLowGain Monitor low gain cells
+  /// \return Histogram representation of the pedestal container
+  TH1* getHistogramRepresentation(bool isLowGain, bool isLEDMON) const;
+
+  /// \brief Convert the pedestal container to a 2D histogram
+  /// \param isLowGain Monitor low gain cells
+  /// \return 2D Histogram representation (heatmap with respect to column and row) of the pedestal container
+  TH2* getHistogramRepresentation2D(bool isLowGain, bool isLEDMON) const;
+
+ private:
+  std::array mPedestalValuesHG;     ///< Container for the pedestal values (high gain)
+  std::array mPedestalValuesLG;     ///< Container for the pedestal values (low gain)
+  std::array mPedestalValuesLEDMONHG; ///< Container for the LEDMON pedestal values (high gain)
+  std::array mPedestalValuesLEDMONLG; ///< Container for the LEDMON pedestal values (low gain)
+
+  ClassDefNV(Pedestal, 1);
+};
+
+} // namespace emcal
+
+} // namespace o2
+#endif
diff --git a/Detectors/EMCAL/calib/include/EMCALCalib/TimeCalibrationParams.h b/Detectors/EMCAL/calib/include/EMCALCalib/TimeCalibrationParams.h
index 8733448beea2b..954d1421c8a25 100644
--- a/Detectors/EMCAL/calib/include/EMCALCalib/TimeCalibrationParams.h
+++ b/Detectors/EMCAL/calib/include/EMCALCalib/TimeCalibrationParams.h
@@ -63,12 +63,14 @@ class TimeCalibrationParams
   /// \param cellID Absolute ID of cell
   /// \param time is the calibration coefficient
   /// \param isLowGain is flag whether this cell is LG or HG
+  /// \throw CalibContainerIndexException in case the cell ID exceeds the range of cells in EMCAL
   void addTimeCalibParam(unsigned short cellID, short time, bool isLowGain);
 
   /// \brief Get the time calibration coefficient for a certain cell
   /// \param cellID Absolute ID of cell
   /// \param isLowGain is flag whether this cell is LG or HG
   /// \return time calibration coefficient of the cell
+  /// \throw CalibContainerIndexException in case the cell ID exceeds the range of cells in EMCAL
   short getTimeCalibParam(unsigned short cellID, bool isLowGain) const;
 
   /// \brief Convert the time calibration coefficient array to a histogram
diff --git a/Detectors/EMCAL/calib/include/EMCALCalib/TimeCalibrationSlewingParams.h b/Detectors/EMCAL/calib/include/EMCALCalib/TimeCalibrationSlewingParams.h
new file mode 100644
index 0000000000000..21b69d675b339
--- /dev/null
+++ b/Detectors/EMCAL/calib/include/EMCALCalib/TimeCalibrationSlewingParams.h
@@ -0,0 +1,73 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \class TimeCalibrationSlewingParams
+/// \brief CCDB container for the time calibration slewing coefficients
+/// \ingroup EMCALcalib
+/// \author Joshua Konig , Goethe-University Frankfurt
+/// \since Apr 5th, 2023
+
+#ifndef TIMECALIBRATIONSLEWINGPARAMS_H_
+#define TIMECALIBRATIONSLEWINGPARAMS_H_
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+namespace o2
+{
+
+namespace emcal
+{
+
+class TimeCalibrationSlewingParams
+{
+
+ public:
+  /// \brief Constructor
+  TimeCalibrationSlewingParams() = default;
+
+  /// \brief Constructor
+  /// \param arr parameters of third oder polynomial function
+  TimeCalibrationSlewingParams(std::array arr);
+
+  /// \brief Destructor
+  ~TimeCalibrationSlewingParams() = default;
+
+  /// \brief Comparison of two time calibration coefficients
+  /// \return true if the two list of time calibration coefficients are the same, false otherwise
+  bool operator==(const TimeCalibrationSlewingParams& other) const;
+
+  /// \brief Add time calibration slewing function to the container
+  /// \param arr parameter for 3rd order polynomial function
+  void addTimeSlewingParam(const std::array arr);
+
+  /// \brief Get time calibration slewing parameter index of function
+  /// \param index index for parameter
+  double getTimeSlewingParam(const unsigned int index) const;
+
+  /// \brief Get value of time calib slewing for a given energy
+  /// \param energy energy of cell
+  double eval(const double energy) const;
+
+ private:
+  std::array arrParams; ///< parameter for 3rd order polynomial
+
+  // ClassDefOverride(TimeCalibrationSlewingParams, 1);
+  ClassDefNV(TimeCalibrationSlewingParams, 1);
+};
+
+} // namespace emcal
+
+} // namespace o2
+#endif // TIMECALIBRATIONSLEWINGPARAMS_H_
diff --git a/Detectors/EMCAL/calib/src/BadChannelMap.cxx b/Detectors/EMCAL/calib/src/BadChannelMap.cxx
index 23151f6486a21..03476b3a7508f 100644
--- a/Detectors/EMCAL/calib/src/BadChannelMap.cxx
+++ b/Detectors/EMCAL/calib/src/BadChannelMap.cxx
@@ -11,8 +11,9 @@
 
 #include "EMCALBase/Geometry.h"
 #include "EMCALCalib/BadChannelMap.h"
+#include "EMCALCalib/CalibContainerErrors.h"
 
-#include "FairLogger.h"
+#include 
 
 #include 
 
@@ -22,6 +23,9 @@ using namespace o2::emcal;
 
 void BadChannelMap::addBadChannel(unsigned short channelID, MaskType_t mask)
 {
+  if (channelID >= 17664) {
+    throw CalibContainerIndexException(channelID);
+  }
   switch (mask) {
     case MaskType_t::GOOD_CELL:
       mBadCells.reset(channelID);
@@ -48,6 +52,9 @@ void BadChannelMap::addBadChannel(unsigned short channelID, MaskType_t mask)
 
 BadChannelMap::MaskType_t BadChannelMap::getChannelStatus(unsigned short channelID) const
 {
+  if (channelID >= 17664) {
+    throw CalibContainerIndexException(channelID);
+  }
   auto status = MaskType_t::GOOD_CELL;
   if (mDeadCells.test(channelID)) {
     status = MaskType_t::DEAD_CELL;
@@ -65,25 +72,24 @@ TH2* BadChannelMap::getHistogramRepresentation() const
             MAXCOLS = 96;
   auto hist = new TH2S("badchannelmap", "Bad Channel Map", MAXCOLS, -0.5, double(MAXCOLS) - 0.5, MAXROWS, -0.5, double(MAXROWS) - 0.5);
   hist->SetDirectory(nullptr);
-  auto geo = Geometry::GetInstance();
-  if (!geo) {
-    LOG(error) << "Geometry needs to be initialized";
-    return hist;
-  }
-
-  for (size_t cellID = 0; cellID < mBadCells.size(); cellID++) {
-    int value(0);
-    if (mBadCells.test(cellID)) {
-      value = 1;
-    } else if (mDeadCells.test(cellID)) {
-      value = 2;
-    } else if (mWarmCells.test(cellID)) {
-      value = 3;
-    }
-    if (value) {
-      auto position = geo->GlobalRowColFromIndex(cellID);
-      hist->Fill(std::get<1>(position), std::get<0>(position), value);
+  try {
+    auto geo = Geometry::GetInstance();
+    for (size_t cellID = 0; cellID < mBadCells.size(); cellID++) {
+      int value(0);
+      if (mBadCells.test(cellID)) {
+        value = 1;
+      } else if (mDeadCells.test(cellID)) {
+        value = 2;
+      } else if (mWarmCells.test(cellID)) {
+        value = 3;
+      }
+      if (value) {
+        auto position = geo->GlobalRowColFromIndex(cellID);
+        hist->Fill(std::get<1>(position), std::get<0>(position), value);
+      }
     }
+  } catch (o2::emcal::GeometryNotInitializedException& e) {
+    LOG(error) << "Geometry needs to be initialized";
   }
   return hist;
 }
diff --git a/Detectors/EMCAL/calib/src/CalibContainerErrors.cxx b/Detectors/EMCAL/calib/src/CalibContainerErrors.cxx
new file mode 100644
index 0000000000000..c55f4e402c925
--- /dev/null
+++ b/Detectors/EMCAL/calib/src/CalibContainerErrors.cxx
@@ -0,0 +1,26 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include 
+#include "EMCALCalib/CalibContainerErrors.h"
+
+using namespace o2::emcal;
+
+CalibContainerIndexException::CalibContainerIndexException(unsigned int index) : std::exception(), mIndex(index), mErrorMessage()
+{
+  mErrorMessage = "Invalid index: " + std::to_string(mIndex);
+}
+
+std::ostream& o2::emcal::operator<<(std::ostream& stream, const o2::emcal::CalibContainerIndexException& obj)
+{
+  stream << obj.what();
+  return stream;
+}
\ No newline at end of file
diff --git a/Detectors/EMCAL/calib/src/CalibDB.cxx b/Detectors/EMCAL/calib/src/CalibDB.cxx
index dd5094003736d..44dcf728b598c 100644
--- a/Detectors/EMCAL/calib/src/CalibDB.cxx
+++ b/Detectors/EMCAL/calib/src/CalibDB.cxx
@@ -15,9 +15,11 @@
 #include "EMCALCalib/TimeCalibParamL1Phase.h"
 #include "EMCALCalib/TempCalibParamSM.h"
 #include "EMCALCalib/GainCalibrationFactors.h"
+#include "EMCALCalib/EMCALChannelScaleFactors.h"
 #include "EMCALCalib/FeeDCS.h"
 #include "EMCALCalib/CalibDB.h"
 #include "EMCALCalib/ElmbData.h"
+#include "EMCALCalib/Pedestal.h"
 
 using namespace o2::emcal;
 
@@ -29,6 +31,8 @@ CalibDB::CalibDB(const std::string_view server) : CalibDB()
 void CalibDB::init()
 {
   mCCDBManager.init(mCCDBServer);
+  auto& mgr = CcdbManager::instance();
+  mgr.setURL(mCCDBServer);
   mInit = true;
 }
 
@@ -96,12 +100,21 @@ void CalibDB::storeTemperatureSensorData(ElmbData* dcs, const std::map& metadata, ULong_t rangestart, ULong_t rangeend)
+{
+  if (!mInit) {
+    init();
+  }
+  mCCDBManager.storeAsTFileAny(pedestals, getCDBPathTemperatureSensor(), metadata, rangestart, rangeend);
+}
+
 BadChannelMap* CalibDB::readBadChannelMap(ULong_t timestamp, const std::map& metadata)
 {
   if (!mInit) {
     init();
   }
-  BadChannelMap* result = mCCDBManager.retrieveFromTFileAny(getCDBPathBadChannelMap(), metadata, timestamp);
+  auto& mgr = CcdbManager::instance();
+  BadChannelMap* result = mgr.getForTimeStamp(getCDBPathBadChannelMap(), timestamp);
   if (!result) {
     throw ObjectNotFoundException(mCCDBServer, getCDBPathBadChannelMap(), metadata, timestamp);
   }
@@ -113,7 +126,8 @@ TimeCalibrationParams* CalibDB::readTimeCalibParam(ULong_t timestamp, const std:
   if (!mInit) {
     init();
   }
-  TimeCalibrationParams* result = mCCDBManager.retrieveFromTFileAny(getCDBPathTimeCalibrationParams(), metadata, timestamp);
+  auto& mgr = CcdbManager::instance();
+  TimeCalibrationParams* result = mgr.getForTimeStamp(getCDBPathTimeCalibrationParams(), timestamp);
   if (!result) {
     throw ObjectNotFoundException(mCCDBServer, getCDBPathTimeCalibrationParams(), metadata, timestamp);
   }
@@ -125,7 +139,8 @@ TimeCalibParamL1Phase* CalibDB::readTimeCalibParamL1Phase(ULong_t timestamp, con
   if (!mInit) {
     init();
   }
-  TimeCalibParamL1Phase* result = mCCDBManager.retrieveFromTFileAny(getCDBPathL1Phase(), metadata, timestamp);
+  auto& mgr = CcdbManager::instance();
+  TimeCalibParamL1Phase* result = mgr.getForTimeStamp(getCDBPathL1Phase(), timestamp);
   if (!result) {
     throw ObjectNotFoundException(mCCDBServer, getCDBPathL1Phase(), metadata, timestamp);
   }
@@ -137,7 +152,8 @@ TempCalibrationParams* CalibDB::readTempCalibParam(ULong_t timestamp, const std:
   if (!mInit) {
     init();
   }
-  TempCalibrationParams* result = mCCDBManager.retrieveFromTFileAny(getCDBPathTemperatureCalibrationParams(), metadata, timestamp);
+  auto& mgr = CcdbManager::instance();
+  TempCalibrationParams* result = mgr.getForTimeStamp(getCDBPathTemperatureCalibrationParams(), timestamp);
   if (!result) {
     throw ObjectNotFoundException(mCCDBServer, getCDBPathTemperatureCalibrationParams(), metadata, timestamp);
   }
@@ -149,7 +165,8 @@ TempCalibParamSM* CalibDB::readTempCalibParamSM(ULong_t timestamp, const std::ma
   if (!mInit) {
     init();
   }
-  TempCalibParamSM* result = mCCDBManager.retrieveFromTFileAny(getCDBPathTemperatureCalibrationParamsSM(), metadata, timestamp);
+  auto& mgr = CcdbManager::instance();
+  TempCalibParamSM* result = mgr.getForTimeStamp(getCDBPathTemperatureCalibrationParamsSM(), timestamp);
   if (!result) {
     throw ObjectNotFoundException(mCCDBServer, getCDBPathTemperatureCalibrationParamsSM(), metadata, timestamp);
   }
@@ -161,19 +178,34 @@ GainCalibrationFactors* CalibDB::readGainCalibFactors(ULong_t timestamp, const s
   if (!mInit) {
     init();
   }
-  GainCalibrationFactors* result = mCCDBManager.retrieveFromTFileAny(getCDBPathGainCalibrationParams(), metadata, timestamp);
+  auto& mgr = CcdbManager::instance();
+  GainCalibrationFactors* result = mgr.getForTimeStamp(getCDBPathGainCalibrationParams(), timestamp);
   if (!result) {
     throw ObjectNotFoundException(mCCDBServer, getCDBPathGainCalibrationParams(), metadata, timestamp);
   }
   return result;
 }
 
+EMCALChannelScaleFactors* CalibDB::readChannelScaleFactors(ULong_t timestamp, const std::map& metadata)
+{
+  if (!mInit) {
+    init();
+  }
+  auto& mgr = CcdbManager::instance();
+  EMCALChannelScaleFactors* result = mgr.getForTimeStamp(getCDBPathChannelScaleFactors(), timestamp);
+  if (!result) {
+    throw ObjectNotFoundException(mCCDBServer, getCDBPathChannelScaleFactors(), metadata, timestamp);
+  }
+  return result;
+}
+
 FeeDCS* CalibDB::readFeeDCSData(ULong_t timestamp, const std::map& metadata)
 {
   if (!mInit) {
     init();
   }
-  FeeDCS* result = mCCDBManager.retrieveFromTFileAny(getCDBPathFeeDCS(), metadata, timestamp);
+  auto& mgr = CcdbManager::instance();
+  FeeDCS* result = mgr.getForTimeStamp(getCDBPathFeeDCS(), timestamp);
   if (!result) {
     throw ObjectNotFoundException(mCCDBServer, getCDBPathFeeDCS(), metadata, timestamp);
   }
@@ -185,9 +217,23 @@ ElmbData* CalibDB::readTemperatureSensorData(ULong_t timestamp, const std::map(getCDBPathTemperatureSensor(), metadata, timestamp);
+  auto& mgr = CcdbManager::instance();
+  ElmbData* result = mgr.getForTimeStamp(getCDBPathTemperatureSensor(), timestamp);
   if (!result) {
     throw ObjectNotFoundException(mCCDBServer, getCDBPathTemperatureSensor(), metadata, timestamp);
   }
   return result;
 }
+
+Pedestal* CalibDB::readPedestalData(ULong_t timestamp, const std::map& metadata)
+{
+  if (!mInit) {
+    init();
+  }
+  auto& mgr = CcdbManager::instance();
+  Pedestal* result = mgr.getForTimeStamp(getCDBPathChannelPedestals(), timestamp);
+  if (!result) {
+    throw ObjectNotFoundException(mCCDBServer, getCDBPathChannelPedestals(), metadata, timestamp);
+  }
+  return result;
+}
\ No newline at end of file
diff --git a/Detectors/EMCAL/calib/src/CellRecalibrator.cxx b/Detectors/EMCAL/calib/src/CellRecalibrator.cxx
new file mode 100644
index 0000000000000..e5529d59ae2dd
--- /dev/null
+++ b/Detectors/EMCAL/calib/src/CellRecalibrator.cxx
@@ -0,0 +1,30 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include 
+#include 
+
+using namespace o2::emcal;
+
+void CellRecalibrator::printStream(std::ostream& stream) const
+{
+  stream << "Calibrator settings: \n"
+         << "==========================================================="
+         << "  Bad channel map:              " << (hasBadChannelMap() ? "yes" : "no") << "\n"
+         << "  Time calibration params:      " << (hasTimeCalibration() ? "yes" : "no") << "\n"
+         << "  Energy calibration factors:   " << (hasGainCalibration() ? "yes" : "no") << "\n";
+}
+
+std::ostream& o2::emcal::operator<<(std::ostream& in, const o2::emcal::CellRecalibrator& calibrator)
+{
+  calibrator.printStream(in);
+  return in;
+}
\ No newline at end of file
diff --git a/Detectors/EMCAL/calib/src/EMCALCalibLinkDef.h b/Detectors/EMCAL/calib/src/EMCALCalibLinkDef.h
index 8fae66b9558e7..7d2328a6f80aa 100644
--- a/Detectors/EMCAL/calib/src/EMCALCalibLinkDef.h
+++ b/Detectors/EMCAL/calib/src/EMCALCalibLinkDef.h
@@ -18,10 +18,12 @@
 #pragma link C++ class o2::emcal::CalibDB + ;
 #pragma link C++ class o2::emcal::BadChannelMap + ;
 #pragma link C++ class o2::emcal::TimeCalibrationParams + ;
+#pragma link C++ class o2::emcal::TimeCalibrationSlewingParams + ;
 #pragma link C++ class o2::emcal::TimeCalibParamL1Phase + ;
 #pragma link C++ class o2::emcal::TempCalibrationParams + ;
 #pragma link C++ class o2::emcal::TempCalibParamSM + ;
 #pragma link C++ class o2::emcal::GainCalibrationFactors + ;
+#pragma link C++ class o2::emcal::Pedestal + ;
 #pragma link C++ class o2::emcal::TriggerTRUDCS + ;
 #pragma link C++ class o2::emcal::TriggerSTUDCS + ;
 #pragma link C++ class o2::emcal::TriggerSTUErrorCounter + ;
diff --git a/Detectors/EMCAL/calib/src/EMCALChannelScaleFactors.cxx b/Detectors/EMCAL/calib/src/EMCALChannelScaleFactors.cxx
index c47215b692078..2771f31370119 100644
--- a/Detectors/EMCAL/calib/src/EMCALChannelScaleFactors.cxx
+++ b/Detectors/EMCAL/calib/src/EMCALChannelScaleFactors.cxx
@@ -9,6 +9,7 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 
+#include "EMCALCalib/CalibContainerErrors.h"
 #include "EMCALCalib/EMCALChannelScaleFactors.h"
 
 namespace o2
@@ -16,19 +17,19 @@ namespace o2
 namespace emcal
 {
 
-void EMCALChannelScaleFactors::insertVal(const int cellID, float E_min, float E_max, float scale)
+void EMCALChannelScaleFactors::insertVal(unsigned int cellID, float E_min, float E_max, float scale)
 {
   if (cellID >= NCells || cellID < 0) {
-    throw InvalidCellIDException(cellID);
+    throw CalibContainerIndexException(cellID);
   } else {
     ScaleFactors.at(cellID)[EnergyIntervals(E_min, E_max)] = scale;
   }
 }
 
-float EMCALChannelScaleFactors::getScaleVal(const int cellID, float E) const
+float EMCALChannelScaleFactors::getScaleVal(unsigned int cellID, float E) const
 {
   if (cellID >= NCells || cellID < 0) {
-    throw InvalidCellIDException(cellID);
+    throw CalibContainerIndexException(cellID);
   } else {
     for (const auto& [energy, scale] : ScaleFactors[cellID]) {
       if (energy.isInInterval(E)) {
diff --git a/Detectors/EMCAL/calib/src/GainCalibrationFactors.cxx b/Detectors/EMCAL/calib/src/GainCalibrationFactors.cxx
index 7f5b332baef99..9dbe11a92cd05 100644
--- a/Detectors/EMCAL/calib/src/GainCalibrationFactors.cxx
+++ b/Detectors/EMCAL/calib/src/GainCalibrationFactors.cxx
@@ -9,9 +9,10 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 
+#include "EMCALCalib/CalibContainerErrors.h"
 #include "EMCALCalib/GainCalibrationFactors.h"
 
-#include "FairLogger.h"
+#include 
 
 #include 
 
@@ -21,11 +22,17 @@ using namespace o2::emcal;
 
 void GainCalibrationFactors::addGainCalibFactor(unsigned short iCell, float gainFactor)
 {
+  if (iCell >= mGainCalibFactors.size()) {
+    throw CalibContainerIndexException(iCell);
+  }
   mGainCalibFactors[iCell] = gainFactor;
 }
 
 float GainCalibrationFactors::getGainCalibFactors(unsigned short iCell) const
 {
+  if (iCell >= mGainCalibFactors.size()) {
+    throw CalibContainerIndexException(iCell);
+  }
   return mGainCalibFactors[iCell];
 }
 
diff --git a/Detectors/EMCAL/calib/src/Pedestal.cxx b/Detectors/EMCAL/calib/src/Pedestal.cxx
new file mode 100644
index 0000000000000..7d67205fc7943
--- /dev/null
+++ b/Detectors/EMCAL/calib/src/Pedestal.cxx
@@ -0,0 +1,153 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "EMCALBase/Geometry.h"
+#include "EMCALCalib/CalibContainerErrors.h"
+#include "EMCALCalib/Pedestal.h"
+
+#include 
+
+#include 
+#include 
+
+#include 
+
+#include 
+
+using namespace o2::emcal;
+
+void Pedestal::addPedestalValue(unsigned short cellID, short pedestal, bool isLowGain, bool isLEDMON)
+{
+  if (cellID >= mPedestalValuesHG.size()) {
+    throw CalibContainerIndexException(cellID);
+  }
+  if (isLEDMON) {
+    if (isLowGain) {
+      mPedestalValuesLEDMONLG[cellID] = pedestal;
+    } else {
+      mPedestalValuesLEDMONHG[cellID] = pedestal;
+    }
+  } else {
+    if (isLowGain) {
+      mPedestalValuesLG[cellID] = pedestal;
+    } else {
+      mPedestalValuesHG[cellID] = pedestal;
+    }
+  }
+}
+
+short Pedestal::getPedestalValue(unsigned short cellID, bool isLowGain, bool isLEDMON) const
+{
+  if (cellID >= mPedestalValuesHG.size()) {
+    throw CalibContainerIndexException(cellID);
+  }
+  if (isLEDMON) {
+    if (isLowGain) {
+      return mPedestalValuesLEDMONLG[cellID];
+    } else {
+      return mPedestalValuesLEDMONHG[cellID];
+    }
+  } else {
+    if (isLowGain) {
+      return mPedestalValuesLG[cellID];
+    } else {
+      return mPedestalValuesHG[cellID];
+    }
+  }
+}
+
+TH1* Pedestal::getHistogramRepresentation(bool isLowGain, bool isLEDMON) const
+{
+  gsl::span data;
+  std::string histname, histtitle;
+  if (isLEDMON) {
+    if (isLowGain) {
+      histname = "PedestalLG_LEDMON";
+      histtitle = "LEDMON Pedestal values Params low Gain";
+      data = gsl::span(mPedestalValuesLEDMONLG);
+    } else {
+      histname = "PedestalHG_LEDMON";
+      histtitle = "LEDMON Pedestal values Params high Gain";
+      data = gsl::span(mPedestalValuesLEDMONHG);
+    }
+  } else {
+    if (isLowGain) {
+      histname = "PedestalLG";
+      histtitle = "Pedestal values Params low Gain";
+      data = gsl::span(mPedestalValuesLG);
+    } else {
+      histname = "PedestalHG";
+      histtitle = "Pedestal values Params high Gain";
+      data = gsl::span(mPedestalValuesHG);
+    }
+  }
+
+  auto hist = new TH1S(histname.data(), histtitle.data(), data.size(), -0.5, data.size() - 0.5);
+  hist->SetDirectory(nullptr);
+  for (std::size_t icell{0}; icell < data.size(); ++icell) {
+    hist->SetBinContent(icell + 1, data[icell]);
+  }
+  return hist;
+}
+
+TH2* Pedestal::getHistogramRepresentation2D(bool isLowGain, bool isLEDMON) const
+{
+  gsl::span data;
+  std::string histname, histtitle;
+  if (isLEDMON) {
+    if (isLowGain) {
+      histname = "PedestalLG_LEDMON";
+      histtitle = "LEDMON Pedestal values Params low Gain";
+      data = gsl::span(mPedestalValuesLEDMONLG);
+    } else {
+      histname = "PedestalHG_LEDMON";
+      histtitle = "LEDMON Pedestal values Params high Gain";
+      data = gsl::span(mPedestalValuesLEDMONHG);
+    }
+  } else {
+    if (isLowGain) {
+      histname = "PedestalLG";
+      histtitle = "Pedestal values Params low Gain";
+      data = gsl::span(mPedestalValuesLG);
+    } else {
+      histname = "PedestalHG";
+      histtitle = "Pedestal values Params high Gain";
+      data = gsl::span(mPedestalValuesHG);
+    }
+  }
+
+  const int MAXROWS = isLEDMON ? 10 : 208,
+            MAXCOLS = isLEDMON ? 48 : 96;
+
+  auto hist = new TH2S(histname.data(), histtitle.data(), MAXCOLS, -0.5, double(MAXCOLS) - 0.5, MAXROWS, -0.5, double(MAXROWS) - 0.5);
+  hist->SetDirectory(nullptr);
+  try {
+    auto geo = Geometry::GetInstance();
+    for (size_t ichan = 0; ichan < data.size(); ichan++) {
+      if (isLEDMON) {
+        int col = ichan % 48,
+            row = ichan / 48;
+        hist->Fill(col, row, data[ichan]);
+      } else {
+        auto position = geo->GlobalRowColFromIndex(ichan);
+        hist->Fill(std::get<1>(position), std::get<0>(position), data[ichan]);
+      }
+    }
+  } catch (o2::emcal::GeometryNotInitializedException& e) {
+    LOG(error) << "Geometry needs to be initialized";
+  }
+  return hist;
+}
+
+bool Pedestal::operator==(const Pedestal& other) const
+{
+  return mPedestalValuesHG == other.mPedestalValuesHG && mPedestalValuesLG == other.mPedestalValuesLG && mPedestalValuesLEDMONHG == other.mPedestalValuesLEDMONHG && mPedestalValuesLEDMONLG == other.mPedestalValuesLEDMONLG;
+}
diff --git a/Detectors/EMCAL/calib/src/TempCalibParamSM.cxx b/Detectors/EMCAL/calib/src/TempCalibParamSM.cxx
index 79d2dc551c24d..f52b8cc396e23 100644
--- a/Detectors/EMCAL/calib/src/TempCalibParamSM.cxx
+++ b/Detectors/EMCAL/calib/src/TempCalibParamSM.cxx
@@ -11,7 +11,7 @@
 
 #include "EMCALCalib/TempCalibParamSM.h"
 
-#include "FairLogger.h"
+#include 
 
 #include 
 
diff --git a/Detectors/EMCAL/calib/src/TempCalibrationParams.cxx b/Detectors/EMCAL/calib/src/TempCalibrationParams.cxx
index 7234395cc5ba6..752c9fe34aab9 100644
--- a/Detectors/EMCAL/calib/src/TempCalibrationParams.cxx
+++ b/Detectors/EMCAL/calib/src/TempCalibrationParams.cxx
@@ -11,7 +11,7 @@
 
 #include "EMCALCalib/TempCalibrationParams.h"
 
-#include "FairLogger.h"
+#include 
 
 #include 
 
diff --git a/Detectors/EMCAL/calib/src/TimeCalibParamL1Phase.cxx b/Detectors/EMCAL/calib/src/TimeCalibParamL1Phase.cxx
index 9f1b27b1befb2..eca2017569b93 100644
--- a/Detectors/EMCAL/calib/src/TimeCalibParamL1Phase.cxx
+++ b/Detectors/EMCAL/calib/src/TimeCalibParamL1Phase.cxx
@@ -11,7 +11,7 @@
 
 #include "EMCALCalib/TimeCalibParamL1Phase.h"
 
-#include "FairLogger.h"
+#include 
 
 #include 
 
diff --git a/Detectors/EMCAL/calib/src/TimeCalibrationParams.cxx b/Detectors/EMCAL/calib/src/TimeCalibrationParams.cxx
index 0653706426da7..e8ef4d9655c61 100644
--- a/Detectors/EMCAL/calib/src/TimeCalibrationParams.cxx
+++ b/Detectors/EMCAL/calib/src/TimeCalibrationParams.cxx
@@ -9,9 +9,10 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 
+#include "EMCALCalib/CalibContainerErrors.h"
 #include "EMCALCalib/TimeCalibrationParams.h"
 
-#include "FairLogger.h"
+#include 
 
 #include 
 
@@ -21,6 +22,9 @@ using namespace o2::emcal;
 
 void TimeCalibrationParams::addTimeCalibParam(unsigned short cellID, short time, bool isLowGain)
 {
+  if (cellID >= mTimeCalibParamsHG.size()) {
+    throw CalibContainerIndexException(cellID);
+  }
   if (!isLowGain) {
     mTimeCalibParamsHG[cellID] = time;
   } else {
@@ -30,6 +34,9 @@ void TimeCalibrationParams::addTimeCalibParam(unsigned short cellID, short time,
 
 short TimeCalibrationParams::getTimeCalibParam(unsigned short cellID, bool isLowGain) const
 {
+  if (cellID >= mTimeCalibParamsHG.size()) {
+    throw CalibContainerIndexException(cellID);
+  }
   if (isLowGain) {
     return mTimeCalibParamsLG[cellID];
   } else {
diff --git a/Detectors/EMCAL/calib/src/TimeCalibrationSlewingParams.cxx b/Detectors/EMCAL/calib/src/TimeCalibrationSlewingParams.cxx
new file mode 100644
index 0000000000000..5598f63863bf5
--- /dev/null
+++ b/Detectors/EMCAL/calib/src/TimeCalibrationSlewingParams.cxx
@@ -0,0 +1,59 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "EMCALCalib/TimeCalibrationSlewingParams.h"
+
+// using namespace o2::emcal;
+namespace o2
+{
+
+namespace emcal
+{
+
+bool TimeCalibrationSlewingParams::operator==(const TimeCalibrationSlewingParams& other) const
+{
+  for (int i = 0; i < 4; ++i) {
+    if (std::abs(arrParams[i] - other.getTimeSlewingParam(i)) > DBL_EPSILON) {
+      return false;
+    }
+  }
+  return true;
+}
+
+TimeCalibrationSlewingParams::TimeCalibrationSlewingParams(std::array arr)
+{
+  arrParams = arr;
+}
+
+void TimeCalibrationSlewingParams::addTimeSlewingParam(std::array arr)
+{
+  arrParams = arr;
+}
+
+double TimeCalibrationSlewingParams::getTimeSlewingParam(unsigned int index) const
+{
+  if (index >= 4) {
+    return -1;
+  }
+  return arrParams[index];
+}
+
+double TimeCalibrationSlewingParams::eval(double energy) const
+{
+  double val = 0;
+  for (unsigned int i = 0; i < arrParams.size(); ++i) {
+    val += arrParams[i] * std::pow(energy, i);
+  }
+  return val;
+}
+
+} // namespace emcal
+} // namespace o2
\ No newline at end of file
diff --git a/Detectors/EMCAL/calib/src/TriggerDCS.cxx b/Detectors/EMCAL/calib/src/TriggerDCS.cxx
index 3a2137f519bad..4cd8a243a1cc8 100644
--- a/Detectors/EMCAL/calib/src/TriggerDCS.cxx
+++ b/Detectors/EMCAL/calib/src/TriggerDCS.cxx
@@ -13,7 +13,7 @@
 #include "EMCALCalib/TriggerTRUDCS.h"
 #include "EMCALCalib/TriggerSTUDCS.h"
 
-#include "FairLogger.h"
+#include 
 
 #include 
 #include 
diff --git a/Detectors/EMCAL/calib/src/TriggerSTUDCS.cxx b/Detectors/EMCAL/calib/src/TriggerSTUDCS.cxx
index 548e71b6b7426..d3d769de4eaa5 100644
--- a/Detectors/EMCAL/calib/src/TriggerSTUDCS.cxx
+++ b/Detectors/EMCAL/calib/src/TriggerSTUDCS.cxx
@@ -11,7 +11,7 @@
 
 #include "EMCALCalib/TriggerSTUDCS.h"
 
-#include "FairLogger.h"
+#include 
 
 #include 
 #include 
diff --git a/Detectors/EMCAL/calib/src/TriggerSTUErrorCounter.cxx b/Detectors/EMCAL/calib/src/TriggerSTUErrorCounter.cxx
index dc9618573c4a6..7d5e61d5e68e3 100644
--- a/Detectors/EMCAL/calib/src/TriggerSTUErrorCounter.cxx
+++ b/Detectors/EMCAL/calib/src/TriggerSTUErrorCounter.cxx
@@ -11,7 +11,7 @@
 
 #include "EMCALCalib/TriggerSTUErrorCounter.h"
 
-#include "FairLogger.h"
+#include 
 
 #include 
 #include 
diff --git a/Detectors/EMCAL/calib/src/TriggerTRUDCS.cxx b/Detectors/EMCAL/calib/src/TriggerTRUDCS.cxx
index 169124de0e60f..22beb7d2ecf6a 100644
--- a/Detectors/EMCAL/calib/src/TriggerTRUDCS.cxx
+++ b/Detectors/EMCAL/calib/src/TriggerTRUDCS.cxx
@@ -11,7 +11,7 @@
 
 #include "EMCALCalib/TriggerTRUDCS.h"
 
-#include "FairLogger.h"
+#include 
 
 #include 
 #include 
diff --git a/Detectors/EMCAL/calib/test/testPedestal.cxx b/Detectors/EMCAL/calib/test/testPedestal.cxx
new file mode 100644
index 0000000000000..1adcfd8b9ea25
--- /dev/null
+++ b/Detectors/EMCAL/calib/test/testPedestal.cxx
@@ -0,0 +1,124 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#define BOOST_TEST_MODULE Test_EMCAL_Calib
+#define BOOST_TEST_MAIN
+#define BOOST_TEST_DYN_LINK
+
+#include 
+#include 
+#include 
+#include "EMCALCalib/CalibContainerErrors.h"
+#include "EMCALCalib/Pedestal.h"
+
+/// \brief Standard tests for Pedestal container
+
+namespace o2
+{
+namespace emcal
+{
+
+using pedestalarray = std::vector;
+
+pedestalarray createRandomPedestals(bool isLEDMON)
+{
+  std::random_device rd{};
+  std::mt19937 gen{rd()};
+  std::normal_distribution gaussrand{40., 5.};
+  pedestalarray pedestalcontainer(isLEDMON ? 480 : 17664);
+  for (std::size_t ichan{0}; ichan < pedestalcontainer.size(); ++ichan) {
+    pedestalcontainer[ichan] = std::round(gaussrand(gen));
+  }
+
+  return pedestalcontainer;
+}
+
+pedestalarray shiftPedestalValue(const pedestalarray& input, short shift = 1)
+{
+  pedestalarray shifted;
+  for (std::size_t ichan{0}; ichan < input.size(); ++ichan) {
+    shifted[ichan] = input[ichan] + shift;
+  }
+  return shifted;
+}
+
+BOOST_AUTO_TEST_CASE(testPedestal)
+{
+  auto pedestalsHG = createRandomPedestals(false),
+       pedestalsLG = createRandomPedestals(false),
+       pedestalsLEDMONHG = createRandomPedestals(true),
+       pedestalsLEDMONLG = createRandomPedestals(true);
+
+  o2::emcal::Pedestal pedestalObject;
+  for (std::size_t ichan{0}; ichan < pedestalsHG.size(); ++ichan) {
+    pedestalObject.addPedestalValue(ichan, pedestalsHG[ichan], false, false);
+    pedestalObject.addPedestalValue(ichan, pedestalsLG[ichan], true, false);
+  }
+  for (std::size_t ichan{0}; ichan < pedestalsLEDMONHG.size(); ++ichan) {
+    pedestalObject.addPedestalValue(ichan, pedestalsLEDMONHG[ichan], false, true);
+    pedestalObject.addPedestalValue(ichan, pedestalsLEDMONLG[ichan], true, true);
+  }
+
+  // test adding entries beyond range
+  for (std::size_t ichan{17665}; ichan < 18000; ++ichan) {
+    BOOST_CHECK_EXCEPTION(pedestalObject.addPedestalValue(ichan, 2, false, true), o2::emcal::CalibContainerIndexException, [ichan](const o2::emcal::CalibContainerIndexException& e) { return e.getIndex() == ichan; });
+    BOOST_CHECK_EXCEPTION(pedestalObject.addPedestalValue(ichan, 3, true, false), o2::emcal::CalibContainerIndexException, [ichan](const o2::emcal::CalibContainerIndexException& e) { return e.getIndex() == ichan; });
+  }
+
+  // test reading values in range
+  for (std::size_t ichan{0}; ichan < pedestalsHG.size(); ++ichan) {
+    BOOST_CHECK_EQUAL(pedestalObject.getPedestalValue(ichan, false, false), pedestalsHG[ichan]);
+    BOOST_CHECK_EQUAL(pedestalObject.getPedestalValue(ichan, true, false), pedestalsLG[ichan]);
+  }
+  for (std::size_t ichan{0}; ichan < pedestalsLEDMONHG.size(); ++ichan) {
+    BOOST_CHECK_EQUAL(pedestalObject.getPedestalValue(ichan, false, true), pedestalsLEDMONHG[ichan]);
+    BOOST_CHECK_EQUAL(pedestalObject.getPedestalValue(ichan, true, true), pedestalsLEDMONLG[ichan]);
+  }
+
+  // test reading entries beyond range
+  for (std::size_t ichan{17665}; ichan < 18000; ++ichan) {
+    BOOST_CHECK_EXCEPTION(pedestalObject.getPedestalValue(ichan, false, false), o2::emcal::CalibContainerIndexException, [ichan](const o2::emcal::CalibContainerIndexException& e) { return e.getIndex() == ichan; });
+    BOOST_CHECK_EXCEPTION(pedestalObject.getPedestalValue(ichan, true, false), o2::emcal::CalibContainerIndexException, [ichan](const o2::emcal::CalibContainerIndexException& e) { return e.getIndex() == ichan; });
+    BOOST_CHECK_EXCEPTION(pedestalObject.getPedestalValue(ichan, false, true), o2::emcal::CalibContainerIndexException, [ichan](const o2::emcal::CalibContainerIndexException& e) { return e.getIndex() == ichan; });
+    BOOST_CHECK_EXCEPTION(pedestalObject.getPedestalValue(ichan, true, true), o2::emcal::CalibContainerIndexException, [ichan](const o2::emcal::CalibContainerIndexException& e) { return e.getIndex() == ichan; });
+  }
+
+  // tests for operator==
+  // shift pedestal by 1 for false test
+  // Test all cases:
+  // - same object
+  // - same HG, different LG
+  // - same LG, different HG
+  // - both HG and LG different
+  /*
+  auto shiftedPedestalsHG = shiftPedestalValue(pedestalsHG, 1),
+       shiftedPedestalsLG = shiftPedestalValue(pedestalsLG, 1);
+  o2::emcal::Pedestal same, differLow, differHigh, differBoth;
+  for (std::size_t ichan{0}; ichan < pedestalsHG.size(); ++ichan) {
+    same.addPedestalValue(ichan, pedestalsHG[ichan], false);
+    same.addPedestalValue(ichan, pedestalsLG[ichan], true);
+    differLow.addPedestalValue(ichan, pedestalsHG[ichan], false);
+    differLow.addPedestalValue(ichan, shiftedPedestalsLG[ichan], true);
+    differHigh.addPedestalValue(ichan, shiftedPedestalsHG[ichan], false);
+    differHigh.addPedestalValue(ichan, pedestalsLG[ichan], true);
+    differBoth.addPedestalValue(ichan, shiftedPedestalsHG[ichan], false);
+    differBoth.addPedestalValue(ichan, shiftedPedestalsLG[ichan], true);
+  }
+  BOOST_CHECK_EQUAL(pedestalObject, same);
+  BOOST_CHECK_NE(pedestalObject, differLow);
+  BOOST_CHECK_NE(pedestalObject, differHigh);
+  BOOST_CHECK_NE(pedestalObject, differBoth);
+  */
+}
+
+} // namespace emcal
+
+} // namespace o2
\ No newline at end of file
diff --git a/Detectors/EMCAL/calibration/CMakeLists.txt b/Detectors/EMCAL/calibration/CMakeLists.txt
index 28a6d4e11029b..7fec9fcef0f93 100644
--- a/Detectors/EMCAL/calibration/CMakeLists.txt
+++ b/Detectors/EMCAL/calibration/CMakeLists.txt
@@ -16,11 +16,18 @@ o2_add_library(EMCALCalibration
                         src/EMCALCalibExtractor.cxx
                         src/EMCALCalibParams.cxx
                         src/EMCDCSProcessor.cxx
+                        src/EMCALPedestalHelper.cxx
+                        src/PedestalCalibDevice.cxx
+                        src/PedestalProcessorDevice.cxx
+                        src/PedestalProcessorData.cxx
+                        src/EMCALTempCalibExtractor.cxx
                PUBLIC_LINK_LIBRARIES O2::CCDB O2::EMCALBase
                                      O2::EMCALCalib
+                                     O2::EMCALReconstruction
                                      O2::CommonUtils
                                      O2::DetectorsCalibration
                                      O2::DetectorsDCS
+                                     O2::DetectorsRaw
                                      O2::DataFormatsEMCAL
                                      O2::Framework
                                      O2::Algorithm
@@ -38,6 +45,9 @@ o2_target_root_dictionary(EMCALCalibration
                                   include/EMCALCalibration/EMCALTimeCalibData.h
                                   include/EMCALCalibration/EMCALCalibParams.h
                                   include/EMCALCalibration/EMCDCSProcessor.h
+                                  include/EMCALCalibration/EMCALPedestalHelper.h
+                                  include/EMCALCalibration/PedestalProcessorData.h
+                                  include/EMCALCalibration/EMCALTempCalibExtractor.h
                           LINKDEF src/EMCALCalibrationLinkDef.h)
 
 o2_add_executable(emcal-channel-calib-workflow
@@ -52,6 +62,22 @@ if (OpenMP_CXX_FOUND)
     target_link_libraries(${targetName} PRIVATE OpenMP::OpenMP_CXX)
 endif()
 
+o2_add_executable(emcal-pedestal-processor-workflow
+                  COMPONENT_NAME calibration
+                  SOURCES testWorkflow/emc-pedestal-processor-workflow.cxx
+                  PUBLIC_LINK_LIBRARIES O2::Framework
+                                        O2::EMCALCalibration
+                                        O2::DetectorsRaw
+                                        O2::DetectorsCalibration)
+
+o2_add_executable(emcal-pedestal-calib-workflow
+                  COMPONENT_NAME calibration
+                  SOURCES testWorkflow/emc-pedestal-calib-workflow.cxx
+                  PUBLIC_LINK_LIBRARIES O2::Framework
+                                        O2::EMCALCalibration
+                                        O2::DetectorsRaw
+                                        O2::DetectorsCalibration)
+
 o2_add_executable(run-calib-offline
                   COMPONENT_NAME emcal
                   TARGETVARNAME targetName
@@ -64,6 +90,13 @@ if (OpenMP_CXX_FOUND)
     target_link_libraries(${targetName} PRIVATE OpenMP::OpenMP_CXX)
 endif()
 
+o2_add_test(PedestalProcessorData
+        SOURCES test/testPedestalProcessorData.cxx
+        PUBLIC_LINK_LIBRARIES O2::EMCALCalibration
+        COMPONENT_NAME emcal
+        LABELS emcal
+        ENVIRONMENT O2_ROOT=${CMAKE_BINARY_DIR}/stage)
+
 o2_add_test_root_macro(macros/makeEMCALCCDBEntryForDCS.C
                                 PUBLIC_LINK_LIBRARIES O2::DetectorsDCS O2::CCDB
                                 LABELS emcal COMPILE_ONLY)
diff --git a/Detectors/EMCAL/calibration/include/EMCALCalibration/EMCALCalibExtractor.h b/Detectors/EMCAL/calibration/include/EMCALCalibration/EMCALCalibExtractor.h
index 9dd6ae436b6c2..035dc92ecfe09 100644
--- a/Detectors/EMCAL/calibration/include/EMCALCalibration/EMCALCalibExtractor.h
+++ b/Detectors/EMCAL/calibration/include/EMCALCalibration/EMCALCalibExtractor.h
@@ -19,15 +19,22 @@
 #define EMCALCALIBEXTRACTOR_H_
 
 #include 
+#include 
 #include 
 #include "CCDB/BasicCCDBManager.h"
 #include "EMCALCalib/BadChannelMap.h"
+#include "EMCALCalib/EMCALChannelScaleFactors.h"
 #include "EMCALCalib/TimeCalibrationParams.h"
+#include "EMCALCalibration/EMCALCalibParams.h"
 #include "CommonUtils/BoostHistogramUtils.h"
 #include "EMCALBase/Geometry.h"
+#include "EMCALCalibration/EMCALCalibParams.h"
+#include "EMCALCalib/Pedestal.h"
+#include "EMCALCalibration/PedestalProcessorData.h"
 #include 
 
 #include 
+#include 
 
 #if (defined(WITH_OPENMP) && !defined(__CLING__))
 #include 
@@ -37,28 +44,38 @@ namespace o2
 {
 namespace emcal
 {
-
+using boostHisto = boost::histogram::histogram, boost::histogram::axis::integer<>>, boost::histogram::unlimited_storage>>;
 class EMCALCalibExtractor
 {
-  using boostHisto = boost::histogram::histogram, boost::histogram::axis::integer<>>, boost::histogram::unlimited_storage>>;
   using slice_t = int;
   using cell_t = int;
 
   /// \brief Stuct for the maps needed for the bad channel calibration
   struct BadChannelCalibInfo {
-    std::map> energyPerHitMap;   // energy/per hit per cell per slice
-    std::map> goodCellWindowMap; // for each slice, the emin and the emax of the good cell window
+    std::map> energyPerHitMap;        // energy/per hit per cell per slice
+    std::map> nHitsMap;               // number of hits per cell per slice
+    std::map> goodCellWindowMap;      // for each slice, the emin and the emax of the good cell window
+    std::map> goodCellWindowNHitsMap; // for each slice, the nHitsMin and the mHitsMax of the good cell window
+  };
+
+  struct BadChannelCalibTimeInfo {
+    std::array sigmaCell;         // sigma value of time distribution for single cells
+    double goodCellWindow;                       // cut value for good cells
+    std::array fracHitsPreTrigg;  // fraction of hits before the main time peak (pre-trigger pile-up)
+    double goodCellWindowFracHitsPreTrigg;       // cut value for good cells for pre-trigger pile-up
+    std::array fracHitsPostTrigg; // fraction of hits after the main time peak (post-trigger pile-up)
+    double goodCellWindowFracHitsPostTrigg;      // cut value for good cells for post-trigger pile-up
   };
 
  public:
   EMCALCalibExtractor()
   {
     LOG(info) << "initialized EMCALCalibExtractor";
-    if (!mGeometry) {
+    try {
+      // Try to access geometry initialized ountside
       mGeometry = o2::emcal::Geometry::GetInstance();
-      if (!mGeometry) {
-        mGeometry = o2::emcal::Geometry::GetInstanceFromRunNumber(300000); // fallback option
-      }
+    } catch (o2::emcal::GeometryNotInitializedException& e) {
+      mGeometry = o2::emcal::Geometry::GetInstanceFromRunNumber(300000); // fallback option
     }
     // mNcells = mGeometry->GetNCells();
   };
@@ -70,7 +87,7 @@ class EMCALCalibExtractor
   void setNThreads(int n) { mNThreads = std::min(n, mNcells); }
   int getNThreads() const { return mNThreads; }
 
-  void setUseScaledHistoForBadChannels(bool useScaledHistoForBadChannels) { mUseScaledHistoForBadChannels = useScaledHistoForBadChannels; }
+  void setBCMScaleFactors(EMCALChannelScaleFactors* scalefactors) { mBCMScaleFactors = scalefactors; }
 
   /// \brief Scaled hits per cell
   /// \param emin -- min. energy for cell amplitudes
@@ -78,14 +95,42 @@ class EMCALCalibExtractor
   boostHisto buildHitAndEnergyMeanScaled(double emin, double emax, boostHisto mCellAmplitude);
 
   /// \brief Function to perform the calibration of bad channels
+  /// \param hist histogram cell energy vs. cell ID. Main histogram for the bad channel calibration
+  /// \param histTime histogram cell time vs. cell ID. If default argument is taken, no calibration based on the timing signal will be performed
   template 
-  o2::emcal::BadChannelMap calibrateBadChannels(boost::histogram::histogram& hist)
+  o2::emcal::BadChannelMap calibrateBadChannels(const boost::histogram::histogram& hist, const boost::histogram::histogram& histTime = boost::histogram::make_histogram(boost::histogram::axis::variable<>{0., 1.}, boost::histogram::axis::variable<>{0., 1.}))
   {
     double time1 = std::chrono::duration_cast(std::chrono::high_resolution_clock::now().time_since_epoch()).count();
-    std::map> slices = {{0, {0.1, 0.3}}, {1, {0.3, 0.5}}, {2, {0.5, 1.0}}, {3, {1.0, 4.0}}};
+    std::map> slices = {{0, {0.1, 0.3}}, {1, {0.3, 0.5}}, {2, {0.5, 1.0}}, {3, {1.0, 4.0}}, {4, {4.0, 39.0}}};
+
+    std::fill(std::begin(mBadCellFracSM), std::end(mBadCellFracSM), 0); // reset all fractions to 0
+    for (unsigned int i = 0; i < mBadCellFracFEC.size(); ++i) {
+      std::fill(std::begin(mBadCellFracFEC[i]), std::end(mBadCellFracFEC[i]), 0);
+    }
+
+    auto histScaled = hist;
+    if (mBCMScaleFactors) {
+      LOG(info) << "Rescaling BCM histo";
+      // rescale the histogram
+      for (int icell = 0; icell < 17644; icell++) {
+        for (int ebin = 0; ebin < hist.axis(0).size(); ebin++) {
+          double lowerE = hist.axis(0).bin(ebin).lower();
+          double upperE = hist.axis(0).bin(ebin).upper();
+          double midE = (lowerE + upperE) / 2.;
+          histScaled.at(ebin, icell) = hist.at(ebin, icell) / mBCMScaleFactors->getScaleVal(icell, midE);
+        }
+      }
+    }
 
     // get all ofthe calibration information that we need in a struct
-    BadChannelCalibInfo calibrationInformation = buildHitAndEnergyMean(slices, hist);
+    BadChannelCalibInfo calibrationInformation = buildHitAndEnergyMean(slices, histScaled);
+
+    // only initialize this if the histo is not the default one
+    const bool doIncludeTime = (histTime.axis(0).size() > 1 && EMCALCalibParams::Instance().useTimeInfoForCalib_bc) ? true : false;
+    BadChannelCalibTimeInfo calibrationTimeInfo;
+    if (doIncludeTime) {
+      calibrationTimeInfo = buildTimeMeanAndSigma(histTime);
+    }
 
     o2::emcal::BadChannelMap mOutputBCM;
     // now loop through the cells and determine the mask for a given cell
@@ -102,30 +147,87 @@ class EMCALCalibExtractor
 #endif
 
     for (int cellID = 0; cellID < mNcells; cellID++) {
+      LOG(debug) << "analysing cell " << cellID;
       if (calibrationInformation.energyPerHitMap[0][cellID] == 0) {
         LOG(debug) << "Cell " << cellID << " is dead.";
         mOutputBCM.addBadChannel(cellID, o2::emcal::BadChannelMap::MaskType_t::DEAD_CELL);
+        mBadCellFracSM[mGeometry->GetSuperModuleNumber(cellID)] += 1;
+        mBadCellFracFEC[mGeometry->GetSuperModuleNumber(cellID)][getFECNumberInSM(cellID)] += 1;
       } else {
         bool failed = false;
         for (auto& [sliceIndex, slice] : slices) {
           auto ranges = calibrationInformation.goodCellWindowMap[sliceIndex];
+          auto rangesNHits = calibrationInformation.goodCellWindowNHitsMap[sliceIndex];
           auto meanPerCell = calibrationInformation.energyPerHitMap[sliceIndex][cellID];
-          LOG(debug) << "Mean per cell is " << meanPerCell << " Good Cell Window: [ " << ranges.first << " , " << ranges.second << " ]";
-          if (meanPerCell < ranges.first || meanPerCell > ranges.second) {
-            LOG(debug) << "********* FAILED **********";
+          auto meanPerCellNHits = calibrationInformation.nHitsMap[sliceIndex][cellID];
+          LOG(debug) << "Energy Per Hit: Mean per cell is " << meanPerCell << " Good Cell Window: [ " << ranges.first << " , " << ranges.second << " ]";
+          LOG(debug) << "NHits: Mean per cell is " << meanPerCellNHits << " Good Cell Window: [ " << rangesNHits.first << " , " << rangesNHits.second << " ]";
+
+          // for the cut on the mean number of hits we require at least 2 hits on average
+          double meanNHits = 0.5 * (rangesNHits.first + rangesNHits.second);
+          if (meanNHits < EMCALCalibParams::Instance().minNHitsForNHitCut || (std::abs(ranges.first) < 0.001 && std::abs(ranges.second) < 0.001)) {
+            LOG(debug) << "On average, only " << meanNHits << " found in energy interval [" << slice.first << " - " << slice.second << "]. Will do untight cut on upper limit";
+            if (meanPerCellNHits > EMCALCalibParams::Instance().minNHitsForNHitCut * 10) {
+              LOG(debug) << "********* FAILED for number of hits **********";
+              failed = true;
+              break;
+            }
+            // Case were enough statistics is present
+          } else if (meanPerCellNHits < rangesNHits.first || meanPerCellNHits > rangesNHits.second) {
+            LOG(debug) << "********* FAILED for mean NHits **********";
+            failed = true;
+            break;
+          }
+
+          // for the cut on the mean energy per hit we require at least 100 hits, as otherwise the distribution is very instable
+          if (meanNHits > EMCALCalibParams::Instance().minNHitsForMeanEnergyCut && (meanPerCell < ranges.first || meanPerCell > ranges.second)) {
+            LOG(debug) << "********* FAILED for mean energy **********";
             failed = true;
             break;
           }
         }
+
+        // check if the cell is bad due to timing signal.
+        if (!failed && doIncludeTime) {
+          if (std::abs(calibrationTimeInfo.goodCellWindow) < 0.001) {
+            LOG(warning) << "Good cell window for time distribution is 0. Will skip the cut on time distribution";
+          } else {
+            LOG(debug) << " calibrationTimeInfo.goodCellWindow " << calibrationTimeInfo.goodCellWindow << " calibrationTimeInfo.sigmaCell[cellID] " << calibrationTimeInfo.sigmaCell[cellID];
+            if (calibrationTimeInfo.sigmaCell[cellID] > calibrationTimeInfo.goodCellWindow) {
+              LOG(debug) << "Cell " << cellID << " is flagged due to time distribution";
+              failed = true;
+            } else if (calibrationTimeInfo.fracHitsPreTrigg[cellID] > calibrationTimeInfo.goodCellWindowFracHitsPreTrigg) {
+              LOG(debug) << "Cell " << cellID << " is flagged due to time distribution (pre-trigger)";
+              failed = true;
+            } else if (calibrationTimeInfo.fracHitsPostTrigg[cellID] > calibrationTimeInfo.goodCellWindowFracHitsPostTrigg) {
+              LOG(debug) << "Cell " << cellID << " is flagged due to time distribution (post-trigger)";
+              failed = true;
+            }
+          }
+        }
+
         if (failed) {
           LOG(debug) << "Cell " << cellID << " is bad.";
           mOutputBCM.addBadChannel(cellID, o2::emcal::BadChannelMap::MaskType_t::BAD_CELL);
+          mBadCellFracSM[mGeometry->GetSuperModuleNumber(cellID)] += 1;
+          mBadCellFracFEC[mGeometry->GetSuperModuleNumber(cellID)][getFECNumberInSM(cellID)] += 1;
         } else {
           LOG(debug) << "Cell " << cellID << " is good.";
           mOutputBCM.addBadChannel(cellID, o2::emcal::BadChannelMap::MaskType_t::GOOD_CELL);
         }
       }
     }
+
+    // Check if the fraction of bad+dead cells in a SM is above a certain threshold
+    // If yes, mask the whole SM
+    if (EMCALCalibParams::Instance().fracMaskSMFully_bc < 1) {
+      checkMaskSM(mOutputBCM);
+    }
+    // Same as above for FECs
+    if (EMCALCalibParams::Instance().fracMaskFECFully_bc < 1) {
+      checkMaskFEC(mOutputBCM);
+    }
+
     double time2 = std::chrono::duration_cast(std::chrono::high_resolution_clock::now().time_since_epoch()).count();
     double diffTransfer = time2 - time1;
     LOG(info) << "Total time" << diffTransfer << " ns";
@@ -145,13 +247,14 @@ class EMCALCalibExtractor
     // create the output histo
     BadChannelCalibInfo outputInfo;
     std::map> outputMapEnergyPerHit;
+    std::map> outputMapNHits;
     // initialize the output maps with 0
     for (const auto& [sliceIndex, sliceLimits] : sliceMap) {
       std::array energyPerHit, nHits;
       std::fill(energyPerHit.begin(), energyPerHit.end(), 0.);
       std::fill(nHits.begin(), nHits.end(), 0.);
       outputMapEnergyPerHit[sliceIndex] = energyPerHit;
-      // outputMapNHits[sliceIndex] = nHits;
+      outputMapNHits[sliceIndex] = nHits;
     }
 #if (defined(WITH_OPENMP) && !defined(__CLING__))
     if (mNThreads < 1) {
@@ -180,9 +283,9 @@ class EMCALCalibExtractor
         double sumVal = boost::histogram::algorithm::sum(slicedHist);
         if (sumVal > 0.) {
           // fill the output map with the desired slicing etc.
-          outputMapEnergyPerHit[sliceIndex][cellID] = (meanVal / (sumVal));
+          outputMapEnergyPerHit[sliceIndex][cellID] = meanVal;
+          outputMapNHits[sliceIndex][cellID] = sumVal;
         }
-
       } // end loop over the slices
     }   // end loop over the cells
     for (const auto& [sliceIndex, slice] : sliceMap) {
@@ -191,19 +294,98 @@ class EMCALCalibExtractor
       TRobustEstimator robustEstimator;
       auto& means = outputMapEnergyPerHit[sliceIndex];
       robustEstimator.EvaluateUni(means.size(), means.data(), meanPerSlice, sigmaPerSlice, 0);
+      if (std::abs(meanPerSlice) < 0.001 && std::abs(sigmaPerSlice) < 0.001) {
+        robustEstimator.EvaluateUni(means.size(), means.data(), meanPerSlice, sigmaPerSlice, means.size() * 0.95);
+      }
+
+      Double_t meanPerSlice_NHits = 0.0;  // mean energy per slice to be compared to the cell
+      Double_t sigmaPerSlice_NHits = 0.0; // sigma energy per slice to be compared to the cell
+      TRobustEstimator robustEstimatorNHits;
+      auto& meansNHits = outputMapNHits[sliceIndex];
+      robustEstimatorNHits.EvaluateUni(meansNHits.size(), meansNHits.data(), meanPerSlice_NHits, sigmaPerSlice_NHits, 0);
+      if (std::abs(meanPerSlice_NHits) < 0.001 && std::abs(sigmaPerSlice_NHits) < 0.001) {
+        robustEstimator.EvaluateUni(meansNHits.size(), meansNHits.data(), meanPerSlice_NHits, sigmaPerSlice_NHits, meansNHits.size() * 0.95);
+      }
 
-      LOG(debug) << "Mean per slice is: " << meanPerSlice << " Sigma Per Slice: " << sigmaPerSlice << " with size " << outputMapEnergyPerHit[sliceIndex].size();
+      LOG(debug) << "Energy Per hit: Mean per slice is: " << meanPerSlice << " Sigma Per Slice: " << sigmaPerSlice << " with size " << outputMapEnergyPerHit[sliceIndex].size();
+      LOG(debug) << "NHits: Mean per slice is: " << meanPerSlice_NHits << " Sigma Per Slice: " << sigmaPerSlice_NHits << " with size " << outputMapNHits[sliceIndex].size();
       // calculate the "good cell window from the mean"
-      double maxVal = meanPerSlice + 4.0 * sigmaPerSlice;
-      double minVal = meanPerSlice - 4.0 * sigmaPerSlice;
-      // we need to change this
+      double maxVal = meanPerSlice + mSigma * sigmaPerSlice;
+      double minVal = meanPerSlice - mSigma * sigmaPerSlice;
+      double maxValNHits = meanPerSlice_NHits + mSigma * sigmaPerSlice_NHits;
+      double minValNHits = meanPerSlice_NHits - mSigma * sigmaPerSlice_NHits;
+      // store in the output maps
       outputInfo.goodCellWindowMap[sliceIndex] = {minVal, maxVal};
+      outputInfo.goodCellWindowNHitsMap[sliceIndex] = {minValNHits, maxValNHits};
     }
     // now add these to the calib info struct
     outputInfo.energyPerHitMap = outputMapEnergyPerHit;
+    outputInfo.nHitsMap = outputMapNHits;
 
     return outputInfo;
   }
+
+  //____________________________________________
+  /// \brief calculate the sigma of the time distribution for all cells and caluclate the mean of the sigmas
+  /// \param histCellTime input histogram cellID vs cell time
+  /// \return sigma value for all cells and the upper cut value
+  template 
+  BadChannelCalibTimeInfo buildTimeMeanAndSigma(const boost::histogram::histogram& histCellTime)
+  {
+    BadChannelCalibTimeInfo timeInfo;
+    for (int i = 0; i < mNcells; ++i) {
+      // calculate sigma per cell
+      const int indexLow = histCellTime.axis(1).index(i);
+      const int indexHigh = histCellTime.axis(1).index(i + 1);
+      auto boostHistCellSlice = o2::utils::ProjectBoostHistoXFast(histCellTime, indexLow, indexHigh);
+
+      int maxElementIndex = std::max_element(boostHistCellSlice.begin(), boostHistCellSlice.end()) - boostHistCellSlice.begin() - 1;
+      if (maxElementIndex < 0) {
+        maxElementIndex = 0;
+      }
+      float maxElementCenter = 0.5 * (boostHistCellSlice.axis(0).bin(maxElementIndex).upper() + boostHistCellSlice.axis(0).bin(maxElementIndex).lower());
+      timeInfo.sigmaCell[i] = std::sqrt(o2::utils::getVarianceBoost1D(boostHistCellSlice, -999999, maxElementCenter - 50, maxElementCenter + 50));
+
+      // get number of hits within mean+-25ns (trigger bunch), from -500ns to -25ns before trigger bunch (pre-trigger), and for 25ns to 500ns (post-trigger)
+      double sumTrigg = o2::utils::getIntegralBoostHist(boostHistCellSlice, maxElementCenter - 25, maxElementCenter + 25);
+      double sumPreTrigg = o2::utils::getIntegralBoostHist(boostHistCellSlice, maxElementCenter - 500, maxElementCenter - 25);
+      double sumPostTrigg = o2::utils::getIntegralBoostHist(boostHistCellSlice, maxElementCenter + 25, maxElementCenter + 500);
+
+      // calculate fraction of hits of post and pre-trigger to main trigger bunch
+      timeInfo.fracHitsPreTrigg[i] = sumTrigg == 0 ? 0. : sumPreTrigg / sumTrigg;
+      timeInfo.fracHitsPostTrigg[i] = sumTrigg == 0 ? 0. : sumPostTrigg / sumTrigg;
+    }
+
+    // get the mean sigma and the std. deviation of the sigma distribution
+    // those will be the values we cut on
+    double avMean = 0, avSigma = 0;
+    TRobustEstimator robustEstimator;
+    robustEstimator.EvaluateUni(timeInfo.sigmaCell.size(), timeInfo.sigmaCell.data(), avMean, avSigma, 0.5 * timeInfo.sigmaCell.size());
+    // protection for the following case: For low statistics cases, it can happen that more than half of the cells is in one bin
+    // in that case the sigma will be close to zero. In that case, we take 95% of the data to calculate the truncated mean
+    if (std::abs(avMean) < 0.001 && std::abs(avSigma) < 0.001) {
+      robustEstimator.EvaluateUni(timeInfo.sigmaCell.size(), timeInfo.sigmaCell.data(), avMean, avSigma, 0.95 * timeInfo.sigmaCell.size());
+    }
+    // timeInfo.sigmaCell = meanSigma;
+    timeInfo.goodCellWindow = avMean + (avSigma * o2::emcal::EMCALCalibParams::Instance().sigmaTime_bc); // only upper limit needed
+
+    double avMeanPre = 0, avSigmaPre = 0;
+    robustEstimator.EvaluateUni(timeInfo.fracHitsPreTrigg.size(), timeInfo.fracHitsPreTrigg.data(), avMeanPre, avSigmaPre, 0.5 * timeInfo.fracHitsPreTrigg.size());
+    if (std::abs(avMeanPre) < 0.001 && std::abs(avSigmaPre) < 0.001) {
+      robustEstimator.EvaluateUni(timeInfo.fracHitsPreTrigg.size(), timeInfo.fracHitsPreTrigg.data(), avMeanPre, avSigmaPre, 0.95 * timeInfo.fracHitsPreTrigg.size());
+    }
+    timeInfo.goodCellWindowFracHitsPreTrigg = avMeanPre + (avSigmaPre * o2::emcal::EMCALCalibParams::Instance().sigmaTimePreTrigg_bc); // only upper limit needed
+
+    double avMeanPost = 0, avSigmaPost = 0;
+    robustEstimator.EvaluateUni(timeInfo.fracHitsPostTrigg.size(), timeInfo.fracHitsPostTrigg.data(), avMeanPost, avSigmaPost, 0.5 * timeInfo.fracHitsPostTrigg.size());
+    if (std::abs(avMeanPost) < 0.001 && std::abs(avSigmaPost) < 0.001) {
+      robustEstimator.EvaluateUni(timeInfo.fracHitsPostTrigg.size(), timeInfo.fracHitsPostTrigg.data(), avMeanPost, avSigmaPost, 0.95 * timeInfo.fracHitsPostTrigg.size());
+    }
+    timeInfo.goodCellWindowFracHitsPostTrigg = avMeanPost + (avSigmaPost * o2::emcal::EMCALCalibParams::Instance().sigmaTimePostTrigg_bc); // only upper limit needed
+
+    return timeInfo;
+  }
+
   //____________________________________________
 
   /// \brief Calibrate time for all cells
@@ -212,7 +394,7 @@ class EMCALCalibExtractor
   /// \param maxTime -- max. time considered for fit
   /// \param restrictFitRangeToMax -- restrict the fit range to the maximum entry in the histogram in the range +-restrictFitRangeToMax (default: 25ns)
   template 
-  o2::emcal::TimeCalibrationParams calibrateTime(boost::histogram::histogram& hist, double minTime = 0, double maxTime = 1000, double restrictFitRangeToMax = 25)
+  o2::emcal::TimeCalibrationParams calibrateTime(const boost::histogram::histogram& hist, double minTime = 0, double maxTime = 1000, double restrictFitRangeToMax = 25)
   {
 
     auto histReduced = boost::histogram::algorithm::reduce(hist, boost::histogram::algorithm::shrink(minTime, maxTime), boost::histogram::algorithm::shrink(0, mNcells));
@@ -239,36 +421,109 @@ class EMCALCalibExtractor
       auto boostHist1d = o2::utils::ProjectBoostHistoXFast(histReduced, indexLow, indexHigh);
 
       LOG(debug) << "calibrate cell time " << i << " of " << mNcells;
+      int maxElementIndex = std::max_element(boostHist1d.begin(), boostHist1d.end()) - boostHist1d.begin() - 1;
+      if (maxElementIndex < 0) {
+        maxElementIndex = 0;
+      }
+      float maxElementCenter = 0.5 * (boostHist1d.axis(0).bin(maxElementIndex).upper() + boostHist1d.axis(0).bin(maxElementIndex).lower());
       // Restrict fit range to maximum +- restrictFitRangeToMax
       if (restrictFitRangeToMax > 0) {
-        int maxElementIndex = std::max_element(boostHist1d.begin(), boostHist1d.end()) - boostHist1d.begin() - 1;
-        if (maxElementIndex < 0) {
-          maxElementIndex = 0;
-        }
-        float maxElementCenter = 0.5 * (boostHist1d.axis(0).bin(maxElementIndex).upper() + boostHist1d.axis(0).bin(maxElementIndex).lower());
         boostHist1d = boost::histogram::algorithm::reduce(boostHist1d, boost::histogram::algorithm::shrink(maxElementCenter - restrictFitRangeToMax, maxElementCenter + restrictFitRangeToMax));
       }
 
       try {
         auto fitValues = o2::utils::fitBoostHistoWithGaus(boostHist1d);
-        mean = fitValues.at(1);
+        if (maxElementCenter + EMCALCalibParams::Instance().maxAllowedDeviationFromMax < fitValues.at(1) || maxElementCenter - EMCALCalibParams::Instance().maxAllowedDeviationFromMax > fitValues.at(1)) {
+          mean = maxElementCenter;
+        } else {
+          mean = fitValues.at(1);
+        }
         // add mean to time calib params
-        TCP.addTimeCalibParam(i, mean, 0);
+        TCP.addTimeCalibParam(i, mean, false);                                                // highGain calib factor
+        TCP.addTimeCalibParam(i, mean + EMCALCalibParams::Instance().lowGainOffset_tc, true); // lowGain calib factor
       } catch (o2::utils::FitGausError_t err) {
         LOG(warning) << createErrorMessageFitGaus(err) << "; for cell " << i << " (Will take the parameter of the previous cell: " << mean << "ns)";
-        TCP.addTimeCalibParam(i, mean, 0); // take calib value of last cell; or 400 ns shift default value
+        TCP.addTimeCalibParam(i, mean, false);                                                // take calib value of last cell; or 400 ns shift default value
+        TCP.addTimeCalibParam(i, mean + EMCALCalibParams::Instance().lowGainOffset_tc, true); // take calib value of last cell; or 400 ns shift default value
       }
     }
     return TCP;
   }
 
- private:
-  bool mUseScaledHistoForBadChannels = false; ///< variable to specify whether or not we want to use the scaled histo for the claibration of bad channels.
-  int mSigma = 4;                             ///< number of sigma used in the calibration to define outliers
-  int mNThreads = 1;                          ///< number of threads used for calibration
+  //____________________________________________
+  /// \brief Extract the pedestals from Stat Accumulators
+  /// \param obj PedestalProcessorData containing the data
+  /// \return Pedestal data
+  Pedestal extractPedestals(PedestalProcessorData& obj)
+  {
+    Pedestal pedestalData;
+    // loop over both low and high gain data as well as normal and LEDMON data
+    for (const auto& isLEDMON : {false, true}) {
+      auto maxChannels = isLEDMON ? mLEDMONs : mNcells;
+      for (const auto& isLG : {false, true}) {
+        for (unsigned short iCell = 0; iCell < maxChannels; ++iCell) {
+          auto [mean, rms] = obj.getValue(iCell, isLG, isLEDMON); // get mean and rms for pedestals
+          if (rms > EMCALCalibParams::Instance().maxPedestalRMS) {
+            mean = mMaxPedestalVal;
+          }
+          pedestalData.addPedestalValue(iCell, std::round(mean), isLG, isLEDMON);
+        }
+      }
+    }
+    return pedestalData;
+  }
+
+  //____________________________________________
+  /// \brief Extract the pedestals from TProfile (for old data)
+  /// \param objHG TProfile containing the HG data
+  /// \param objLHG TProfile containing the LG data
+  /// \param isLEDMON if true, data is LED data
+  /// \return Pedestal data
+  Pedestal extractPedestals(TProfile* objHG = nullptr, TProfile* objLG = nullptr, bool isLEDMON = false)
+  {
+    Pedestal pedestalData;
+    auto maxChannels = isLEDMON ? mLEDMONs : mNcells;
+    // loop over both low and high gain data
+    for (const auto& isLG : {false, true}) {
+      auto obj = (isLG == true ? objLG : objHG);
+      if (!obj)
+        continue;
+      for (unsigned short iCell = 0; iCell < maxChannels; ++iCell) {
+        short mean = static_cast(std::round(obj->GetBinContent(iCell + 1)));
+        short rms = static_cast(obj->GetBinError(iCell + 1) / obj->GetBinEntries(iCell + 1));
+        if (rms > EMCALCalibParams::Instance().maxPedestalRMS) {
+          mean = mMaxPedestalVal;
+        }
+        pedestalData.addPedestalValue(iCell, mean, isLG, isLEDMON);
+      }
+    }
+    return pedestalData;
+  }
 
-  o2::emcal::Geometry* mGeometry = nullptr;
-  static constexpr int mNcells = 17664;
+ private:
+  //____________________________________________
+  /// \brief Check if a SM exceeds a certain fraction of dead+bad channels. If yes, mask the entire SM
+  /// \param bcm -- current bad channel map
+  void checkMaskSM(o2::emcal::BadChannelMap& bcm);
+
+  /// \brief Check if a FEC exceeds a certain fraction of dead+bad channels. If yes, mask the entire FEC
+  /// \param bcm -- current bad channel map
+  void checkMaskFEC(o2::emcal::BadChannelMap& bcm);
+
+  /// \brief Get the FEC ID in a SM (IDs are just for internal handling in this task itself)
+  /// \param absCellID -- cell ID
+  unsigned int getFECNumberInSM(int absCellID) const;
+
+  EMCALChannelScaleFactors* mBCMScaleFactors = nullptr;  ///< Scale factors for nentries scaling in bad channel calibration
+  int mSigma = 5;                                        ///< number of sigma used in the calibration to define outliers
+  int mNThreads = 1;                                     ///< number of threads used for calibration
+  std::array mBadCellFracSM;                  ///< Fraction of bad+dead channels per SM
+  std::array, 20> mBadCellFracFEC; ///< Fraction of bad+dead channels per FEC
+
+  o2::emcal::Geometry* mGeometry = nullptr;      ///< pointer to the emcal geometry class
+  static constexpr int mNcells = 17664;          ///< Number of total cells of EMCal + DCal
+  static constexpr int mLEDMONs = 480;           ///< Number of total LEDMONS of EMCal + DCal
+  static constexpr short mMaxPedestalVal = 1023; ///< Maximum value for pedestals
 
   ClassDefNV(EMCALCalibExtractor, 1);
 };
diff --git a/Detectors/EMCAL/calibration/include/EMCALCalibration/EMCALCalibParams.h b/Detectors/EMCAL/calibration/include/EMCALCalibration/EMCALCalibParams.h
index 05cbb68447965..0de733dd953ab 100644
--- a/Detectors/EMCAL/calibration/include/EMCALCalibration/EMCALCalibParams.h
+++ b/Detectors/EMCAL/calibration/include/EMCALCalibration/EMCALCalibParams.h
@@ -29,18 +29,79 @@ namespace emcal
 // class containing the parameters to trigger the calibrations
 struct EMCALCalibParams : public o2::conf::ConfigurableParamHelper {
 
+  // parameters for bad channel calibration
+  unsigned int minNEvents_bc = 1e7;     ///< minimum number of events to trigger the calibration
+  unsigned int minNEntries_bc = 1e6;    ///< minimum number of entries to trigger the calibration
+  bool useNEventsForCalib_bc = true;    ///< use the minimum number of events to trigger the calibration
+  bool useScaledHisto_bc = true;        ///< use the scaled histogram for the bad channel map
+  bool enableTestMode_bc = false;       ///< enable test mode for calibration
+  int nBinsEnergyAxis_bc = 1000;        ///< number of bins for boost histogram energy axis
+  bool useTimeInfoForCalib_bc = true;   ///< weather to use the timing information as a criterion in the bad channel analysis
+  float maxValueEnergyAxis_bc = 40;     ///< maximum value for boost histogram energy axis (minimum is always 0)
+  int nBinsTimeAxis_bc = 1000;          ///< number of bins for boost histogram time axis
+  float rangeTimeAxisLow_bc = -500;     ///< minimum value of time for histogram range
+  float rangeTimeAxisHigh_bc = 500;     ///< maximum value of time for histogram range
+  float minCellEnergyTime_bc = 0.1;     ///< minimum energy needed to fill the time histogram
+  float sigmaTime_bc = 5;               ///< sigma value for the upper cut on the time-variance distribution
+  float sigmaTimePreTrigg_bc = 10.;     ///< sigma value for the upper cut on the fraction of cells in the pre-trigger region
+  float sigmaTimePostTrigg_bc = 10.;    ///< sigma value for the upper cut on the fraction of cells in the post-trigger region
+  unsigned int slotLength_bc = 0;       ///< Lenght of the slot before calibration is triggered. If set to 0 calibration is triggered when hasEnoughData returns true
+  bool UpdateAtEndOfRunOnly_bc = false; ///< switch to enable trigger of calibration only at end of run
+  float minNHitsForMeanEnergyCut = 100; ///< mean number of hits per cell that is needed to cut on the mean energy per hit. Needed for high energy intervals as outliers can distort the distribution
+  float minNHitsForNHitCut = 1;         ///< mean number of hits per cell that is needed to cut on the mean number of hits. Needed for high energy intervals as outliers can distort the distribution
+  float minCellEnergy_bc = 0.1;         ///< minimum cell energy considered for filling the histograms for bad channel calib. Should speedup the filling of the histogram to suppress noise
+  float fractionEvents_bc = 1.;         ///< fraction of events used in bad channel calibration
+  size_t nThreads_bc = 4;               ///< number of threads used for the bad channel calinration for filling the histograms
+  float fracMaskSMFully_bc = 0.5;       ///< fraction of bad+dead channel for a SM to be fully masked
+  float fracMaskFECFully_bc = 0.9;      ///< fraction of bad+dead channel for a FEC to be fully masked
+
+  // parameters for time calibration
+  unsigned int minNEvents_tc = 1e7;      ///< minimum number of events to trigger the calibration
+  unsigned int minNEntries_tc = 1e6;     ///< minimum number of entries to trigger the calibration
+  bool useNEventsForCalib_tc = true;     ///< use the minimum number of events to trigger the calibration
+  float minCellEnergy_tc = 0.5;          ///< minimum cell energy to enter the time calibration (typical minimum seed energy for clusters), time resolution gets better with rising energy
+  int minTimeForFit_tc = -300;           ///< minimum cell time considered for the time calibration in ns
+  int maxTimeForFit_tc = 300;            ///< maximum cell time considered for the time calibration in ns
+  int restrictFitRangeToMax_tc = 25;     ///< window around the largest entry within the minTimeForFit in which the fit is performed in ns
+  int nBinsTimeAxis_tc = 1500;           ///< number of bins used for the time calibration
+  int minValueTimeAxis_tc = -500;        ///< minimum value of the time axis in the time calibration
+  int maxValueTimeAxis_tc = 1000;        ///< maximum value of the time axis in the time calibration
+  float lowGainOffset_tc = 8.8;          ///< Offset between high gain and low gain in ns. This is needed since the low gain calib will not be possible due to insuficient statistics
+  unsigned int slotLength_tc = 0;        ///< Lenght of the slot before calibration is triggered. If set to 0 calibration is triggered when hasEnoughData returns true
+  bool UpdateAtEndOfRunOnly_tc = false;  ///< switch to enable trigger of calibration only at end of run
+  float maxAllowedDeviationFromMax = 10; ///< maximum deviation allowed between the estimated maximum of the fit and the true maximum from the distribution. If deviation is larger then value, the fit likely failed. In this case, the true value is taken
+  float fractionEvents_tc = 1.;          ///< fraction of events used in time calibration
+  size_t nThreads_tc = 2;                ///< number of threads used for the time calinration for filling the histograms
+
+  // common parameters
+  std::string calibType = "time";                      ///< type of calibration to run
+  std::string localRootFilePath = "";                  ///< path to local root file in order to store the calibration histograms (off by default, only to be used for testing)
+  bool enableFastCalib = false;                        ///< switch to enable fast calibration. Instead of filling boost histograms, mean and sigma of cells is calculated on the fly
+  bool enableTimeProfiling = false;                    ///< enable to log how much time is spent in the run function in the calibrator spec. Needed for speed tests offline and at point 2
+  bool setSavedSlotAllowed_EMC = true;                 ///< if true, saving and loading of calibrations from last run and for next run is enabled
+  bool setSavedSlotAllowedSOR_EMC = true;              ///< if true, stored calibrations from last run can be loaded in the next run (if false, storing of the calib histograms is still active in contrast to setSavedSlotAllowed_EMC)
+  long endTimeMargin = 2592000000;                     ///< set end TS to 30 days after slot ends (1000 * 60 * 60 * 24 * 30)
+  std::string selectedClassMasks = "C0TVX-B-NOPF-EMC"; ///< name of EMCal min. bias trigger that is used for calibration
+  int bcShiftCTP = 0;                                  ///< bc shift of CTP digits to align them with EMC bc in case they are misaligned
+  std::string filePathSave = "./emc_calib";            ///< path where calibration histograms are stored at EOR to save them for the next run
+  bool requireSameFill = false;                        ///< if loading calib objects from previous run, require it to be in the same fill as the current one
+  bool requireSameRunType = true;                      ///< if loading calib objects from previous run, require it to be the same run type
+  int tsDiffMax = 48;                                  ///< if loading calib objects from previous run, limit time between the object being stored and loaded again (in hours)
+  unsigned int minNEventsSaveSlot = 100000;            ///< minimum amount a slot has to have in order to be taken into accoutn in finalize slot. THis is also relevant if the slot should be saved at the EOR
+  bool useStaticStartTimeSlot = true;                  ///< if set to true, allows to use the start timestamp set manually. This is important if data from a previous run was loaded as otherwise, the start-timestamp will not include data from the previous run
+
+  // Parameters for pedestal calibration
+  short maxPedestalRMS = 10; ///< Maximum value for RMS for pedestals (has to be tuned)
+
+  // old parameters. Keep them for a bit (can be deleted after september 5th) as otherwise ccdb and o2 version might not be in synch
   unsigned int minNEvents = 1e7;              ///< minimum number of events to trigger the calibration
   unsigned int minNEntries = 1e6;             ///< minimum number of entries to trigger the calibration
   bool useNEventsForCalib = true;             ///< use the minimum number of events to trigger the calibration
-  std::string calibType = "time";             ///< type of calibration to run
-  std::string localRootFilePath = "";         ///< path to local root file in order to store the calibration histograms (off by default, only to be used for testing)
   bool useScaledHistoForBadChannelMap = true; ///< use the scaled histogram for the bad channel map
   bool enableTestMode = false;                ///< enable test mode for calibration
   float minCellEnergyForTimeCalib = 0.5;      ///< minimum cell energy to enter the time calibration (typical minimum seed energy for clusters), time resolution gets better with rising energy
   unsigned int slotLength = 0;                ///< Lenght of the slot before calibration is triggered. If set to 0 calibration is triggered when hasEnoughData returns true
-  bool UpdateAtEndOfRunOnly = false;          ///< switsch to enable trigger of calibration only at end of run
-  bool enableTimeProfiling = false;           ///< enable to log how much time is spent in the run function in the calibrator spec. Needed for speed tests offline and at point 2
-  bool enableFastCalib = false;               ///< switch to enable fast calibration. Instead of filling boost histograms, mean and sigma of cells is calculated on the fly
+  bool UpdateAtEndOfRunOnly = false;          ///< switch to enable trigger of calibration only at end of run
   int minTimeForFit = -300;                   ///< minimum cell time considered for the time calibration in ns
   int maxTimeForFit = 300;                    ///< maximum cell time considered for the time calibration in ns
   int restrictFitRangeToMax = 25;             ///< window around the largest entry within the minTimeForFit in which the fit is performed in ns
diff --git a/Detectors/EMCAL/calibration/include/EMCALCalibration/EMCALChannelCalibrator.h b/Detectors/EMCAL/calibration/include/EMCALCalibration/EMCALChannelCalibrator.h
index e6e0e1407f860..80d19fe723979 100644
--- a/Detectors/EMCAL/calibration/include/EMCALCalibration/EMCALChannelCalibrator.h
+++ b/Detectors/EMCAL/calibration/include/EMCALCalibration/EMCALChannelCalibrator.h
@@ -22,12 +22,14 @@
 #include "EMCALCalibration/EMCALChannelData.h"
 #include "EMCALCalibration/EMCALCalibExtractor.h"
 #include "EMCALCalibration/EMCALCalibParams.h"
+#include "EMCALCalib/GainCalibrationFactors.h"
 #include "DetectorsCalibration/TimeSlotCalibration.h"
 #include "DetectorsCalibration/TimeSlot.h"
 #include "DataFormatsEMCAL/Cell.h"
 #include "EMCALBase/Geometry.h"
 #include "CCDB/CcdbObjectInfo.h"
 #include "EMCALCalib/CalibDB.h"
+#include "DataFormatsParameters/GRPECSObject.h"
 
 #include "Framework/Logger.h"
 #include "CommonUtils/MemFileHelper.h"
@@ -41,17 +43,14 @@
 #include 
 #include 
 
-using boostHisto2d = boost::histogram::histogram, boost::histogram::axis::regular>, boost::histogram::unlimited_storage>>;
-
 namespace o2
 {
 namespace emcal
 {
 /// \brief class used for managment of bad channel and time calibration
 /// template DataInput can be ChannelData or TimeData   // o2::emcal::EMCALChannelData, o2::emcal::EMCALTimeCalibData
-/// template HistContainer can be ChannelCalibInitParams or TimeCalibInitParams
-template 
-class EMCALChannelCalibrator : public o2::calibration::TimeSlotCalibration
+template 
+class EMCALChannelCalibrator : public o2::calibration::TimeSlotCalibration
 {
   using TFType = o2::calibration::TFType;
   using Slot = o2::calibration::TimeSlot;
@@ -60,7 +59,10 @@ class EMCALChannelCalibrator : public o2::calibration::TimeSlotCalibration;
 
  public:
-  EMCALChannelCalibrator(int nb = 1000, float r = 0.35) : mNBins(nb), mRange(r){};
+  EMCALChannelCalibrator(int nb = 1000, float r = 0.35) : mNBins(nb), mRange(r)
+  {
+    this->setSaveDirectory(EMCALCalibParams::Instance().filePathSave);
+  };
 
   ~EMCALChannelCalibrator() final = default;
 
@@ -71,6 +73,10 @@ class EMCALChannelCalibrator : public o2::calibration::TimeSlotCalibration& emplaceNewSlot(bool front, TFType tstart, TFType tend) final;
 
+  bool saveLastSlotData(TFile& fl) final;
+
+  bool adoptSavedData(const o2::calibration::TimeSlotMetaData& metadata, TFile& fl) final;
+
   ///\brief Set the testing status.
   void setIsTest(bool isTest) { mTest = isTest; }
   bool isTest() const { return mTest; }
@@ -78,15 +84,44 @@ class EMCALChannelCalibrator : public o2::calibration::TimeSlotCalibration& getOutputVector() const { return mCalibObjectVector; }
 
+  /// \brief get if has enough data should be circumvented at EOR
+  bool getSaveAtEOR() const { return mSaveAtEOR; }
+  /// \brief set if has enough data should be circumvented at EOR
+  void setSaveAtEOR(bool tmp) { mSaveAtEOR = tmp; }
+
+  /// \brief get if has enough data should be circumvented at EOR
+  bool getLoadAtSOR() const { return mLoadAtSOR; }
+  /// \brief set if has enough data should be circumvented at EOR
+  void setLoadAtSOR(bool tmp) { mLoadAtSOR = tmp; }
+
   /// \brief Configure the calibrator
   std::shared_ptr getCalibExtractor() { return mCalibrator; } // return shared pointer!
   /// \brief setter for mCalibrator
   void SetCalibExtractor(std::shared_ptr extr) { mCalibrator = extr; };
 
+  bool setGainCalibrationFactors(o2::emcal::GainCalibrationFactors* gainCalibFactors);
+
+  /// \brief Set current fill number
+  /// \param fn fill number
+  void setFillNr(int fn) { mFillNr = fn; }
+
+  /// \brief Set current run type
+  /// \param rt tun type
+  void setRunType(o2::parameters::GRPECSObject::RunType rt) { mRunType = rt; }
+
+  /// \brief Set current timestamp obtained from data
+  /// \param ts timestamp in hours
+  void setCurrTSInHours(int ts) { mStartTSCalib = ts; }
+
  private:
-  int mNBins = 0;     ///< bins of the histogram for passing
-  float mRange = 0.;  ///< range of the histogram for passing
-  bool mTest = false; ///< flag to be used when running in test mode: it simplify the processing (e.g. does not go through all channels)
+  int mNBins = 0;                                                                               ///< bins of the histogram for passing
+  float mRange = 0.;                                                                            ///< range of the histogram for passing
+  bool mTest = false;                                                                           ///< flag to be used when running in test mode: it simplify the processing (e.g. does not go through all channels)
+  bool mSaveAtEOR = false;                                                                      ///< flag to pretend to have enough data in order to trigger the saving of the calib histograms for loading them at the next run
+  bool mLoadAtSOR = false;                                                                      ///< flag weather to load the calibration histograms from the previous run at the SOR
+  o2::parameters::GRPECSObject::RunType mRunType = o2::parameters::GRPECSObject::RunType::NONE; ///< Run type needed if previous calibration is loaded.
+  int mFillNr = 0;                                                                              ///< fill nr. needed if previous calibration is loaded.
+  int mStartTSCalib = -1;                                                                       ///< First timestamp for calibration. Needed to check if calibration data from previous slot should be loaded
   std::shared_ptr mCalibrator;
 
   // output
@@ -97,51 +132,108 @@ class EMCALChannelCalibrator : public o2::calibration::TimeSlotCalibration
-void EMCALChannelCalibrator::initOutput()
+template 
+void EMCALChannelCalibrator::initOutput()
 {
   mInfoVector.clear();
   mCalibObjectVector.clear();
+  std::string nameFile = "tcp.root";
+  if constexpr (std::is_same::value) {
+    nameFile = "bcm.root";
+  }
+
+  // this->setSaveFileName(nameFile);
   // mNEvents = 0;
   return;
 }
 
 //_____________________________________________
-template 
-bool EMCALChannelCalibrator::hasEnoughData(const o2::calibration::TimeSlot& slot) const
+template 
+bool EMCALChannelCalibrator::hasEnoughData(const o2::calibration::TimeSlot& slot) const
 {
+  // in case of the end of run, we pretend to have enough data to trigger the saving of the calib objects to load them in the next run
+  if (mSaveAtEOR) {
+    return true;
+  }
   const DataInput* c = slot.getContainer();
   return (mTest ? true : c->hasEnoughData());
 }
 
 //_____________________________________________
-template 
-void EMCALChannelCalibrator::finalizeSlot(o2::calibration::TimeSlot& slot)
+template 
+void EMCALChannelCalibrator::finalizeSlot(o2::calibration::TimeSlot& slot)
 {
 
   // Extract results for the single slot
   DataInput* c = slot.getContainer();
   LOG(info) << "Finalize slot " << slot.getTFStart() << " <= TF <= " << slot.getTFEnd();
+  // check if slot contains a minimum amount of data
+  if (c->getNEvents() < EMCALCalibParams::Instance().minNEventsSaveSlot) {
+    LOG(info) << "Slot only contains " << c->getNEvents() << " events. Not saving this slot. " << EMCALCalibParams::Instance().minNEventsSaveSlot << " required";
+    return;
+  }
+
+  if constexpr (std::is_same::value) {
+    if (c->getNEvents() < EMCALCalibParams::Instance().minNEvents_bc) {
+      LOG(info) << "saving the slot with " << c->getNEvents() << " events. " << EMCALCalibParams::Instance().minNEvents_tc << " events needed for calibration";
+      this->saveLastSlot();
+      return;
+    }
+  } else if constexpr (std::is_same::value) {
+    if (c->getNEvents() < EMCALCalibParams::Instance().minNEvents_tc) {
+      LOG(info) << "saving the slot with " << c->getNEvents() << " events. " << EMCALCalibParams::Instance().minNEvents_tc << " events needed for calibration";
+      this->saveLastSlot();
+      return;
+    }
+  }
+
+  // get the start timestamp of the slot. Either use the manually set start ts if available, or just the start time of the slot.
+  long tsStart = slot.getStartTimeMS();
+  if (EMCALCalibParams::Instance().useStaticStartTimeSlot && slot.getStaticStartTimeMS() > 0) {
+    tsStart = slot.getStaticStartTimeMS();
+    LOG(info) << "Adjusting the start TS of the slot from " << slot.getStartTimeMS() << " to " << slot.getStaticStartTimeMS();
+  }
 
   std::map md;
   if constexpr (std::is_same::value) {
     LOG(debug) << "Launching the calibration.";
-    auto bcm = mCalibrator->calibrateBadChannels(c->getHisto());
+    o2::emcal::BadChannelMap bcm = mCalibrator->calibrateBadChannels(c->getHisto(), c->getHistoTime());
     LOG(debug) << "Done with the calibraiton";
     // for the CCDB entry
     auto clName = o2::utils::MemFileHelper::getClassName(bcm);
     auto flName = o2::ccdb::CcdbApi::generateFileName(clName);
-    mInfoVector.emplace_back(CalibDB::getCDBPathBadChannelMap(), clName, flName, md, slot.getStartTimeMS(), o2::ccdb::CcdbObjectInfo::INFINITE_TIMESTAMP);
+    mInfoVector.emplace_back(CalibDB::getCDBPathBadChannelMap(), clName, flName, md, tsStart, slot.getEndTimeMS() + EMCALCalibParams::Instance().endTimeMargin, true);
     mCalibObjectVector.push_back(bcm);
+
+    if ((EMCALCalibParams::Instance().localRootFilePath).find(".root") != std::string::npos) {
+      std::ifstream ffile(EMCALCalibParams::Instance().localRootFilePath.c_str());
+
+      TFile fLocalStorage((EMCALCalibParams::Instance().localRootFilePath).c_str(), ffile.good() == true ? "update" : "recreate");
+      fLocalStorage.cd();
+
+      TH2F* histBCMap = (TH2F*)bcm.getHistogramRepresentation();
+      std::string nameBCHist = "BadChannels_" + std::to_string(tsStart);
+      histBCMap->Write(nameBCHist.c_str(), TObject::kOverwrite);
+
+      TH2F hCalibHist = o2::utils::TH2FFromBoost(c->getHisto());
+      std::string nameBCInputHist = "EnergyVsCellID_" + std::to_string(tsStart);
+      hCalibHist.Write(nameBCInputHist.c_str(), TObject::kOverwrite);
+
+      TH2F hCalibHistTime = o2::utils::TH2FFromBoost(c->getHistoTime());
+      std::string nameBCInputHistTime = "TimeVsCellID_" + std::to_string(tsStart);
+      hCalibHistTime.Write(nameBCInputHistTime.c_str(), TObject::kOverwrite);
+
+      fLocalStorage.Close();
+    }
   } else if constexpr (std::is_same::value) {
-    auto tcd = mCalibrator->calibrateTime(c->getHisto(), EMCALCalibParams::Instance().minTimeForFit, EMCALCalibParams::Instance().maxTimeForFit, EMCALCalibParams::Instance().restrictFitRangeToMax);
+    o2::emcal::TimeCalibrationParams tcd = mCalibrator->calibrateTime(c->getHisto(), EMCALCalibParams::Instance().minTimeForFit_tc, EMCALCalibParams::Instance().maxTimeForFit_tc, EMCALCalibParams::Instance().restrictFitRangeToMax_tc);
 
     // for the CCDB entry
     auto clName = o2::utils::MemFileHelper::getClassName(slot);
     auto flName = o2::ccdb::CcdbApi::generateFileName(clName);
 
-    //prepareCCDBobjectInfo
-    mInfoVector.emplace_back(CalibDB::getCDBPathTimeCalibrationParams(), clName, flName, md, slot.getStartTimeMS(), o2::ccdb::CcdbObjectInfo::INFINITE_TIMESTAMP);
+    // prepareCCDBobjectInfo
+    mInfoVector.emplace_back(CalibDB::getCDBPathTimeCalibrationParams(), clName, flName, md, tsStart, slot.getEndTimeMS() + EMCALCalibParams::Instance().endTimeMargin, true);
     mCalibObjectVector.push_back(tcd);
 
     if ((EMCALCalibParams::Instance().localRootFilePath).find(".root") != std::string::npos) {
@@ -149,28 +241,189 @@ void EMCALChannelCalibrator::finalizeSlot(
 
       TFile fLocalStorage((EMCALCalibParams::Instance().localRootFilePath).c_str(), ffile.good() == true ? "update" : "recreate");
       fLocalStorage.cd();
-      TH1F* histTCparams = (TH1F*)tcd.getHistogramRepresentation(false);
-      std::string nameTCHist = "TCParams_" + std::to_string(slot.getStartTimeMS());
+      TH1F* histTCparams = (TH1F*)tcd.getHistogramRepresentation(false); // high gain calibration
+      std::string nameTCHist = "TCParams_HG_" + std::to_string(tsStart);
       histTCparams->Write(nameTCHist.c_str(), TObject::kOverwrite);
 
+      TH1F* histTCparams_LG = (TH1F*)tcd.getHistogramRepresentation(true); // low gain calibration
+      std::string nameTCHist_LG = "TCParams_LG_" + std::to_string(tsStart);
+      histTCparams_LG->Write(nameTCHist_LG.c_str(), TObject::kOverwrite);
+
       TH2F hCalibHist = o2::utils::TH2FFromBoost(c->getHisto());
-      std::string nameTCInputHist = "TimeVsCellID_" + std::to_string(slot.getStartTimeMS());
+      std::string nameTCInputHist = "TimeVsCellID_" + std::to_string(tsStart);
       hCalibHist.Write(nameTCInputHist.c_str(), TObject::kOverwrite);
       fLocalStorage.Close();
     }
   }
 }
 
-template 
-o2::calibration::TimeSlot& EMCALChannelCalibrator::emplaceNewSlot(bool front, TFType tstart, TFType tend)
+template 
+o2::calibration::TimeSlot& EMCALChannelCalibrator::emplaceNewSlot(bool front, TFType tstart, TFType tend)
 {
-  auto& cont = o2::calibration::TimeSlotCalibration::getSlots();
+  auto& cont = o2::calibration::TimeSlotCalibration::getSlots();
   auto& slot = front ? cont.emplace_front(tstart, tend) : cont.emplace_back(tstart, tend);
-  HistContainer histcont; // initialize struct with (default) ranges for time or channel calibration.
-  slot.setContainer(std::make_unique(histcont));
+  slot.setContainer(std::make_unique());
   return slot;
 }
 
+/// \brief Write histograms for energy and time vs cell ID to file
+/// \param fl file that we write the histograms to
+template 
+bool EMCALChannelCalibrator::saveLastSlotData(TFile& fl)
+{
+  LOG(info) << "EMC calib histos are saved in " << fl.GetName();
+  // we only have 1 slot
+  auto& cont = o2::calibration::TimeSlotCalibration::getSlots();
+  auto& slot = cont.at(0);
+  DataInput* c = slot.getContainer();
+
+  // get the start timestamp of the slot. Either use the manually set start ts if available, or just the start time of the slot.
+  long tsStart = slot.getStartTimeMS();
+  if (EMCALCalibParams::Instance().useStaticStartTimeSlot && slot.getStaticStartTimeMS() > 0) {
+    tsStart = slot.getStaticStartTimeMS();
+  }
+  // timestamp in hours and seconds
+  int timeNowHour = static_cast(tsStart / o2::ccdb::CcdbObjectInfo::HOUR);
+  int timeNowSec = static_cast(tsStart / o2::ccdb::CcdbObjectInfo::SECOND);
+
+  // create global event properties histogram. Same for both calibration types
+  TH1I hGlobalProperties("hGlobalProperties", "hGlobalProperties", 4, -0.5, 3.5);
+  hGlobalProperties.GetXaxis()->SetBinLabel(1, "Fill nr.");
+  hGlobalProperties.GetXaxis()->SetBinLabel(2, "run type");
+  hGlobalProperties.GetXaxis()->SetBinLabel(3, "ts in hours");
+  hGlobalProperties.GetXaxis()->SetBinLabel(4, "ts in seconds");
+  hGlobalProperties.SetBinContent(1, mFillNr);
+  hGlobalProperties.SetBinContent(2, mRunType);
+  hGlobalProperties.SetBinContent(3, timeNowHour);
+  hGlobalProperties.SetBinContent(4, timeNowSec);
+
+  if constexpr (std::is_same::value) {
+    auto hist = c->getHisto();
+    auto histTime = c->getHistoTime();
+
+    TH2F hEnergy = o2::utils::TH2FFromBoost(hist);
+    TH2F hTime = o2::utils::TH2FFromBoost(histTime, "histTime");
+    TH1D hNEvents("hNEvents", "hNEvents", 1, 0, 1);
+    hNEvents.SetBinContent(1, c->getNEvents());
+
+    fl.cd();
+    hEnergy.Write("EnergyVsCellID");
+    hTime.Write("TimeVsCellID");
+    hNEvents.Write("NEvents");
+    hGlobalProperties.Write("GlobalProperties");
+
+  } else if constexpr (std::is_same::value) {
+    auto histTime = c->getHisto();
+    TH2F hTime = o2::utils::TH2FFromBoost(histTime);
+    TH1D hNEvents("hNEvents", "hNEvents", 1, 0, 1);
+    hNEvents.SetBinContent(1, c->getNEvents());
+
+    fl.cd();
+    hTime.Write("TimeVsCellID");
+    hNEvents.Write("NEvents");
+    hGlobalProperties.Write("GlobalProperties");
+  }
+
+  return true;
+}
+
+/// \brief Read histograms for energy and time vs cell ID to file
+/// \param metadata metadata description of the data
+/// \param fl file that we write the histograms to
+template 
+bool EMCALChannelCalibrator::adoptSavedData(const o2::calibration::TimeSlotMetaData& metadata, TFile& fl)
+{
+  LOG(info) << "Loading data from previous run";
+
+  if (!this->getSavedSlotAllowed() || !this->getLoadAtSOR())
+    return true;
+
+  auto& cont = o2::calibration::TimeSlotCalibration::getSlots();
+
+  if (cont.size() == 0) {
+    LOG(warning) << "cont.size() is 0, calibration objects from previous run cannot be loaded...";
+    return true;
+  }
+  auto& slot = cont.at(0);
+  DataInput* c = slot.getContainer();
+
+  // check run type and fill
+  TH1I* hGlobalProperties = (TH1I*)fl.Get("GlobalProperties");
+  if (!hGlobalProperties) {
+    LOG(error) << "GlobalProperties histogram not found. Will not load previous calibration histograms";
+  } else {
+    int fillNr = hGlobalProperties->GetBinContent(1);
+    int runType = hGlobalProperties->GetBinContent(2);
+    int tsOld = hGlobalProperties->GetBinContent(3);
+    int tsOldSec = (hGlobalProperties->GetNbinsX() > 3) ? hGlobalProperties->GetBinContent(4) : 0;                                                 // Protection as 4th bin was only added later
+    int tsDiff = (mStartTSCalib > 0 ? mStartTSCalib : static_cast(o2::ccdb::getCurrentTimestamp() / o2::ccdb::CcdbObjectInfo::HOUR)) - tsOld; // get current timestamp if mStartTSCalib is not set
+    LOG(debug) << "tsOld " << tsOld << "   tsNow " << (mStartTSCalib > 0 ? mStartTSCalib : static_cast(o2::ccdb::getCurrentTimestamp() / o2::ccdb::CcdbObjectInfo::HOUR)) << "   tsDiff " << tsDiff;
+
+    if (EMCALCalibParams::Instance().requireSameRunType && runType != static_cast(mRunType)) {
+      LOG(info) << "adoptSavedData: Same run type required but run types differ: " << runType << " != " << static_cast(mRunType);
+      return false;
+    }
+    if (EMCALCalibParams::Instance().requireSameFill && fillNr != mFillNr) {
+      LOG(info) << "adoptSavedData: Same fill nr. required but fills differ: " << fillNr << " != " << mFillNr;
+      return false;
+    }
+    if (EMCALCalibParams::Instance().tsDiffMax > 0 && (EMCALCalibParams::Instance().tsDiffMax < tsDiff || tsDiff < 0)) {
+      LOG(info) << "adoptSavedData: Maximum difference in ts is: " << EMCALCalibParams::Instance().tsDiffMax << " but " << tsDiff << " is given";
+      return false;
+    }
+    if (EMCALCalibParams::Instance().useStaticStartTimeSlot && tsOldSec > 0) {
+      slot.setStaticStartTimeMS(static_cast(tsOldSec * o2::ccdb::CcdbObjectInfo::SECOND));
+      LOG(info) << "adoptSavedData: Setting the start timestamp to " << static_cast(tsOldSec * o2::ccdb::CcdbObjectInfo::SECOND);
+    }
+  }
+
+  if constexpr (std::is_same::value) {
+    TH2D* hEnergy = (TH2D*)fl.Get("EnergyVsCellID");
+    TH2D* hTime = (TH2D*)fl.Get("TimeVsCellID");
+    if (!hEnergy || !hTime) {
+      return false;
+    }
+    auto hEnergyBoost = o2::utils::boostHistoFromRoot_2D(hEnergy);
+    auto hTimeBoost = o2::utils::boostHistoFromRoot_2D(hTime);
+
+    c->setHisto(hEnergyBoost);
+    c->setHistoTime(hTimeBoost);
+
+  } else if constexpr (std::is_same::value) {
+    TH2D* hTime = (TH2D*)fl.Get("TimeVsCellID");
+    if (!hTime) {
+      return false;
+    }
+    auto hTimeBoost = o2::utils::boostHistoFromRoot_2D(hTime);
+
+    c->setHisto(hTimeBoost);
+  }
+  TH1D* hEvents = (TH1D*)fl.Get("NEvents");
+  if (!hEvents) {
+    return false;
+  }
+  c->setNEvents(hEvents->GetBinContent(1));
+  LOG(info) << "successfully loaded histograms with " << hEvents->GetBinContent(1) << " events";
+
+  return true;
+}
+
+template 
+bool EMCALChannelCalibrator::setGainCalibrationFactors(o2::emcal::GainCalibrationFactors* gainCalibFactors)
+{
+
+  auto& cont = o2::calibration::TimeSlotCalibration::getSlots();
+  if (cont.size() == 0) {
+    return false; // time slot object not yet there
+  }
+
+  auto& slot = cont.at(0);
+  DataInput* c = slot.getContainer();
+  c->setGainCalibFactors(gainCalibFactors);
+
+  return true;
+}
+
 } // end namespace emcal
 } // end namespace o2
 
diff --git a/Detectors/EMCAL/calibration/include/EMCALCalibration/EMCALChannelData.h b/Detectors/EMCAL/calibration/include/EMCALCalibration/EMCALChannelData.h
index b4f8b32abc1cd..0919c157c6610 100644
--- a/Detectors/EMCAL/calibration/include/EMCALCalibration/EMCALChannelData.h
+++ b/Detectors/EMCAL/calibration/include/EMCALCalibration/EMCALChannelData.h
@@ -23,6 +23,7 @@
 // #include "CommonUtils/BoostHistogramUtils.h"
 // #include "EMCALCalib/BadChannelMap.h"
 // #include "EMCALCalib/TimeCalibrationParams.h"
+#include "EMCALCalib/GainCalibrationFactors.h"
 #include "EMCALCalib/EMCALChannelScaleFactors.h"
 #include "EMCALCalibration/EMCALCalibExtractor.h"
 #include "EMCALCalibration/EMCALCalibParams.h"
@@ -38,6 +39,8 @@
 #include 
 // #include 
 
+#include 
+
 // #include 
 
 namespace o2
@@ -46,16 +49,11 @@ namespace emcal
 {
 class EMCALCalibExtractor;
 
-struct ChannelCalibInitParams {
-  unsigned int nbins = 1000;
-  std::array range = {0, 0.35};
-};
-
 class EMCALChannelData
 {
-  //using Slot = o2::calibration::TimeSlot;
+  // using Slot = o2::calibration::TimeSlot;
   using Cells = o2::emcal::Cell;
-  using boostHisto = boost::histogram::histogram, boost::histogram::axis::integer<>>, boost::histogram::unlimited_storage>>;
+  using boostHisto = boost::histogram::histogram, boost::histogram::axis::regular>>;
   using BadChannelMap = o2::emcal::BadChannelMap;
 
  public:
@@ -63,13 +61,21 @@ class EMCALChannelData
   o2::emcal::Geometry* mGeometry = o2::emcal::Geometry::GetInstanceFromRunNumber(300000);
   int NCELLS = mGeometry->GetNCells();
 
-  EMCALChannelData(const ChannelCalibInitParams& hist) : mNBins(hist.nbins), mRange(1)
+  EMCALChannelData() : mNBins(EMCALCalibParams::Instance().nBinsEnergyAxis_bc), mRange(EMCALCalibParams::Instance().maxValueEnergyAxis_bc), mNBinsTime(EMCALCalibParams::Instance().nBinsTimeAxis_bc), mRangeTimeLow(EMCALCalibParams::Instance().rangeTimeAxisLow_bc), mRangeTimeHigh(EMCALCalibParams::Instance().rangeTimeAxisHigh_bc), mNThreads(EMCALCalibParams::Instance().nThreads_bc)
   {
-    // boost histogram with amplitude vs. cell ID, specify the range and binning of the amplitude axis
-    mHisto = boost::histogram::make_histogram(boost::histogram::axis::regular<>(mNBins, 0, mRange, "t-texp"), boost::histogram::axis::integer<>(0, NCELLS, "CELL ID"));
+
     // NCELLS includes DCal, treat as one calibration
     o2::emcal::Geometry* mGeometry = o2::emcal::Geometry::GetInstanceFromRunNumber(300000);
     int NCELLS = mGeometry->GetNCells();
+
+    mVecNEntriesInHisto.resize(mNThreads);
+    mHisto.resize(mNThreads);
+    mHistoTime.resize(mNThreads);
+    for (size_t i = 0; i < mNThreads; ++i) {
+      mHisto[i] = boost::histogram::make_histogram(boost::histogram::axis::regular<>(mNBins, 0., mRange), boost::histogram::axis::regular<>(NCELLS, -0.5, NCELLS - 0.5));
+      mHistoTime[i] = boost::histogram::make_histogram(boost::histogram::axis::regular<>(mNBinsTime, mRangeTimeLow, mRangeTimeHigh), boost::histogram::axis::regular<>(NCELLS, -0.5, NCELLS - 0.5));
+      mVecNEntriesInHisto[i] = 0;
+    }
   }
 
   ~EMCALChannelData() = default;
@@ -85,13 +91,40 @@ class EMCALChannelData
   /// \brief Fill the container with the cell ID and amplitude.
   void fill(const gsl::span data);
   /// \brief Merge the data of two slots.
-  void merge(const EMCALChannelData* prev);
+  void merge(EMCALChannelData* prev);
   // int findBin(float v) const;
   /// \brief Check if enough stataistics was accumulated to perform calibration
   bool hasEnoughData() const;
   /// \brief Get current calibration histogram
-  boostHisto& getHisto() { return mHisto; }
-  const boostHisto& getHisto() const { return mHisto; }
+  const boostHisto& getHisto()
+  {
+    // set the summed histogram to one of the existing histograms
+    mHistoSummed = mHisto[0];
+    // reset the histogram
+    mHistoSummed.reset();
+    // Sum up all entries
+    for (const auto& h : mHisto) {
+      mHistoSummed += h;
+    }
+    return mHistoSummed;
+  }
+
+  /// \brief Set new calibration histogram
+  void setHisto(boostHisto hist, int nthr = 0) { mHisto[nthr] = hist; }
+
+  /// \brief Get current calibration histogram with time information
+  const boostHisto& getHistoTime()
+  {
+    mHistoTimeSummed = mHistoTime[0];
+    mHistoTimeSummed.reset();
+    for (const auto& h : mHistoTime) {
+      mHistoTimeSummed += h;
+    }
+    return mHistoTimeSummed;
+  }
+
+  /// \brief Set new calibration histogram with timing info
+  void setHistoTime(boostHisto hist, int nthr = 0) { mHistoTime[nthr] = hist; }
 
   /// \brief Peform the calibration and flag the bad channel map
   /// Average energy per hit histogram is fitted with a gaussian
@@ -110,18 +143,39 @@ class EMCALChannelData
 
   long unsigned int getNEntriesInHisto() const { return mNEntriesInHisto; }
   void setNEntriesInHisto(long unsigned int n) { mNEntriesInHisto = n; }
+  void addEntriesInHisto(long unsigned int n) { mNEntriesInHisto += n; }
+
+  void setGainCalibFactors(o2::emcal::GainCalibrationFactors* calibFactors)
+  {
+    mGainCalibFactors = calibFactors;
+    for (unsigned int i = 0; i < mArrGainCalibFactors.size(); ++i) {
+      mArrGainCalibFactors[i] = mGainCalibFactors->getGainCalibFactors(i);
+    }
+    mApplyGainCalib = true;
+  }
 
  private:
-  float mRange = 0.35; // looked at old QA plots where max was 0.35 GeV, might need to be changed
-  int mNBins = 1000;
-  boostHisto mHisto;
-  int mEvents = 0;
+  float mRange = 10;                                    ///< Maximum energy range of boost histogram (will be overwritten by values in the EMCALCalibParams)
+  int mNBins = 1000;                                    ///< Number of bins in the boost histogram (will be overwritten by values in the EMCALCalibParams)
+  size_t mNThreads = 1;                                 ///< Number of threads used for filling the boost histograms
+  std::vector mHisto;                       ///< vector of 2d boost histogram with cellID vs cell energy
+  boostHisto mHistoSummed;                              ///< summed 2d boost histogram (sum of mHisto)
+  int mNBinsTime = 1000;                                ///< Number of time bins in boost histogram (cell time vs. cell ID)
+  float mRangeTimeLow = -500;                           ///< lower bound of time axis of mHistoTime
+  float mRangeTimeHigh = 500;                           ///< upper bound of time axis of mHistoTime
+  std::vector mHistoTime;                   ///< vector of 2d boost histogram with cellID vs cell time
+  boostHisto mHistoTimeSummed;                          ///< Summed 2d boost histogram with cellID vs cell time
+  int mEvents = 0;                                      ///< event counter
   long unsigned int mNEntriesInHisto = 0;               ///< Number of entries in the histogram
+  std::vector mVecNEntriesInHisto;   ///< Number of entries in the histogram for each thread per event
   boostHisto mEsumHisto;                                ///< contains the average energy per hit for each cell
   boostHisto mEsumHistoScaled;                          ///< contains the average energy (scaled) per hit for each cell
   boostHisto mCellAmplitude;                            ///< is the input for the calibration, hist of cell E vs. ID
   bool mTest = false;                                   ///< flag to be used when running in test mode: it simplify the processing
   BadChannelMap mOutputBCM;                             ///< output bad channel map for the calibration
+  bool mApplyGainCalib = false;                         ///< Switch if gain calibration is applied or not
+  o2::emcal::GainCalibrationFactors* mGainCalibFactors; ///< Gain calibration factors applied to the data before filling the histograms
+  std::array mArrGainCalibFactors;       ///< array of gain calibration factors
   std::shared_ptr mCalibExtractor; ///< calib extractor
 
   ClassDefNV(EMCALChannelData, 1);
diff --git a/Detectors/EMCAL/calibration/include/EMCALCalibration/EMCALPedestalHelper.h b/Detectors/EMCAL/calibration/include/EMCALCalibration/EMCALPedestalHelper.h
new file mode 100644
index 0000000000000..109fa795fd43a
--- /dev/null
+++ b/Detectors/EMCAL/calibration/include/EMCALCalibration/EMCALPedestalHelper.h
@@ -0,0 +1,68 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef EMCAL_PEDESTAL_HELPER_H_
+#define EMCAL_PEDESTAL_HELPER_H_
+
+#include "CCDB/CcdbApi.h"
+#include "CCDB/BasicCCDBManager.h"
+#include "CCDB/CCDBTimeStampUtils.h"
+#include "EMCALReconstruction/Channel.h"
+#include "EMCALBase/Geometry.h"
+#include "EMCALBase/Mapper.h"
+#include "EMCALCalib/CalibDB.h"
+#include "EMCALCalib/Pedestal.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+namespace o2::emcal
+{
+
+class EMCALPedestalHelper
+{
+
+ public:
+  EMCALPedestalHelper() = default;
+  ~EMCALPedestalHelper() = default;
+
+  /// \brief Encodes the pedestal object into a string. This function fills fMeanPed which is then converted to a string in createInstructionString
+  /// \param obj pedestal object as stored in production ccdb
+  /// \param runNum current runnumber. If -1, will not be added to string that goes in the ccdb, otherwise runNum is the first entrey in the string
+  std::vector createPedestalInstruction(const Pedestal& obj, const int runNum = -1);
+
+  /// \brief print the vector produced by createInstructionString in a textfile
+  void dumpInstructions(const std::string_view filename, const gsl::span& data);
+
+ private:
+  /// \brief initialize fMeanPed with zeros
+  void setZero();
+
+  /// \brief converts fMeanPed to a vector of char
+  /// \param runNum current runnumber. If -1, will not be added to string that goes in the ccdb, otherwise runNum is the first entrey in the string
+  std::vector createInstructionString(const int runNum = -1);
+
+  static constexpr short kNSM = 20;    ///< number of SuperModules
+  static constexpr short kNRCU = 2;    ///< number of readout crates (and DDLs) per SM
+  static constexpr short kNDTC = 40;   ///< links for full SRU
+  static constexpr short kNBranch = 2; ///< low gain/high gain
+  static constexpr short kNFEC = 10;   ///< 0..9, when including LED Ref
+  static constexpr short kNChip = 5;   ///< really 0,2..4, i.e. skip #1
+  static constexpr short kNChan = 16;
+  short fMeanPed[kNSM][kNRCU][kNBranch][kNFEC][kNChip][kNChan];
+};
+
+} // namespace o2::emcal
+
+#endif
\ No newline at end of file
diff --git a/Detectors/EMCAL/calibration/include/EMCALCalibration/EMCALTempCalibExtractor.h b/Detectors/EMCAL/calibration/include/EMCALCalibration/EMCALTempCalibExtractor.h
new file mode 100644
index 0000000000000..5dbaec4c933f8
--- /dev/null
+++ b/Detectors/EMCAL/calibration/include/EMCALCalibration/EMCALTempCalibExtractor.h
@@ -0,0 +1,93 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \class EMCALTempCalibExtractor
+/// \brief  Calculate gain correction factors based on the temperature and the cell-by-cell temperature dependent correction factors (slope and intercept)
+/// \author Joshua König
+/// \ingroup EMCALCalib
+/// \since June 30, 2025
+
+#ifndef EMCALTEMPCALIBEXTRACTOR_H_
+#define EMCALTEMPCALIBEXTRACTOR_H_
+
+#include 
+#include 
+#include 
+#include "CCDB/BasicCCDBManager.h"
+#include "EMCALCalib/ElmbData.h"
+#include "EMCALCalib/TempCalibrationParams.h"
+#include "EMCALBase/Geometry.h"
+
+namespace o2
+{
+namespace emcal
+{
+
+class EMCALTempCalibExtractor
+{
+
+ public:
+  /// \brief Constructor
+  EMCALTempCalibExtractor()
+  {
+    LOG(info) << "initialized EMCALTempCalibExtractor";
+    try {
+      // Try to access geometry initialized ountside
+      mGeometry = o2::emcal::Geometry::GetInstance();
+    } catch (o2::emcal::GeometryNotInitializedException& e) {
+      mGeometry = o2::emcal::Geometry::GetInstanceFromRunNumber(300000); // fallback option
+    }
+  };
+  /// \brief Destructor
+  ~EMCALTempCalibExtractor() = default;
+
+  /// \brief Initialize temperature data and slope for each cell from the ccdb
+  /// \param path path to the slope data
+  /// \param timestamp timestamp for the ccdb objects or runnumber (will detect automatically if its a runnumber and convert it)
+  void InitializeFromCCDB(std::string path, uint64_t timestamp);
+
+  /// \brief get average temperature in a supermodule
+  /// \param iSM SM number
+  /// \param ElmbData object where temperature sensor values are stored
+  /// \return average temperature in a supermodule
+  float getTemperatureForSM(const unsigned short iSM, o2::emcal::ElmbData* ElmbData) const;
+
+  /// \brief get gain calibration factor depending on the temperature and the slope of the cell
+  /// \param cellID cell ID
+  /// \return gain calibration factor
+  float getGainCalibFactor(const unsigned short cellID) const;
+
+  /// \brief set temperature range in which sensor ddata is assumed to be good
+  /// \param low lower temperature
+  /// \param high upper temperature
+  void setAcceptedEnergyRange(float low, float high);
+
+  /// \brief set if median (true) or mean (false) should be used for averaging of the temperature in a SM
+  void setUseMedian(const bool tmp) { mUseMedian = tmp; }
+
+  /// \brief get sensor IDs for a specific supermodule
+  /// \param iSM SM number
+  /// \return vector of sensor IDs
+  std::vector getSensorsForSM(const unsigned short iSM) const;
+
+ private:
+  static constexpr unsigned short mNCells = 17664;      ///< Number of EMCal cells
+  std::array mGainCalibFactors;         ///< gain calibration factors that are calculated based on the temperature and the slopes for each cell
+  o2::emcal::Geometry* mGeometry;                       ///< pointer to the EMCal geometry
+  std::array mAcceptedTempRange = {15., 30.}; ///< Temperature range where sensors are believed to send good data. Temperatures outside this range will be rejected
+  bool mUseMedian = true;                               /// switch to decide if temperature within a SM should be calculated as the mean or the median of the individual sensor data
+};
+
+} // namespace emcal
+
+} // namespace o2
+
+#endif
\ No newline at end of file
diff --git a/Detectors/EMCAL/calibration/include/EMCALCalibration/EMCALTimeCalibData.h b/Detectors/EMCAL/calibration/include/EMCALCalibration/EMCALTimeCalibData.h
index d7ebc11673323..4eb89608c8f33 100644
--- a/Detectors/EMCAL/calibration/include/EMCALCalibration/EMCALTimeCalibData.h
+++ b/Detectors/EMCAL/calibration/include/EMCALCalibration/EMCALTimeCalibData.h
@@ -24,6 +24,7 @@
 #include "EMCALBase/Geometry.h"
 #include "CCDB/CcdbObjectInfo.h"
 #include "EMCALCalib/TimeCalibrationParams.h"
+#include "EMCALCalib/GainCalibrationFactors.h"
 #include "EMCALCalibration/EMCALCalibParams.h"
 
 #include "Framework/Logger.h"
@@ -41,28 +42,25 @@ namespace o2
 namespace emcal
 {
 
-// class containing the initialization parameters for histograms (time bins/range etc.)
-struct TimeCalibInitParams {
- public:
-  unsigned int mTimeBins = 1500;
-  std::array mTimeRange = {-500., 1000.}; // time range in ns
-  unsigned int mEnergyBins = 5000;
-  std::array mEnergyRange = {0., 50.}; // energy range in GeV
-};
-
 class EMCALTimeCalibData
 {
  public:
   using Cells = o2::emcal::Cell;
-  using boostHisto = boost::histogram::histogram, boost::histogram::axis::regular<>>, boost::histogram::unlimited_storage>>;
+  using boostHisto = boost::histogram::histogram, boost::histogram::axis::regular>>;
 
   o2::emcal::Geometry* mGeometry = o2::emcal::Geometry::GetInstanceFromRunNumber(300000);
   int NCELLS = mGeometry->GetNCells();
 
-  EMCALTimeCalibData(const TimeCalibInitParams& par)
+  EMCALTimeCalibData()
   {
-    // boost histogram with amplitude vs. cell ID, specify the range and binning of the amplitude axis
-    mTimeHisto = boost::histogram::make_histogram(boost::histogram::axis::regular<>(par.mTimeBins, par.mTimeRange.at(0), par.mTimeRange.at(1), "t (ns)"), boost::histogram::axis::regular<>(NCELLS, -0.5, NCELLS - 0.5, "CELL ID"));
+
+    mNThreads = EMCALCalibParams::Instance().nThreads_tc;
+    mTimeHisto.resize(mNThreads);
+    mVecNEntriesInHisto.resize(mNThreads);
+    for (size_t i = 0; i < mNThreads; ++i) {
+      mTimeHisto[i] = boost::histogram::make_histogram(boost::histogram::axis::regular<>(EMCALCalibParams::Instance().nBinsTimeAxis_tc, EMCALCalibParams::Instance().minValueTimeAxis_tc, EMCALCalibParams::Instance().maxValueTimeAxis_tc), boost::histogram::axis::regular<>(NCELLS, -0.5, NCELLS - 0.5));
+      mVecNEntriesInHisto[i] = 0;
+    }
     LOG(debug) << "initialize time histogram with " << NCELLS << " cells";
   }
 
@@ -72,7 +70,7 @@ class EMCALTimeCalibData
   void fill(const gsl::span data);
 
   /// \brief Merge the data of two slots.
-  void merge(const EMCALTimeCalibData* prev);
+  void merge(EMCALTimeCalibData* prev);
 
   /// \brief Check if enough data for calibration has been accumulated
   bool hasEnoughData() const;
@@ -91,22 +89,49 @@ class EMCALTimeCalibData
   long unsigned int getNEntriesInHisto() const { return mNEntriesInHisto; }
   /// \brief Set the number of entries in histogram
   void setNEntriesInHisto(long unsigned int n) { mNEntriesInHisto = n; }
+  /// \brief Add the number of entries in histogram
+  void addNEntriesInHisto(long unsigned int n) { mNEntriesInHisto += n; }
 
   /// \brief Get current histogram
-  boostHisto& getHisto() { return mTimeHisto; }
-  const boostHisto& getHisto() const { return mTimeHisto; }
+  const boostHisto& getHisto()
+  {
+    // set the summed histogram to one of the existing histograms
+    mHistoSummed = mTimeHisto[0];
+    // reset the histogram
+    mHistoSummed.reset();
+    // Sum up all entries
+    for (const auto& h : mTimeHisto) {
+      mHistoSummed += h;
+    }
+    return mHistoSummed;
+  }
 
+  /// \brief Set gain calibration factors applied to the cell energy before filling the histograms
+  /// \param calibFactors gain calibration object
+  void setGainCalibFactors(o2::emcal::GainCalibrationFactors* calibFactors)
+  {
+    mGainCalibFactors = calibFactors;
+    mApplyGainCalib = true;
+  }
+
+  /// \brief Set new calibration histogram
+  void setHisto(boostHisto hist) { mTimeHisto[0] = hist; }
+
+  /// \brief print stream
   void PrintStream(std::ostream& stream) const;
 
   /// \brief Actual function where calibration is done. Has to be called in has enough data when enough data is there
   o2::emcal::TimeCalibrationParams process();
 
  private:
-  boostHisto mTimeHisto;                ///< histogram with cell time vs. cell ID
-  TimeCalibInitParams mTimeCalibParams; ///< initialization parameters for histogram
-
-  int mEvents = 0;                        ///< current number of events
-  long unsigned int mNEntriesInHisto = 0; ///< number of entries in histogram
+  unsigned int mNThreads = 1;
+  std::vector mTimeHisto;                   ///< vector of histogram with cell time vs. cell ID (size = number of threads)
+  boostHisto mHistoSummed;                              ///< summed histogram (sum of all histograms in mTimeHisto)
+  int mEvents = 0;                                      ///< current number of events
+  long unsigned int mNEntriesInHisto = 0;               ///< Number of entries in the histogram
+  std::vector mVecNEntriesInHisto;   ///< Number of entries in the histogram for each thread per event
+  bool mApplyGainCalib = false;                         ///< Switch if gain calibration is applied or not
+  o2::emcal::GainCalibrationFactors* mGainCalibFactors; ///< Gain calibration factors applied to the data before filling the histograms
 
   ClassDefNV(EMCALTimeCalibData, 1);
 };
diff --git a/Detectors/EMCAL/calibration/include/EMCALCalibration/PedestalCalibDevice.h b/Detectors/EMCAL/calibration/include/EMCALCalibration/PedestalCalibDevice.h
new file mode 100644
index 0000000000000..6d4cfa8ff2775
--- /dev/null
+++ b/Detectors/EMCAL/calibration/include/EMCALCalibration/PedestalCalibDevice.h
@@ -0,0 +1,61 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef EMCAL_PEDESTAL_CALIB_DEVICE_H_
+#define EMCAL_PEDESTAL_CALIB_DEVICE_H_
+
+#include 
+#include 
+
+#include "Framework/DataProcessorSpec.h"
+#include "Framework/Task.h"
+#include "EMCALBase/Mapper.h"
+#include "EMCALBase/Geometry.h"
+#include "EMCALCalibration/PedestalProcessorData.h"
+#include "EMCALCalibration/EMCALCalibExtractor.h"
+#include "EMCALCalibration/EMCALPedestalHelper.h"
+
+namespace o2::emcal
+{
+
+class PedestalCalibDevice : o2::framework::Task
+{
+ public:
+  PedestalCalibDevice(bool dumpToFile, bool addRunNum) : mDumpToFile(dumpToFile), mAddRunNumber(addRunNum){};
+  ~PedestalCalibDevice() final = default;
+
+  void init(framework::InitContext& ctx) final;
+
+  void run(framework::ProcessingContext& ctx) final;
+
+  void sendData(o2::framework::EndOfStreamContext& ec, const Pedestal& data) const;
+
+  void endOfStream(o2::framework::EndOfStreamContext& ec) final;
+
+  void resetStartTS() { mStartTS = o2::ccdb::getCurrentTimestamp(); }
+
+  static const char* getPedDataBinding() { return "PEDData"; }
+
+ private:
+  Geometry* mGeometry = nullptr;                  ///< pointer to the emcal geometry class
+  o2::emcal::EMCALCalibExtractor mCalibExtractor; ///< instance of the calibration extraction class                                                                ///< Calibration postprocessing
+  PedestalProcessorData mPedestalData;            ///< pedestal data to accumulate data
+  long int mStartTS = 0;                          ///< timestamp at the start of run used for the object in the ccdb
+  bool mDumpToFile;                               ///< if output of pedestal calib (DCS ccdb) should be written to text file
+  int mRun = 0;                                   ///< current run number
+  bool mAddRunNumber = false;                     ///< if true, runNumber will be added to ccdb string
+};
+
+o2::framework::DataProcessorSpec getPedestalCalibDevice(bool dumpToFile, bool addRunNum);
+
+} // end namespace o2::emcal
+
+#endif
\ No newline at end of file
diff --git a/Detectors/EMCAL/calibration/include/EMCALCalibration/PedestalProcessorData.h b/Detectors/EMCAL/calibration/include/EMCALCalibration/PedestalProcessorData.h
new file mode 100644
index 0000000000000..8030c9bc4739a
--- /dev/null
+++ b/Detectors/EMCAL/calibration/include/EMCALCalibration/PedestalProcessorData.h
@@ -0,0 +1,154 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#ifndef EMCAL_PEDESTAL_PROCESSOR_DATA_H_
+#define EMCAL_PEDESTAL_PROCESSOR_DATA_H_
+#include 
+#include 
+#include 
+#include 
+#include 
+#include "MathUtils/Utils.h"
+
+#include "Rtypes.h"
+
+namespace o2::emcal
+{
+
+/// \class PedestalProcessorData
+/// \brief Exchange container between PedestalProcessorDevice and PedestalAggregatorDevice
+/// \ingroup EMCALCalib
+/// \author Markus Fasel , Oak Ridge National Laboratory
+/// \since March 22, 2024
+///
+/// Object containing arrays of stat accumulators that behave like flat profile histograms
+/// calculating mean and RMS of a set of ADC values. Corresponding arrays are used for
+/// both FEC and LEDMON channels, and in both cases for high and low gain. Distinction between
+/// channel and gain type is done via arguments in the fill and get functions, always defining
+/// the true cases with LEDMON and low gain.
+class PedestalProcessorData
+{
+ private:
+ public:
+  /// \class ChannelIndexException
+  /// \brief Handling access to invalid channel index (out-of-bounds)
+  class ChannelIndexException : public std::exception
+  {
+   private:
+    unsigned short mChannelIndex; ///< Index of the channel raising the exception
+    unsigned short mMaxChannels;  ///< Max. number of channels for a given channel type
+    std::string mErrorMessage;    ///< Buffer for the error message
+
+   public:
+    /// \brief Constructor
+    /// \param channelIndex Index raising the exception
+    /// \param maxChannels Maximun number of channels for a given type
+    ChannelIndexException(unsigned short channelIndex, unsigned short maxChannels) : std::exception(), mChannelIndex(channelIndex), mMaxChannels(maxChannels)
+    {
+      mErrorMessage = "Channel index " + std::to_string(mChannelIndex) + " not found (max " + std::to_string(mMaxChannels) + ")";
+    }
+
+    /// \brief  Destructor
+    ~ChannelIndexException() noexcept final = default;
+
+    /// \brief Get error message of the exception
+    /// \return Error message
+    const char* what() const noexcept final { return mErrorMessage.data(); }
+
+    /// \brief Get channel index raising the exception
+    /// \return Channel index
+    unsigned short getChannelIndex() const noexcept { return mChannelIndex; }
+
+    /// \brief Get max number for channels for the type the exception was raised
+    /// \return Number of channels
+    unsigned short getMaxChannels() const noexcept { return mMaxChannels; }
+  };
+
+  using ProfileHistFEC = std::array;
+  using ProfileHistLEDMON = std::array;
+  using PedestalValue = std::tuple;
+
+  /// \brief Constructor
+  PedestalProcessorData() = default;
+
+  /// \brief Destructor
+  ~PedestalProcessorData() = default;
+
+  /// \brief Accumulation operator
+  /// \param other Object to add to this object
+  /// \return This object after accumulation
+  ///
+  /// Adding stat. accumulators for all channels to this object. The state of this
+  /// object is modified.
+  PedestalProcessorData& operator+=(const PedestalProcessorData& other);
+
+  /// \brief Fill ADC value for certain channel
+  /// \param adc ADC value
+  /// \param tower Absolute tower ID
+  /// \param lowGain Switch between low and high gain (true = low gain)
+  /// \param LEDMON Switch between LEDMON and FEE data (true = LEDMON)
+  /// \throw ChannelIndexException for channel index out-of-range
+  void fillADC(unsigned short adc, unsigned short tower, bool lowGain, bool LEDMON);
+
+  /// \brief Get mean ADC and RMS for a certain channel
+  /// \param tower Absolute tower ID
+  /// \param lowGain Switch between low and high gain (true = low gain)
+  /// \param LEDMON Switch between LEDMON and FEE data (true = LEDMON)
+  /// \return std::tuple with mean and rms of the ADC distribution for the given channl
+  /// \throw ChannelIndexException for channel index out-of-range
+  PedestalValue getValue(unsigned short tower, bool lowGain, bool LEDMON) const;
+
+  /// \brief Get number of entries for a certain channel
+  /// \param tower Absolute tower ID
+  /// \param lowGain Switch between low and high gain (true = low gain)
+  /// \param LEDMON Switch between LEDMON and FEE data (true = LEDMON)
+  /// \return Number of entries
+  /// \throw ChannelIndexException for channel index out-of-range
+  int getEntriesForChannel(unsigned short tower, bool lowGain, bool LEDMON) const;
+
+  /// \brief Reset object
+  ///
+  /// Set all stat accumulators to 0.
+  void reset();
+
+  /// \brief Provide access to accumulated data for FEC channels
+  /// \param lowGain Low gain data
+  /// \return Accumulated data for low gain (if lowGain) or high gain
+  const ProfileHistFEC& accessFECData(bool lowGain) const { return lowGain ? mDataFECLG : mDataFECHG; }
+
+  /// \brief Provide access to accumulated data for LEDMON channels
+  /// \param lowGain Low gain data
+  /// \return Accumulated data for low gain (if lowGain) or high gain
+  const ProfileHistLEDMON& accessLEDMONData(bool lowGain) const { return lowGain ? mDataLEDMONLG : mDataLEDMONHG; }
+
+ private:
+  ProfileHistFEC mDataFECHG;       ///< Profile for FEC channels, high gain
+  ProfileHistFEC mDataFECLG;       ///< Profile for FEC channels, low gain
+  ProfileHistLEDMON mDataLEDMONHG; ///< Profile for LEDMON channels, high gain
+  ProfileHistLEDMON mDataLEDMONLG; ///< Profile for LEDMON channels, low gain
+
+  ClassDefNV(PedestalProcessorData, 1);
+};
+
+/// \brief Sum operator for PedestalProcessorData
+/// \param lhs Left hand value of the sum operation
+/// \param rhs Right hand value of the sum operation
+/// \return Sum of the two containers (all channels)
+PedestalProcessorData operator+(const PedestalProcessorData& lhs, const PedestalProcessorData& rhs);
+
+/// @brief Output stream operator for PedestalProcessorData::ChannelIndexException
+/// @param stream Stream used for printing
+/// @param ex Exception to be printed
+/// @return Stream after printing
+std::ostream& operator<<(std::ostream& stream, const PedestalProcessorData::ChannelIndexException& ex);
+
+} // namespace o2::emcal
+
+#endif
\ No newline at end of file
diff --git a/Detectors/EMCAL/calibration/include/EMCALCalibration/PedestalProcessorDevice.h b/Detectors/EMCAL/calibration/include/EMCALCalibration/PedestalProcessorDevice.h
new file mode 100644
index 0000000000000..5f84862838f0c
--- /dev/null
+++ b/Detectors/EMCAL/calibration/include/EMCALCalibration/PedestalProcessorDevice.h
@@ -0,0 +1,130 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef EMCAL_PEDESTAL_PROCESSOR_DEVICE_H_
+#define EMCAL_PEDESTAL_PROCESSOR_DEVICE_H_
+
+#include 
+#include 
+
+#include "Framework/DataProcessorSpec.h"
+#include "Framework/Task.h"
+#include "EMCALBase/Mapper.h"
+#include "EMCALCalibration/PedestalProcessorData.h"
+
+namespace o2::emcal
+{
+
+class Geometry;
+
+/// \class PedestalProcessorDevice
+/// \brief  Processor part of the EMCAL pedestal calibration workflow
+/// \author Markus Fasel , Oak Ridge National Laboratory
+/// \ingroup EMCALCalib
+/// \since March 21, 2024
+class PedestalProcessorDevice : o2::framework::Task
+{
+ private:
+  /// \class ModuleIndexException
+  /// \brief Exception handling errors in calculation of the absolute module ID
+  class ModuleIndexException : public std::exception
+  {
+   public:
+    /// \enum ModuleType_t
+    /// \brief Type of module raising the exception
+    enum class ModuleType_t {
+      CELL_MODULE,  ///< Cell module type
+      LEDMON_MODULE ///< LEDMON module type
+    };
+
+    /// \brief Constructor for cell indices
+    /// \param moduleIndex Index of the module raising the exception
+    /// \param column Column of the cell
+    /// \param row Row of the cell
+    /// \param columnshifted Shifted column index
+    /// \param rowshifted Shifted row index
+    ModuleIndexException(int moduleIndex, int column, int row, int columnshifted, int rowshifted) : mModuleType(ModuleType_t::CELL_MODULE),
+                                                                                                    mIndex(moduleIndex),
+                                                                                                    mColumn(column),
+                                                                                                    mRow(row),
+                                                                                                    mColumnShifted(columnshifted),
+                                                                                                    mRowShifted(rowshifted) {}
+
+    /// \brief Constructor for LEDMON indices
+    /// \param moduleIndex Index of the module raising the exception
+    ModuleIndexException(int moduleIndex) : mModuleType(ModuleType_t::LEDMON_MODULE), mIndex(moduleIndex) {}
+
+    /// \brief Destructor
+    ~ModuleIndexException() noexcept final = default;
+
+    /// \brief Access to error message
+    /// \return Error message
+    const char* what() const noexcept final { return "Invalid cell / LEDMON index"; }
+
+    /// \brief Get type of module raising the exception
+    /// \return Module type
+    ModuleType_t getModuleType() const { return mModuleType; }
+
+    /// \brief Get index of the module raising the exception
+    /// \return Index of the module
+    int getIndex() const { return mIndex; }
+
+    /// \brief Get column raising the exception (cell-case)
+    /// \return Column
+    int getColumn() const { return mColumn; }
+
+    /// \brief Get row raising the exception (cell-case)
+    /// \return Row
+    int getRow() const { return mRow; }
+
+    /// \brief Get shifted column raising the exception (cell-case)
+    /// \return Shifted column
+    int getColumnShifted() const { return mColumnShifted; }
+
+    /// \brief Get shifted row raising the exception (cell-case)
+    /// \return Shifted row
+    int getRowShifted() const { return mRowShifted; }
+
+   private:
+    ModuleType_t mModuleType; ///< Type of the module raising the exception
+    int mIndex = -1;          ///< Index raising the exception
+    int mColumn = -1;         ///< Column of the module (cell-case)
+    int mRow = -1;            ///< Row of the module (cell-case)
+    int mColumnShifted = -1;  ///< shifted column of the module (cell-case)
+    int mRowShifted = -1;     /// << shifted row of the module (cell-case)
+  };
+
+  Geometry* mGeometry = nullptr;
+  std::unique_ptr mMapper = nullptr;
+  PedestalProcessorData mPedestalData;
+
+ protected:
+  int getCellAbsID(int supermoduleID, int column, int row) const;
+  int geLEDMONAbsID(int supermoduleID, int moduleID) const;
+
+ public:
+  PedestalProcessorDevice() = default;
+  ~PedestalProcessorDevice() final = default;
+
+  void init(framework::InitContext& ctx) final;
+
+  void run(framework::ProcessingContext& ctx) final;
+
+  bool isLostTimeframe(framework::ProcessingContext& ctx) const;
+
+  void sendData(framework::ProcessingContext& ctx, const PedestalProcessorData& data) const;
+};
+
+framework::DataProcessorSpec getPedestalProcessorDevice(bool askDistSTF);
+
+} // namespace o2::emcal
+
+#endif
\ No newline at end of file
diff --git a/Detectors/EMCAL/calibration/run/runCalibOffline.cxx b/Detectors/EMCAL/calibration/run/runCalibOffline.cxx
index 2d825f17e8803..3ee505980ce29 100644
--- a/Detectors/EMCAL/calibration/run/runCalibOffline.cxx
+++ b/Detectors/EMCAL/calibration/run/runCalibOffline.cxx
@@ -44,20 +44,22 @@ int main(int argc, char** argv)
   std::string ccdbServerPath;
   bool doBadChannelCalib;
   bool debugMode = false;
-  std::string nameCalibInputHist; // hCellIdVsTimeAbove300 for time, hCellIdVsEnergy for bad channel
-  std::string namePathStoreLocal; // name for path + histogram to store the calibration locally in root TH1 format
-
-  unsigned int nthreads; // number of threads used by openMP
-
-  unsigned long rangestart; //30/10/2021, 01:02:32 for run 505566 -> 1635548552000
-  unsigned long rangeend;   // 30/10/2021, 02:31:10 for run 505566 -> 1635553870000
+  bool doLocal = false;
+  bool doScale = false;
+  bool doBCCalibWithTime = false;
+  std::string nameCalibInputHist;    // hCellIdVsTimeAbove300 for time, hCellIdVsEnergy for bad channel
+  std::string nameCalibInputHistAdd; // additional input histogram for bad channel calibration if time should be considered
+  std::string namePathStoreLocal;    // name for path + histogram to store the calibration locally in root TH1 format
+  unsigned int nthreads;             // number of threads used by openMP
+  unsigned long rangestart;          // 30/10/2021, 01:02:32 for run 505566 -> 1635548552000
+  unsigned long rangeend;            // 30/10/2021, 02:31:10 for run 505566 -> 1635553870000
 
   double timeRangeLow;
   double timeRangeHigh;
 
   try {
     bpo::options_description desc("Allowed options");
-    desc.add_options()("help", "Print this help message")("CalibInputPath", bpo::value()->required(), "Set root input histogram")("ccdbServerPath", bpo::value()->default_value(o2::base::NameConf::getCCDBServer()), "Set path to ccdb server")("debug", bpo::value()->default_value(false), "Enable debug statements")("mode", bpo::value()->required(), "Set if time or bad channel calib")("nameInputHisto", bpo::value()->default_value("hCellIdVsTimeAbove300"), "Set name of input histogram")("nthreads", bpo::value()->default_value(1), "Set number of threads for OpenMP")("timestampStart", bpo::value()->default_value(1635548552000), "Set timestamp from start of run")("timestampEnd", bpo::value()->default_value(1635553870000), "Set timestamp from end of run")("namePathStoreLocal", bpo::value()->default_value(""), "Set path to store histo of time calib locally")("timeRangeLow", bpo::value()->default_value(1), "Set lower boundary of fit interval for time calibration (in ns)")("timeRangeHigh", bpo::value()->default_value(1000), "Set upper boundary of fit interval for time calibration (in ns)");
+    desc.add_options()("help", "Print this help message")("CalibInputPath", bpo::value()->required(), "Set root input histogram")("ccdbServerPath", bpo::value()->default_value(o2::base::NameConf::getCCDBServer()), "Set path to ccdb server")("debug", bpo::value()->default_value(false), "Enable debug statements")("storeCalibLocally", bpo::value()->default_value(false), "Enable local storage of calib")("scaleBadChannelMap", bpo::value()->default_value(false), "Enable the application of scale factors")("mode", bpo::value()->required(), "Set if time or bad channel calib")("nameInputHisto", bpo::value()->default_value("hCellIdVsTimeAbove300"), "Set name of input histogram")("nameInputHistoAdditional", bpo::value()->default_value(""), "Set name of additional input histogram")("nthreads", bpo::value()->default_value(1), "Set number of threads for OpenMP")("timestampStart", bpo::value()->default_value(1635548552000), "Set timestamp from start of run")("timestampEnd", bpo::value()->default_value(1635553870000), "Set timestamp from end of run")("namePathStoreLocal", bpo::value()->default_value(""), "Set path to store histo of time calib locally")("timeRangeLow", bpo::value()->default_value(1), "Set lower boundary of fit interval for time calibration (in ns)")("timeRangeHigh", bpo::value()->default_value(1000), "Set upper boundary of fit interval for time calibration (in ns)");
 
     bpo::store(bpo::parse_command_line(argc, argv, desc), vm);
 
@@ -86,6 +88,18 @@ int main(int argc, char** argv)
       debugMode = vm["debug"].as();
     }
 
+    if (vm.count("storeCalibLocally")) {
+      std::cout << "Enable local storage of calib" << std::endl;
+      doLocal = vm["storeCalibLocally"].as();
+    }
+
+    if (vm.count("scaleBadChannelMap")) {
+      doScale = vm["scaleBadChannelMap"].as();
+      if (doScale) {
+        std::cout << "Enable scaling of the bad channel map" << std::endl;
+      }
+    }
+
     if (vm.count("mode")) {
       std::cout << "mode was set to "
                 << vm["mode"].as() << ".\n";
@@ -108,6 +122,12 @@ int main(int argc, char** argv)
       nameCalibInputHist = vm["nameInputHisto"].as();
     }
 
+    if (vm.count("nameInputHistoAdditional")) {
+      std::cout << "nameInputHistoAdditional was set to "
+                << vm["nameInputHistoAdditional"].as() << ".\n";
+      nameCalibInputHistAdd = vm["nameInputHistoAdditional"].as();
+    }
+
     if (vm.count("nthreads")) {
       std::cout << "number of threads was set to "
                 << vm["nthreads"].as() << ".\n";
@@ -166,31 +186,57 @@ int main(int argc, char** argv)
     return 0;
   }
 
+  // load calibration histogram (cellID vs energy for BC calibration, cellID vs time for time calibration)
   TH2D* hCalibInputHist_ROOT = (TH2D*)fTimeCalibInput->Get(nameCalibInputHist.c_str());
   if (!hCalibInputHist_ROOT) {
     printf("%s not there... returning\n", nameCalibInputHist.c_str());
     return 0;
   }
 
+  // load time vs cellID histogram for the bad channel calibration if specified
+  TH2D* hCalibInputHistAdd_ROOT = nullptr;
+  if (!nameCalibInputHistAdd.empty()) {
+    doBCCalibWithTime = true;
+    hCalibInputHistAdd_ROOT = (TH2D*)fTimeCalibInput->Get(nameCalibInputHistAdd.c_str());
+    if (!hCalibInputHistAdd_ROOT) {
+      printf("%s not there... returning\n", nameCalibInputHist.c_str());
+      return 0;
+    }
+  }
+
   // instance of the calib extractor
   o2::emcal::EMCALCalibExtractor CalibExtractor;
   CalibExtractor.setNThreads(nthreads);
 
   // convert the test root histogram to boost
-  auto hCalibInputHist = o2::utils::boostHistoFromRoot_2D(hCalibInputHist_ROOT);
+  boostHisto2d_VarAxis hCalibInputHist = o2::utils::boostHistoFromRoot_2D(hCalibInputHist_ROOT);
 
   // instance of CalibDB
   o2::emcal::CalibDB calibdb(ccdbServerPath);
 
   if (doBadChannelCalib) {
+    std::map dummymeta;
+    if (doScale) {
+      CalibExtractor.setBCMScaleFactors(calibdb.readChannelScaleFactors(1546300800001, dummymeta));
+    }
     printf("perform bad channel analysis\n");
     o2::emcal::BadChannelMap BCMap;
 
-    BCMap = CalibExtractor.calibrateBadChannels(hCalibInputHist);
-
+    if (doBCCalibWithTime) {
+      boostHisto2d_VarAxis hCalibInputHistAdd = o2::utils::boostHistoFromRoot_2D(hCalibInputHistAdd_ROOT);
+      BCMap = CalibExtractor.calibrateBadChannels(hCalibInputHist, hCalibInputHistAdd);
+    } else {
+      BCMap = CalibExtractor.calibrateBadChannels(hCalibInputHist);
+    }
     // store bad channel map in ccdb via emcal calibdb
-    std::map metadata;
-    calibdb.storeBadChannelMap(&BCMap, metadata, rangestart, rangeend);
+    if (doLocal) {
+      std::unique_ptr writer(TFile::Open(Form("bcm_%lu.root", rangestart), "RECREATE"));
+      writer->WriteObjectAny(&BCMap, "o2::emcal::BadChannelMap", "ccdb_object");
+    } else {
+      std::map metadata;
+      calibdb.storeBadChannelMap(&BCMap, metadata, rangestart, rangeend);
+    }
+
   } else {
     printf("perform time calibration analysis\n");
 
@@ -198,9 +244,14 @@ int main(int argc, char** argv)
     o2::emcal::TimeCalibrationParams TCparams;
     TCparams = CalibExtractor.calibrateTime(hCalibInputHist, timeRangeLow, timeRangeHigh);
 
-    // store parameters in ccdb via emcal calibdb
-    std::map metadata;
-    calibdb.storeTimeCalibParam(&TCparams, metadata, rangestart, rangeend);
+    if (doLocal) {
+      std::unique_ptr writer(TFile::Open(Form("timecalib_%lu.root", rangestart), "RECREATE"));
+      writer->WriteObjectAny(&TCparams, "o2::emcal::TimeCalibrationParams", "ccdb_object");
+    } else {
+      // store parameters in ccdb via emcal calibdb
+      std::map metadata;
+      calibdb.storeTimeCalibParam(&TCparams, metadata, rangestart, rangeend);
+    }
 
     if (namePathStoreLocal.find(".root") != std::string::npos) {
       TFile fLocalStorage(namePathStoreLocal.c_str(), "update");
diff --git a/Detectors/EMCAL/calibration/src/EMCALCalibExtractor.cxx b/Detectors/EMCAL/calibration/src/EMCALCalibExtractor.cxx
index 26f131e92d084..2219220d93c8e 100644
--- a/Detectors/EMCAL/calibration/src/EMCALCalibExtractor.cxx
+++ b/Detectors/EMCAL/calibration/src/EMCALCalibExtractor.cxx
@@ -15,7 +15,6 @@ namespace o2
 {
 namespace emcal
 {
-using boostHisto = boost::histogram::histogram, boost::histogram::axis::integer<>>, boost::histogram::unlimited_storage>>;
 
 //-------------------------------------------------------------------------------------------
 // This function builds the scaled hit distribution
@@ -30,8 +29,7 @@ using boostHisto = boost::histogram::histogram(100, 0, 100, "t-texp"), boost::histogram::axis::integer<>(0, mNcells, "CELL ID"));
-
+  boostHisto eSumHistoScaled = boost::histogram::make_histogram(boost::histogram::axis::regular<>(100, 0, 100, "t-texp"), boost::histogram::axis::integer<>(0, mNcells, "CELL ID"));
   // create a slice for each cell with energies ranging from emin to emax
   auto hEnergyCol = boost::histogram::make_histogram(boost::histogram::axis::regular<>(100, 0, 100., "t-texp"));
   auto hEnergyRow = boost::histogram::make_histogram(boost::histogram::axis::regular<>(250, 0, 250., "t-texp"));
@@ -39,7 +37,7 @@ boostHisto EMCALCalibExtractor::buildHitAndEnergyMeanScaled(double emin, double
   auto hEnergyScaled = boost::histogram::make_histogram(boost::histogram::axis::regular<>(100, 0, 100, "t-texp"), boost::histogram::axis::integer<>(0, mNcells, "CELL ID"));
 
   //...........................................
-  //start iterative process of scaling of cells
+  // start iterative process of scaling of cells
   //...........................................
   for (int iter = 1; iter < 5; iter++) {
     // array of vectors for calculating the mean hits per col/row
@@ -48,9 +46,8 @@ boostHisto EMCALCalibExtractor::buildHitAndEnergyMeanScaled(double emin, double
 
     for (int cellID = 0; cellID < mNcells; cellID++) {
       auto tempSlice = boost::histogram::algorithm::reduce(cellAmplitude, boost::histogram::algorithm::shrink(cellID, cellID), boost::histogram::algorithm::shrink(emin, emax));
-      auto geo = Geometry::GetInstance();
       // (0 - row, 1 - column)
-      auto position = geo->GlobalRowColFromIndex(cellID);
+      auto position = mGeometry->GlobalRowColFromIndex(cellID);
       int row = std::get<0>(position);
       int col = std::get<1>(position);
 
@@ -100,11 +97,10 @@ boostHisto EMCALCalibExtractor::buildHitAndEnergyMeanScaled(double emin, double
     auto colResult = o2::utils::fitBoostHistoWithGaus(hEnergyCol);
     double meanValCol = colResult.at(1);
 
-    //Scale each cell by the deviation of the mean of the column and the global mean
+    // Scale each cell by the deviation of the mean of the column and the global mean
     for (int iCell = 0; iCell < mNcells; iCell++) {
-      auto geo = Geometry::GetInstance();
       // (0 - row, 1 - column)
-      auto position = geo->GlobalRowColFromIndex(iCell);
+      auto position = mGeometry->GlobalRowColFromIndex(iCell);
       int col = std::get<1>(position);
       if (hEnergyCol.at(col) > 0.) {
         // will need to change the 100 depending on the number of energy bins we end up having
@@ -114,11 +110,10 @@ boostHisto EMCALCalibExtractor::buildHitAndEnergyMeanScaled(double emin, double
       }
     }
 
-    //Scale each cell by the deviation of the mean of the row and the global mean
+    // Scale each cell by the deviation of the mean of the row and the global mean
     for (int iCell = 0; iCell < mNcells; iCell++) {
-      auto geo = Geometry::GetInstance();
       // (0 - row, 1 - column)
-      auto position = geo->GlobalRowColFromIndex(iCell);
+      auto position = mGeometry->GlobalRowColFromIndex(iCell);
       int row = std::get<0>(position);
       if (hEnergyRow.at(row) > 0.) {
         // will need to change the 100 depending on the number of energy bins we end up having
@@ -158,7 +153,59 @@ boostHisto EMCALCalibExtractor::buildHitAndEnergyMeanScaled(double emin, double
 
   return eSumHistoScaled;
 }
+
+//____________________________________________
+void EMCALCalibExtractor::checkMaskSM(o2::emcal::BadChannelMap& bcm)
+{
+
+  for (unsigned int i = 0; i < mBadCellFracSM.size(); ++i) {
+    if (mGeometry->GetSMType(i) == o2::emcal::EMCALSMType::EMCAL_STANDARD) {
+      mBadCellFracSM[i] /= 1152.;
+    } else if (mGeometry->GetSMType(i) == o2::emcal::EMCALSMType::EMCAL_THIRD || mGeometry->GetSMType(i) == o2::emcal::EMCALSMType::DCAL_EXT) {
+      mBadCellFracSM[i] /= 384.;
+    } else if (mGeometry->GetSMType(i) == o2::emcal::EMCALSMType::DCAL_STANDARD) {
+      mBadCellFracSM[i] /= 768.;
+    }
+  }
+  for (unsigned int i = 0; i < mNcells; ++i) {
+    if (mBadCellFracSM[mGeometry->GetSuperModuleNumber(i)] > EMCALCalibParams::Instance().fracMaskSMFully_bc) {
+      if (bcm.getChannelStatus(i) == o2::emcal::BadChannelMap::MaskType_t::GOOD_CELL) { // only mask good cells, to keep information about dead channels
+        bcm.addBadChannel(i, o2::emcal::BadChannelMap::MaskType_t::BAD_CELL);
+      }
+    }
+  }
+}
+
+//____________________________________________
+void EMCALCalibExtractor::checkMaskFEC(o2::emcal::BadChannelMap& bcm)
+{
+  for (unsigned int iSM = 0; iSM < mBadCellFracFEC.size(); ++iSM) {
+    for (unsigned int iFEC = 0; iFEC < mBadCellFracFEC[iSM].size(); ++iFEC) {
+      mBadCellFracFEC[iSM][iFEC] /= 32.; // 32 channels per FEC
+    }
+  }
+
+  for (unsigned int i = 0; i < mNcells; ++i) {
+    if (mBadCellFracFEC[mGeometry->GetSuperModuleNumber(i)][getFECNumberInSM(i)] > EMCALCalibParams::Instance().fracMaskFECFully_bc) {
+      if (bcm.getChannelStatus(i) == o2::emcal::BadChannelMap::MaskType_t::GOOD_CELL) { // only mask good cells, to keep information about dead channels
+        bcm.addBadChannel(i, o2::emcal::BadChannelMap::MaskType_t::BAD_CELL);
+      }
+    }
+  }
+}
+
 //____________________________________________
+unsigned int EMCALCalibExtractor::getFECNumberInSM(int absCellID) const
+{
+  std::tuple RowCol = mGeometry->GlobalRowColFromIndex(absCellID);
+  std::tuple PosInSM = mGeometry->GetPositionInSupermoduleFromGlobalRowCol(std::get<0>(RowCol), std::get<1>(RowCol));
+  int iSM = std::get<0>(PosInSM);
+  int col = std::get<1>(PosInSM);
+  int row = std::get<2>(PosInSM);
+  int FECid = static_cast(row / 4.) + 12 * static_cast(col / 8.);
+  LOG(debug) << "FECid " << FECid << "   iSM " << iSM << "   row " << row << "   col " << col;
+  return FECid;
+}
 
 } // end namespace emcal
-} //end namespace o2
+} // end namespace o2
diff --git a/Detectors/EMCAL/calibration/src/EMCALCalibrationLinkDef.h b/Detectors/EMCAL/calibration/src/EMCALCalibrationLinkDef.h
index 02600dc1fcf60..0cc0451ed81b7 100644
--- a/Detectors/EMCAL/calibration/src/EMCALCalibrationLinkDef.h
+++ b/Detectors/EMCAL/calibration/src/EMCALCalibrationLinkDef.h
@@ -15,16 +15,17 @@
 #pragma link off all classes;
 #pragma link off all functions;
 
-#pragma link C++ class o2::emcal::EMCALChannelCalibrator < o2::emcal::EMCALChannelData, o2::emcal::BadChannelMap, o2::emcal::ChannelCalibInitParams> + ;
+#pragma link C++ class o2::emcal::EMCALChannelCalibrator < o2::emcal::EMCALChannelData, o2::emcal::BadChannelMap> + ;
 #pragma link C++ class o2::calibration::TimeSlot < o2::emcal::EMCALChannelData> + ;
-#pragma link C++ class o2::calibration::TimeSlotCalibration < o2::emcal::Cell, o2::emcal::EMCALChannelData> + ;
+#pragma link C++ class o2::calibration::TimeSlotCalibration < o2::emcal::EMCALChannelData> + ;
 
-#pragma link C++ class o2::emcal::EMCALChannelCalibrator < o2::emcal::EMCALTimeCalibData, o2::emcal::TimeCalibrationParams, o2::emcal::TimeCalibInitParams> + ;
+#pragma link C++ class o2::emcal::EMCALChannelCalibrator < o2::emcal::EMCALTimeCalibData, o2::emcal::TimeCalibrationParams> + ;
 #pragma link C++ class o2::calibration::TimeSlot < o2::emcal::EMCALTimeCalibData> + ;
-#pragma link C++ class o2::calibration::TimeSlotCalibration < o2::emcal::Cell, o2::emcal::EMCALTimeCalibData> + ;
+#pragma link C++ class o2::calibration::TimeSlotCalibration < o2::emcal::EMCALTimeCalibData> + ;
 
 #pragma link C++ class o2::emcal::EMCALCalibParams + ;
 #pragma link C++ class o2::conf::ConfigurableParamHelper < o2::emcal::EMCALCalibParams> + ;
 #pragma link C++ class o2::emcal::EMCDCSProcessor + ;
+#pragma link C++ class o2::emcal::PedestalProcessorData + ;
 
 #endif
diff --git a/Detectors/EMCAL/calibration/src/EMCALChannelData.cxx b/Detectors/EMCAL/calibration/src/EMCALChannelData.cxx
index 558d416e74c8a..81927ce497de6 100644
--- a/Detectors/EMCAL/calibration/src/EMCALChannelData.cxx
+++ b/Detectors/EMCAL/calibration/src/EMCALChannelData.cxx
@@ -23,6 +23,9 @@
 // #include 
 // #include 
 // #include 
+#if (defined(WITH_OPENMP) && !defined(__CLING__))
+#include 
+#endif
 
 namespace o2
 {
@@ -36,7 +39,7 @@ using boost::histogram::indexed;
 //_____________________________________________
 void EMCALChannelData::PrintStream(std::ostream& stream) const
 {
-  stream << "EMCAL Cell ID:  " << mHisto << "\n";
+  stream << "EMCAL Cell ID:  " << mHisto[0] << "\n";
 }
 //_____________________________________________
 std::ostream& operator<<(std::ostream& stream, const EMCALChannelData& emcdata)
@@ -47,14 +50,73 @@ std::ostream& operator<<(std::ostream& stream, const EMCALChannelData& emcdata)
 //_____________________________________________
 void EMCALChannelData::fill(const gsl::span data)
 {
-  //the fill function is called once per event
+  // the fill function is called once per event
   mEvents++;
-  for (auto cell : data) {
-    Double_t cellEnergy = cell.getEnergy();
-    Int_t id = cell.getTower();
-    LOG(debug) << "inserting in cell ID " << id << ": energy = " << cellEnergy;
-    mHisto(cellEnergy, id);
-    mNEntriesInHisto++;
+
+  if (data.size() == 0) {
+    return;
+  }
+
+  auto fillfunction = [this](int thread, const gsl::span& data, double minCellEnergy, double minCellEnergyTime) {
+    LOG(debug) << "filling in thread " << thread << " ncells = " << data.size();
+    auto& mCurrentHist = mHisto[thread];
+    auto& mCurrentHistTime = mHistoTime[thread];
+    unsigned int nEntries = 0; // counter set inside function increases speed compared to simply using mVecNEntriesInHisto[thread]. Added to global counter at end of function
+    for (const auto& cell : data) {
+      int id = cell.getTower();
+      double cellEnergy = cell.getEnergy();
+
+      if (mApplyGainCalib) {
+        LOG(debug) << " gain calib factor for cell " << id << " = " << mArrGainCalibFactors[id];
+        cellEnergy *= mArrGainCalibFactors[id];
+      }
+
+      if (cellEnergy < minCellEnergy) {
+        LOG(debug) << "skipping cell ID " << id << ": with energy = " << cellEnergy << " below  threshold of " << minCellEnergy;
+        continue;
+      }
+
+      LOG(debug) << "inserting in cell ID " << id << ": energy = " << cellEnergy;
+      mCurrentHist(cellEnergy, id);
+      nEntries++;
+
+      if (cellEnergy > minCellEnergyTime) {
+        double cellTime = cell.getTimeStamp();
+        LOG(debug) << "inserting in cell ID " << id << ": time = " << cellTime;
+        mCurrentHistTime(cellTime, id);
+      }
+    }
+    mVecNEntriesInHisto[thread] += nEntries;
+  };
+
+  std::vector> ranges(mNThreads);
+  auto size_per_thread = static_cast(std::ceil((static_cast(data.size()) / mNThreads)));
+  unsigned int currentfirst = 0;
+  for (int ithread = 0; ithread < mNThreads; ithread++) {
+    unsigned int nelements = std::min(size_per_thread, static_cast(data.size() - 1 - currentfirst));
+    ranges[ithread] = data.subspan(currentfirst, nelements);
+    currentfirst += nelements;
+  }
+
+  double minCellEnergy = o2::emcal::EMCALCalibParams::Instance().minCellEnergy_bc;
+  double minCellEnergyTime = o2::emcal::EMCALCalibParams::Instance().minCellEnergyTime_bc;
+
+#if (defined(WITH_OPENMP) && !defined(__CLING__))
+  LOG(debug) << "Number of threads that will be used = " << mNThreads;
+#pragma omp parallel for num_threads(mNThreads)
+#else
+  LOG(debug) << "OPEN MP will not be used for the bad channel calibration";
+#endif
+  for (int ithread = 0; ithread < mNThreads; ithread++) {
+    fillfunction(ithread, ranges[ithread], minCellEnergy, minCellEnergyTime);
+  }
+
+  // only sum up entries if needed
+  if (!o2::emcal::EMCALCalibParams::Instance().useNEventsForCalib_bc) {
+    for (auto& nEntr : mVecNEntriesInHisto) {
+      mNEntriesInHisto += nEntr;
+      nEntr = 0;
+    }
   }
 }
 //_____________________________________________
@@ -63,11 +125,12 @@ void EMCALChannelData::print()
   LOG(debug) << *this;
 }
 //_____________________________________________
-void EMCALChannelData::merge(const EMCALChannelData* prev)
+void EMCALChannelData::merge(EMCALChannelData* prev)
 {
   mEvents += prev->getNEvents();
   mNEntriesInHisto += prev->getNEntriesInHisto();
-  mHisto += prev->getHisto();
+  mHisto[0] += prev->getHisto();
+  mHistoTime[0] += prev->getHisto();
 }
 
 //_____________________________________________
@@ -75,13 +138,16 @@ bool EMCALChannelData::hasEnoughData() const
 {
   bool enough = false;
 
-  LOG(debug) << "mNEntriesInHisto: " << mNEntriesInHisto << " needed: " << EMCALCalibParams::Instance().minNEntries << "  mEvents = " << mEvents;
-  // use enrties in histogram for calibration
-  if (!EMCALCalibParams::Instance().useNEventsForCalib && mNEntriesInHisto > EMCALCalibParams::Instance().minNEntries) {
-    enough = true;
+  LOG(debug) << "mNEntriesInHisto: " << mNEntriesInHisto * mNThreads << " needed: " << EMCALCalibParams::Instance().minNEntries_bc << "  mEvents = " << mEvents;
+  // use entries in histogram for calibration
+  if (!EMCALCalibParams::Instance().useNEventsForCalib_bc) {
+    long unsigned int nEntries = 0;
+    if (mNEntriesInHisto > EMCALCalibParams::Instance().minNEntries_bc) {
+      enough = true;
+    }
   }
   // use number of events (from emcal trigger record) for calibration
-  if (EMCALCalibParams::Instance().useNEventsForCalib && mEvents > EMCALCalibParams::Instance().minNEvents) {
+  if (EMCALCalibParams::Instance().useNEventsForCalib_bc && mEvents > EMCALCalibParams::Instance().minNEvents_bc) {
     enough = true;
   }
 
@@ -91,7 +157,7 @@ bool EMCALChannelData::hasEnoughData() const
 //_____________________________________________
 void EMCALChannelData::analyzeSlot()
 {
-  mOutputBCM = mCalibExtractor->calibrateBadChannels(mEsumHisto);
+  mOutputBCM = mCalibExtractor->calibrateBadChannels(mEsumHisto, getHistoTime());
 }
 //____________________________________________
 
diff --git a/Detectors/EMCAL/calibration/src/EMCALPedestalHelper.cxx b/Detectors/EMCAL/calibration/src/EMCALPedestalHelper.cxx
new file mode 100644
index 0000000000000..38a6e386c9ca6
--- /dev/null
+++ b/Detectors/EMCAL/calibration/src/EMCALPedestalHelper.cxx
@@ -0,0 +1,199 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "EMCALCalibration/EMCALPedestalHelper.h"
+using namespace o2::emcal;
+
+std::vector EMCALPedestalHelper::createPedestalInstruction(const Pedestal& obj, const int runNum)
+{
+
+  setZero();
+
+  o2::emcal::Geometry* geo = o2::emcal::Geometry::GetInstanceFromRunNumber(300000);
+  o2::emcal::MappingHandler mapper;
+
+  for (int itower = 0; itower < 17664; itower++) {
+    auto [ddl, row, col] = geo->getOnlineID(itower);
+    auto [sm, mod, iphi, ieta] = geo->GetCellIndex(itower);
+    const auto& mapping = mapper.getMappingForDDL(ddl);
+    int ircu = ddl % 2;
+    auto addressLG = mapping.getHardwareAddress(row, col, o2::emcal::ChannelType_t::LOW_GAIN),
+         addressHG = mapping.getHardwareAddress(row, col, o2::emcal::ChannelType_t::HIGH_GAIN);
+    auto fecLG = o2::emcal::Channel::getFecIndexFromHwAddress(addressLG),
+         fecHG = o2::emcal::Channel::getFecIndexFromHwAddress(addressHG),
+         branchLG = o2::emcal::Channel::getBranchIndexFromHwAddress(addressLG),
+         branchHG = o2::emcal::Channel::getBranchIndexFromHwAddress(addressHG),
+         chipLG = o2::emcal::Channel::getAltroIndexFromHwAddress(addressLG),
+         chipHG = o2::emcal::Channel::getAltroIndexFromHwAddress(addressHG),
+         channelLG = o2::emcal::Channel::getChannelIndexFromHwAddress(addressLG),
+         channelHG = o2::emcal::Channel::getChannelIndexFromHwAddress(addressHG);
+    fMeanPed[sm][ircu][branchHG][fecHG][chipHG][channelHG] = obj.getPedestalValue(itower, false, false);
+    fMeanPed[sm][ircu][branchLG][fecLG][chipLG][channelLG] = obj.getPedestalValue(itower, true, false);
+  }
+
+  for (int iledmon = 0; iledmon < 480; iledmon++) {
+    int sm = iledmon / 24,
+        col = iledmon % 24,
+        ircu = 0, // LEDMONS always on RCU 0
+      iddl = 2 * sm + ircu;
+    const auto& mapping = mapper.getMappingForDDL(iddl);
+    auto addressLG = mapping.getHardwareAddress(0, col, o2::emcal::ChannelType_t::LEDMON),
+         addressHG = mapping.getHardwareAddress(1, col, o2::emcal::ChannelType_t::LEDMON);
+    auto fecLG = o2::emcal::Channel::getFecIndexFromHwAddress(addressLG),
+         fecHG = o2::emcal::Channel::getFecIndexFromHwAddress(addressHG),
+         branchLG = o2::emcal::Channel::getBranchIndexFromHwAddress(addressLG),
+         branchHG = o2::emcal::Channel::getBranchIndexFromHwAddress(addressHG),
+         chipLG = o2::emcal::Channel::getAltroIndexFromHwAddress(addressLG),
+         chipHG = o2::emcal::Channel::getAltroIndexFromHwAddress(addressHG),
+         channelLG = o2::emcal::Channel::getChannelIndexFromHwAddress(addressLG),
+         channelHG = o2::emcal::Channel::getChannelIndexFromHwAddress(addressHG);
+    fMeanPed[sm][ircu][branchHG][fecHG][chipHG][channelHG] = obj.getPedestalValue(iledmon, false, true);
+    fMeanPed[sm][ircu][branchLG][fecLG][chipLG][channelLG] = obj.getPedestalValue(iledmon, true, true);
+  }
+
+  return createInstructionString(runNum);
+}
+
+void EMCALPedestalHelper::setZero()
+{
+  for (int ism = 0; ism < kNSM; ism++) {
+    for (int ircu = 0; ircu < kNRCU; ircu++) {
+      for (int ibranch = 0; ibranch < kNBranch; ibranch++) {
+        for (int ifec = 0; ifec < kNFEC; ifec++) {
+          for (int ichip = 0; ichip < kNChip; ichip++) {
+            for (int ichan = 0; ichan < kNChan; ichan++) {
+              fMeanPed[ism][ircu][ibranch][ifec][ichip][ichan] = 0;
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+std::vector EMCALPedestalHelper::createInstructionString(const int runNum)
+{
+  std::stringstream fout;
+
+  if (runNum > 0) {
+    fout << runNum << std::endl;
+  }
+
+  unsigned int lineValue = 0;
+
+  const unsigned int FECheaderCode = 0xC0000000;
+  //  const unsigned int FECwordCode   = 0x80000000;
+  const unsigned int FEClineCode = 0x40000000;
+
+  const unsigned int TrailerLineCode = 0xFFFFFFFF;
+
+  short iSM = 0;
+  short iRCU = 0;
+  short ibranch = 0;
+  short iFEC = 0;
+  short ichip = 0;
+  short ichan = 0;
+  short Ped = 0;
+  short iDTC = 0;
+
+  for (iSM = 0; iSM < kNSM; iSM++) {
+    int iside = iSM % 2;
+    int isect = iSM / 2;
+    if (iSM > 11) {
+      isect += 3; // skip non-installed sectors
+    }
+
+    std::bitset activeDTC;
+    for (iDTC = 0; iDTC < kNDTC; iDTC++) {
+      if (iDTC == 10 || iDTC == 20 || iDTC == 30) { // skip TRU
+        activeDTC[iDTC] = 0;
+      } else {
+        if (iSM < 10) { // not special third SMs or DCal SMs
+          activeDTC[iDTC] = 1;
+        } else {
+          if (iSM == 10 || iSM == 19) { // SMA5 or SMC12
+            if (iDTC < 14) {
+              activeDTC[iDTC] = 1;
+            } else {
+              activeDTC[iDTC] = 0;
+            }
+          } else if (iSM == 11 || iSM == 18) { // SMC5 or SMA12
+            if (iDTC == 0 || iDTC >= 27) {
+              activeDTC[iDTC] = 1;
+            } else {
+              activeDTC[iDTC] = 0;
+            }
+          } else {
+            // DCal... no FECs in  9,11-13, 23-26, 36-39
+            if ((iDTC >= 9 && iDTC <= 13) || (iDTC >= 23 && iDTC <= 26) ||
+                (iDTC >= 36 && iDTC <= 39)) {
+              activeDTC[iDTC] = 0;
+            } else {
+              activeDTC[iDTC] = 1;
+            }
+          } // DCal
+        }   // non-EMCal
+      }     // non-TRU
+    }
+
+    // OK, let's generate the files for all active FECs/DTCs
+    for (iDTC = 0; iDTC < kNDTC; iDTC++) {
+      if (activeDTC[iDTC] == 0) {
+        continue;
+      }
+
+      lineValue = FECheaderCode | isect << 9 | iside << 8 | iDTC;
+      fout << lineValue << std::endl;
+
+      iRCU = iDTC / 20;
+      ibranch = (iDTC % 20) / 10;
+      iFEC = iDTC % 10;
+      int ipos = iFEC + 10 * ibranch;
+
+      int dtcselUpper = 0;
+      int dtcselLower = 0;
+      if (iRCU == 0) {
+        dtcselLower = (1 << ipos);
+      } else { // crate == 1
+        dtcselUpper = (1 << ipos);
+      }
+
+      for (ichip = 0; ichip < kNChip; ichip++) { // ALTRO 0,2,3,4
+        if (ichip != 1) {
+          for (ichan = 0; ichan < kNChan; ichan++) {
+            if (iFEC != 0 || (ichan < 8 || ichan > 11)) {
+              Ped = fMeanPed[iSM][iRCU][ibranch][iFEC][ichip][ichan];
+              int writeAddr = (ichip << 4) | ichan;
+              lineValue = FEClineCode | (writeAddr << 12) | Ped;
+              fout << lineValue << std::endl;
+            }
+          }
+        }
+      } // chip
+
+    } // iDTC
+  }   // iSM
+
+  if (runNum > 0) {
+    fout << TrailerLineCode << std::endl;
+  }
+
+  const std::string instructionString(fout.str());
+  std::vector output(instructionString.begin(), instructionString.end());
+  return output;
+}
+
+void EMCALPedestalHelper::dumpInstructions(const std::string_view filename, const gsl::span& data)
+{
+  std::ofstream fout(filename.data());
+  fout << data.data();
+  fout.close();
+}
\ No newline at end of file
diff --git a/Detectors/EMCAL/calibration/src/EMCALTempCalibExtractor.cxx b/Detectors/EMCAL/calibration/src/EMCALTempCalibExtractor.cxx
new file mode 100644
index 0000000000000..02e25696f161d
--- /dev/null
+++ b/Detectors/EMCAL/calibration/src/EMCALTempCalibExtractor.cxx
@@ -0,0 +1,127 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "EMCALCalibration/EMCALTempCalibExtractor.h"
+#include "EMCALCalib/CalibDB.h"
+#include "CCDB/CcdbApi.h"
+#include "CCDB/BasicCCDBManager.h"
+#include 
+
+namespace o2
+{
+namespace emcal
+{
+
+void EMCALTempCalibExtractor::InitializeFromCCDB(std::string path, uint64_t timestamp)
+{
+
+  auto& ccdbMgr = o2::ccdb::BasicCCDBManager::instance();
+  uint64_t maxRunNr = 1000000;
+  if (timestamp < maxRunNr) {
+    LOG(info) << "assuming input is run " << timestamp << " will convert it to timstamp";
+    auto [sor, eor] = ccdbMgr.getRunDuration(timestamp);
+    uint64_t sixtySec = 60000;
+    timestamp = eor - sixtySec; // safety margin of 1min at EOR
+    LOG(info) << "set timestamp to " << timestamp;
+  }
+
+  o2::emcal::CalibDB calibdb("http://alice-ccdb.cern.ch");
+  std::map metadata;
+  auto tempSensorData = calibdb.readTemperatureSensorData(timestamp, metadata);
+
+  // also obtain cell dependent correction factors
+  TempCalibrationParams* params = ccdbMgr.getForTimeStamp(path, timestamp);
+
+  std::map mapSMTemperature;
+  for (unsigned short i = 0; i < mNCells; ++i) {
+    const unsigned short iSM = mGeometry->GetSuperModuleNumber(i);
+    if (mapSMTemperature.count(iSM) == 0) {
+      mapSMTemperature[iSM] = getTemperatureForSM(iSM, tempSensorData);
+    }
+    float corrFac = params->getTempCalibParamA0(i) + params->getTempCalibParamSlope(i) * mapSMTemperature[iSM];
+    mGainCalibFactors[i] = corrFac;
+  }
+}
+
+float EMCALTempCalibExtractor::getTemperatureForSM(const unsigned short iSM, o2::emcal::ElmbData* ElmbData) const
+{
+  if (iSM < 0 || iSM > 20) {
+    LOG(error) << "SM " << iSM << "does not exist!"; // could be replaced with a proper exception
+    return 0.;
+  }
+  std::vector vecSensorID = getSensorsForSM(iSM);
+
+  // Obtain temperature for these sensors
+  std::vector vecTemperature;
+  for (const auto& iSensor : vecSensorID) {
+    float temp = ElmbData->getMean(iSensor);
+    if (temp < mAcceptedTempRange[0] || temp > mAcceptedTempRange[1]) {
+      continue;
+    }
+    vecTemperature.push_back(temp);
+  }
+
+  const unsigned int nEntries = vecTemperature.size();
+  if (nEntries == 0) {
+    LOG(warning) << "No sensor data between " << mAcceptedTempRange[0] << " and " << mAcceptedTempRange[1] << "degree found... for SM " << iSM << "  Setting to default 20 degree";
+    return 20.; //
+  }
+
+  // get median energy
+  float tempSM = 0.;
+  if (mUseMedian) {
+    std::sort(vecTemperature.begin(), vecTemperature.end());
+    if (nEntries % 2 == 0) {
+      // even number of elements: average the two middle ones
+      tempSM = (vecTemperature[nEntries / 2 - 1] + vecTemperature[nEntries / 2]) / 2.0;
+    } else {
+      // odd number of elements: return the middle one
+      tempSM = vecTemperature[nEntries / 2];
+    }
+  } else { // use Mean temperature
+    float sum = std::accumulate(vecTemperature.begin(), vecTemperature.end(), 0.0);
+    tempSM = sum / vecTemperature.size();
+  }
+  return tempSM;
+}
+
+float EMCALTempCalibExtractor::getGainCalibFactor(const unsigned short cellID) const
+{
+  if (cellID >= mNCells) {
+    LOG(error) << "cell ID" << cellID << " does not exist";
+    return 1.;
+  }
+  return mGainCalibFactors[cellID];
+}
+
+std::vector EMCALTempCalibExtractor::getSensorsForSM(const unsigned short iSM) const
+{
+  unsigned short nSensors = 8;
+  if (iSM == 10 || iSM == 11 || iSM == 18 || iSM == 19) { // 1/3 SM of EMCal only have 4 sensors
+    nSensors = 4;
+  }
+
+  std::vector vecSensorID;
+  for (unsigned short iELMBSensor = iSM * 8; iELMBSensor < iSM * 8 + nSensors; iELMBSensor++) {
+    vecSensorID.push_back(iELMBSensor);
+  }
+  return vecSensorID;
+}
+
+void EMCALTempCalibExtractor::setAcceptedEnergyRange(float low, float high)
+{
+  mAcceptedTempRange[0] = low;
+  mAcceptedTempRange[1] = high;
+}
+
+} // namespace emcal
+
+} // namespace o2
\ No newline at end of file
diff --git a/Detectors/EMCAL/calibration/src/EMCALTimeCalibData.cxx b/Detectors/EMCAL/calibration/src/EMCALTimeCalibData.cxx
index bdc4740c41f71..c75bc1f1ac48a 100644
--- a/Detectors/EMCAL/calibration/src/EMCALTimeCalibData.cxx
+++ b/Detectors/EMCAL/calibration/src/EMCALTimeCalibData.cxx
@@ -36,7 +36,7 @@ using boost::histogram::indexed;
 //_____________________________________________
 void EMCALTimeCalibData::PrintStream(std::ostream& stream) const
 {
-  stream << "EMCAL Cell ID:  " << mTimeHisto << "\n";
+  stream << "EMCAL Cell ID:  " << mTimeHisto[0] << "\n";
 }
 //_____________________________________________
 void EMCALTimeCalibData::print()
@@ -50,43 +50,87 @@ std::ostream& operator<<(std::ostream& stream, const EMCALTimeCalibData& emcdata
   return stream;
 }
 //_____________________________________________
-void EMCALTimeCalibData::merge(const EMCALTimeCalibData* prev)
+void EMCALTimeCalibData::merge(EMCALTimeCalibData* prev)
 {
   mEvents += prev->getNEvents();
   mNEntriesInHisto += prev->getNEntriesInHisto();
-  mTimeHisto += prev->getHisto();
+  mTimeHisto[0] += prev->getHisto();
 }
 //_____________________________________________
 bool EMCALTimeCalibData::hasEnoughData() const
 {
   bool enough = false;
 
-  LOG(debug) << "mNEntriesInHisto: " << mNEntriesInHisto << " needed: " << EMCALCalibParams::Instance().minNEntries << "  mEvents = " << mEvents;
+  LOG(debug) << "mNEntriesInHisto: " << mNEntriesInHisto << " needed: " << EMCALCalibParams::Instance().minNEntries_tc << "  mEvents = " << mEvents;
   // use enrties in histogram for calibration
-  if (!EMCALCalibParams::Instance().useNEventsForCalib && mNEntriesInHisto > EMCALCalibParams::Instance().minNEntries) {
+  if (!EMCALCalibParams::Instance().useNEventsForCalib_tc && mNEntriesInHisto > EMCALCalibParams::Instance().minNEntries_tc) {
     enough = true;
   }
   // use number of events (from emcal trigger record) for calibration
-  if (EMCALCalibParams::Instance().useNEventsForCalib && mEvents > EMCALCalibParams::Instance().minNEvents) {
+  if (EMCALCalibParams::Instance().useNEventsForCalib_tc && mEvents > EMCALCalibParams::Instance().minNEvents_tc) {
     enough = true;
   }
 
   return enough;
 }
+
 //_____________________________________________
 void EMCALTimeCalibData::fill(const gsl::span data)
 {
   // the fill function is called once per event
   mEvents++;
 
-  for (auto cell : data) {
-    double cellEnergy = cell.getEnergy();
-    double cellTime = cell.getTimeStamp();
-    int id = cell.getTower();
-    if (cellEnergy > EMCALCalibParams::Instance().minCellEnergyForTimeCalib) {
-      LOG(debug) << "inserting in cell ID " << id << ": cellTime = " << cellTime;
-      mTimeHisto(cellTime, id);
-      mNEntriesInHisto++;
+  if (data.size() == 0) {
+    return;
+  }
+  auto fillfunction = [this](int thread, const gsl::span data, double minCellEnergy) {
+    LOG(debug) << "filling in thread " << thread << " ncells = " << data.size();
+    auto& mCurrentHist = mTimeHisto[thread];
+    unsigned int nEntries = 0;
+    for (auto cell : data) {
+      double cellEnergy = cell.getEnergy();
+      double cellTime = cell.getTimeStamp();
+      int id = cell.getTower();
+      if (mApplyGainCalib) {
+        LOG(debug) << " gain calib factor for cell " << id << " = " << mGainCalibFactors->getGainCalibFactors(id);
+        cellEnergy *= mGainCalibFactors->getGainCalibFactors(id);
+      }
+      if (cellEnergy > minCellEnergy) {
+        LOG(debug) << "inserting in cell ID " << id << ": cellTime = " << cellTime;
+        mCurrentHist(cellTime, id);
+        nEntries++;
+      }
+    }
+    mVecNEntriesInHisto[thread] += nEntries;
+  };
+
+  std::vector> ranges(mNThreads);
+  auto size_per_thread = static_cast(std::ceil((static_cast(data.size()) / mNThreads)));
+  unsigned int currentfirst = 0;
+  for (int ithread = 0; ithread < mNThreads; ithread++) {
+    unsigned int nelements = std::min(size_per_thread, static_cast(data.size() - 1 - currentfirst));
+    ranges[ithread] = data.subspan(currentfirst, nelements);
+    currentfirst += nelements;
+    LOG(debug) << "currentfirst " << currentfirst << "  nelements " << nelements;
+  }
+
+  double minCellEnergy = EMCALCalibParams::Instance().minCellEnergy_tc;
+
+#if (defined(WITH_OPENMP) && !defined(__CLING__))
+  LOG(debug) << "Number of threads that will be used = " << mNThreads;
+#pragma omp parallel for num_threads(mNThreads)
+#else
+  LOG(debug) << "OPEN MP will not be used for the bad channel calibration";
+#endif
+  for (int ithread = 0; ithread < mNThreads; ithread++) {
+    fillfunction(ithread, ranges[ithread], minCellEnergy);
+  }
+
+  // only sum up entries if needed
+  if (!o2::emcal::EMCALCalibParams::Instance().useNEventsForCalib_tc) {
+    for (auto& nEntr : mVecNEntriesInHisto) {
+      mNEntriesInHisto += nEntr;
+      nEntr = 0;
     }
   }
 }
diff --git a/Detectors/EMCAL/calibration/src/EMCDCSProcessor.cxx b/Detectors/EMCAL/calibration/src/EMCDCSProcessor.cxx
index 210ef349f66f1..82e1488d4e1a4 100644
--- a/Detectors/EMCAL/calibration/src/EMCDCSProcessor.cxx
+++ b/Detectors/EMCAL/calibration/src/EMCDCSProcessor.cxx
@@ -142,7 +142,9 @@ void EMCDCSProcessor::FillElmbDP(const DPCOM& dpcom)
 
   if ((index = alias.find("EMC_PT")) != std::string::npos) {
     std::sscanf(alias.data(), "EMC_PT_%d.Temperature", &iPT);
-    LOG(debug) << "alias=" << alias.data() << ":  iPT=" << iPT << ", val=" << val;
+    if (mVerbose) {
+      LOG(debug) << "alias=" << alias.data() << ":  iPT=" << iPT << ", val=" << val;
+    }
 
     if (iPT < 0 || iPT > 159) {
       LOG(error) << "Wrong Sensor Index iPT=" << iPT << " for DP " << alias.data();
diff --git a/Detectors/EMCAL/calibration/src/PedestalCalibDevice.cxx b/Detectors/EMCAL/calibration/src/PedestalCalibDevice.cxx
new file mode 100644
index 0000000000000..2f4c6ee204a31
--- /dev/null
+++ b/Detectors/EMCAL/calibration/src/PedestalCalibDevice.cxx
@@ -0,0 +1,116 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "CommonUtils/VerbosityConfig.h"
+#include "DetectorsRaw/RDHUtils.h"
+#include "EMCALBase/Geometry.h"
+#include "Framework/TimingInfo.h"
+#include "EMCALCalibration/PedestalCalibDevice.h"
+#include "EMCALReconstruction/AltroDecoder.h"
+#include "EMCALReconstruction/RawReaderMemory.h"
+#include "DetectorsCalibration/Utils.h"
+#include "EMCALCalib/CalibDB.h"
+#include "Framework/ConcreteDataMatcher.h"
+#include "CommonUtils/MemFileHelper.h"
+
+#include 
+
+using namespace o2::emcal;
+
+void PedestalCalibDevice::init(o2::framework::InitContext& ctx)
+{
+  LOG(debug) << "[EMCALPedestalCalibDevice - init] Initialize converter ";
+  if (!mGeometry) {
+    mGeometry = Geometry::GetInstanceFromRunNumber(300000);
+  }
+  if (!mGeometry) {
+    LOG(error) << "Failure accessing geometry";
+  }
+
+  resetStartTS();
+}
+
+void PedestalCalibDevice::run(o2::framework::ProcessingContext& ctx)
+{
+  if (!mRun) {
+    const auto& tinfo = ctx.services().get();
+    if (tinfo.runNumber != 0) {
+      mRun = tinfo.runNumber;
+    }
+  }
+
+  constexpr auto originEMC = o2::header::gDataOriginEMC;
+  auto data = ctx.inputs().get(getPedDataBinding());
+  LOG(debug) << "adding pedestal data";
+  mPedestalData += data;
+}
+
+//________________________________________________________________
+void PedestalCalibDevice::sendData(o2::framework::EndOfStreamContext& ec, const Pedestal& data) const
+{
+  LOG(info) << "sending pedestal data";
+  constexpr auto originEMC = o2::header::gDataOriginEMC;
+
+  std::map md;
+  auto clName = o2::utils::MemFileHelper::getClassName(data);
+  auto flName = o2::ccdb::CcdbApi::generateFileName(clName);
+  o2::ccdb::CcdbObjectInfo objInfo(o2::emcal::CalibDB::getCDBPathChannelPedestals(), clName, flName, md, mStartTS, o2::ccdb::CcdbObjectInfo::INFINITE_TIMESTAMP, true);
+
+  auto image = o2::ccdb::CcdbApi::createObjectImage(&data, &objInfo);
+
+  ec.outputs().snapshot(o2::framework::Output{o2::calibration::Utils::gDataOriginCDBPayload, "EMC_PEDCALIB", 0}, *image.get());
+  ec.outputs().snapshot(o2::framework::Output{o2::calibration::Utils::gDataOriginCDBWrapper, "EMC_PEDCALIB", 0}, objInfo);
+
+  // the following goes to the DCS ccdb
+  EMCALPedestalHelper helper;
+  std::vector vecPedData = helper.createPedestalInstruction(data, mAddRunNumber ? mRun : -1);
+  if (mDumpToFile) {
+    helper.dumpInstructions("EMCAL-Pedestals.txt", vecPedData);
+  }
+
+  auto clNameDCS = o2::utils::MemFileHelper::getClassName(vecPedData);
+  auto flNameDCS = o2::ccdb::CcdbApi::generateFileName(clNameDCS);
+  o2::ccdb::CcdbObjectInfo objInfoDCS("EMC/Calib/PDData", clNameDCS, flNameDCS, md, mStartTS, o2::ccdb::CcdbObjectInfo::INFINITE_TIMESTAMP, true);
+  auto imageDCS = o2::ccdb::CcdbApi::createObjectImage(&vecPedData, &objInfoDCS);
+
+  ec.outputs().snapshot(o2::framework::Output{o2::calibration::Utils::gDataOriginCDBPayload, "EMC_PEDCALIBSTR", 1}, *imageDCS.get());
+  ec.outputs().snapshot(o2::framework::Output{o2::calibration::Utils::gDataOriginCDBWrapper, "EMC_PEDCALIBSTR", 1}, objInfoDCS);
+}
+
+//________________________________________________________________
+void PedestalCalibDevice::endOfStream(o2::framework::EndOfStreamContext& ec)
+{
+
+  Pedestal pedObj = mCalibExtractor.extractPedestals(mPedestalData);
+  sendData(ec, pedObj);
+  // reset Timestamp (probably not needed as program will probably be terminated)
+  resetStartTS();
+}
+
+o2::framework::DataProcessorSpec o2::emcal::getPedestalCalibDevice(bool dumpToFile, bool addRunNum)
+{
+
+  std::vector inputs;
+  inputs.emplace_back(o2::emcal::PedestalCalibDevice::getPedDataBinding(), o2::header::gDataOriginEMC, "PEDDATA", 0, o2::framework::Lifetime::Timeframe);
+  std::vector outputs;
+  outputs.emplace_back(o2::framework::ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBPayload, "EMC_PEDCALIB"}, o2::framework::Lifetime::Sporadic);
+  outputs.emplace_back(o2::framework::ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBWrapper, "EMC_PEDCALIB"}, o2::framework::Lifetime::Sporadic);
+
+  outputs.emplace_back(o2::framework::ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBPayload, "EMC_PEDCALIBSTR"}, o2::framework::Lifetime::Sporadic);
+  outputs.emplace_back(o2::framework::ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBWrapper, "EMC_PEDCALIBSTR"}, o2::framework::Lifetime::Sporadic);
+
+  return o2::framework::DataProcessorSpec{
+    "PedestalCalibrator",
+    inputs,
+    outputs,
+    o2::framework::AlgorithmSpec{o2::framework::adaptFromTask(dumpToFile, addRunNum)},
+    o2::framework::Options{}};
+}
\ No newline at end of file
diff --git a/Detectors/EMCAL/calibration/src/PedestalProcessorData.cxx b/Detectors/EMCAL/calibration/src/PedestalProcessorData.cxx
new file mode 100644
index 0000000000000..33ef3319b2570
--- /dev/null
+++ b/Detectors/EMCAL/calibration/src/PedestalProcessorData.cxx
@@ -0,0 +1,122 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include 
+#include "EMCALCalibration/PedestalProcessorData.h"
+
+using namespace o2::emcal;
+
+PedestalProcessorData& PedestalProcessorData::operator+=(const PedestalProcessorData& other)
+{
+  for (std::size_t ichan{0}; ichan < mDataFECHG.size(); ++ichan) {
+    mDataFECHG[ichan] += other.mDataFECHG[ichan];
+    mDataFECLG[ichan] += other.mDataFECLG[ichan];
+  }
+  for (std::size_t ichan{0}; ichan < mDataLEDMONHG.size(); ++ichan) {
+    mDataLEDMONHG[ichan] += other.mDataLEDMONHG[ichan];
+    mDataLEDMONLG[ichan] += other.mDataLEDMONLG[ichan];
+  }
+  return *this;
+}
+
+void PedestalProcessorData::fillADC(unsigned short adc, unsigned short tower, bool lowGain, bool LEDMON)
+{
+  auto maxentries = LEDMON ? mDataLEDMONHG.size() : mDataFECHG.size();
+  if (tower >= maxentries) {
+    throw ChannelIndexException(tower, maxentries);
+  }
+  if (LEDMON) {
+    if (lowGain) {
+      mDataLEDMONLG[tower].add(adc);
+    } else {
+      mDataLEDMONHG[tower].add(adc);
+    }
+  } else {
+    if (lowGain) {
+      mDataFECLG[tower].add(adc);
+    } else {
+      mDataFECHG[tower].add(adc);
+    }
+  }
+}
+
+PedestalProcessorData::PedestalValue PedestalProcessorData::getValue(unsigned short tower, bool lowGain, bool LEDMON) const
+{
+  auto maxentries = LEDMON ? mDataLEDMONHG.size() : mDataFECHG.size();
+  if (tower >= maxentries) {
+    throw ChannelIndexException(tower, maxentries);
+  }
+  float mean, rms;
+  if (LEDMON) {
+    if (lowGain) {
+      return mDataLEDMONLG[tower].getMeanRMS2();
+    } else {
+      return mDataLEDMONHG[tower].getMeanRMS2();
+    }
+  } else {
+    if (lowGain) {
+      return mDataFECLG[tower].getMeanRMS2();
+    } else {
+      return mDataFECHG[tower].getMeanRMS2();
+    }
+  }
+}
+
+int PedestalProcessorData::getEntriesForChannel(unsigned short tower, bool lowGain, bool LEDMON) const
+{
+  auto maxentries = LEDMON ? mDataLEDMONHG.size() : mDataFECHG.size();
+  if (tower >= maxentries) {
+    throw ChannelIndexException(tower, maxentries);
+  }
+  float mean, rms;
+  if (LEDMON) {
+    if (lowGain) {
+      return mDataLEDMONLG[tower].n;
+    } else {
+      return mDataLEDMONHG[tower].n;
+    }
+  } else {
+    if (lowGain) {
+      return mDataFECLG[tower].n;
+    } else {
+      return mDataFECHG[tower].n;
+    }
+  }
+}
+
+void PedestalProcessorData::reset()
+{
+  for (auto& acc : mDataFECHG) {
+    acc.clear();
+  }
+  for (auto& acc : mDataFECLG) {
+    acc.clear();
+  }
+  for (auto& acc : mDataLEDMONHG) {
+    acc.clear();
+  }
+  for (auto& acc : mDataLEDMONLG) {
+    acc.clear();
+  }
+}
+
+PedestalProcessorData o2::emcal::operator+(const PedestalProcessorData& lhs, const PedestalProcessorData& rhs)
+{
+  PedestalProcessorData result(lhs);
+  result += rhs;
+  return result;
+}
+
+std::ostream& o2::emcal::operator<<(std::ostream& stream, const PedestalProcessorData::ChannelIndexException& ex)
+{
+  stream << ex.what();
+  return stream;
+}
\ No newline at end of file
diff --git a/Detectors/EMCAL/calibration/src/PedestalProcessorDevice.cxx b/Detectors/EMCAL/calibration/src/PedestalProcessorDevice.cxx
new file mode 100644
index 0000000000000..90ddba0ffb75a
--- /dev/null
+++ b/Detectors/EMCAL/calibration/src/PedestalProcessorDevice.cxx
@@ -0,0 +1,228 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "CommonUtils/VerbosityConfig.h"
+#include "DetectorsRaw/RDHUtils.h"
+#include "EMCALBase/Geometry.h"
+#include "EMCALCalibration/PedestalProcessorDevice.h"
+#include "EMCALReconstruction/AltroDecoder.h"
+#include "EMCALReconstruction/RawReaderMemory.h"
+#include "Framework/ConcreteDataMatcher.h"
+#include "Framework/InputRecordWalker.h"
+
+#include 
+
+using namespace o2::emcal;
+
+void PedestalProcessorDevice::init(o2::framework::InitContext& ctx)
+{
+  LOG(debug) << "[EMCALRawToCellConverter - init] Initialize converter ";
+  if (!mGeometry) {
+    mGeometry = Geometry::GetInstanceFromRunNumber(300000);
+  }
+  if (!mGeometry) {
+    LOG(error) << "Failure accessing geometry";
+  }
+
+  if (!mMapper) {
+    mMapper = std::unique_ptr(new o2::emcal::MappingHandler);
+  }
+  if (!mMapper) {
+    LOG(error) << "Failed to initialize mapper";
+  }
+}
+
+void PedestalProcessorDevice::run(o2::framework::ProcessingContext& ctx)
+{
+  constexpr auto originEMC = o2::header::gDataOriginEMC;
+  constexpr auto descRaw = o2::header::gDataDescriptionRawData;
+
+  mPedestalData.reset();
+
+  if (isLostTimeframe(ctx)) {
+    sendData(ctx, mPedestalData);
+    return;
+  }
+
+  std::vector filter{{"filter", framework::ConcreteDataTypeMatcher(originEMC, descRaw)}};
+  for (const auto& rawData : framework::InputRecordWalker(ctx.inputs(), filter)) {
+    // Skip SOX headers
+    auto rdhblock = reinterpret_cast(rawData.payload);
+    if (o2::raw::RDHUtils::getHeaderSize(rdhblock) == static_cast(o2::framework::DataRefUtils::getPayloadSize(rawData))) {
+      continue;
+    }
+
+    o2::emcal::RawReaderMemory rawreader(framework::DataRefUtils::as(rawData));
+    rawreader.setRangeSRUDDLs(0, 39);
+
+    // loop over all the DMA pages
+    while (rawreader.hasNext()) {
+      try {
+        rawreader.next();
+      } catch (RawDecodingError& e) {
+        LOG(error) << e.what();
+        if (e.getErrorType() == RawDecodingError::ErrorType_t::HEADER_DECODING || e.getErrorType() == RawDecodingError::ErrorType_t::HEADER_INVALID) {
+          // We must break in case of header decoding as the offset to the next payload is lost
+          // consequently the parser does not know where to continue leading to an infinity loop
+          break;
+        }
+        // We must skip the page as payload is not consistent
+        // otherwise the next functions will rethrow the exceptions as
+        // the page format does not follow the expected format
+        continue;
+      }
+      for (const auto& e : rawreader.getMinorErrors()) {
+        LOG(warning) << "Found minor Raw decoder error in FEE " << e.getFEEID() << ": " << RawDecodingError::getErrorCodeTitles(RawDecodingError::ErrorTypeToInt(e.getErrorType()));
+      }
+
+      auto& header = rawreader.getRawHeader();
+      auto feeID = raw::RDHUtils::getFEEID(header);
+      auto triggerbits = raw::RDHUtils::getTriggerType(header);
+
+      if (feeID >= 40) {
+        continue; // skip STU ddl
+      }
+
+      // use the altro decoder to decode the raw data, and extract the RCU trailer
+      AltroDecoder decoder(rawreader);
+      // check the words of the payload exception in altrodecoder
+      try {
+        decoder.decode();
+      } catch (AltroDecoderError& e) {
+        LOG(error) << e.what();
+        continue;
+      }
+      for (const auto& e : decoder.getMinorDecodingErrors()) {
+        LOG(warning) << e.what();
+      }
+
+      try {
+
+        const auto& map = mMapper->getMappingForDDL(feeID);
+        uint16_t iSM = feeID / 2;
+
+        // Loop over all the channels
+        int nBunchesNotOK = 0;
+        for (auto& chan : decoder.getChannels()) {
+          int iRow, iCol;
+          ChannelType_t chantype;
+          try {
+            iRow = map.getRow(chan.getHardwareAddress());
+            iCol = map.getColumn(chan.getHardwareAddress());
+            chantype = map.getChannelType(chan.getHardwareAddress());
+          } catch (Mapper::AddressNotFoundException& ex) {
+            LOG(error) << ex.what();
+            continue;
+          }
+
+          if (!(chantype == o2::emcal::ChannelType_t::HIGH_GAIN || chantype == o2::emcal::ChannelType_t::LOW_GAIN || chantype == o2::emcal::ChannelType_t::LEDMON)) {
+            continue;
+          }
+
+          int CellID = -1;
+          bool isLowGain = false;
+          try {
+            if (chantype == o2::emcal::ChannelType_t::HIGH_GAIN || chantype == o2::emcal::ChannelType_t::LOW_GAIN) {
+              // high- / low-gain cell
+              CellID = getCellAbsID(iSM, iCol, iRow);
+              isLowGain = chantype == o2::emcal::ChannelType_t::LOW_GAIN;
+            } else {
+              CellID = geLEDMONAbsID(iSM, iCol); // Module index encoded in colum for LEDMONs
+              isLowGain = iRow == 0;             // For LEDMONs gain type is encoded in the row (0 - low gain, 1 - high gain)
+            }
+          } catch (ModuleIndexException& e) {
+            LOG(error) << e.what();
+            continue;
+          }
+
+          // Fill pedestal object
+          for (auto& bunch : chan.getBunches()) {
+            for (auto e : bunch.getADC()) {
+              mPedestalData.fillADC(e, CellID, isLowGain, chantype == o2::emcal::ChannelType_t::LEDMON);
+            }
+          }
+        }
+      } catch (o2::emcal::MappingHandler::DDLInvalid& ddlerror) {
+        // Unable to catch mapping
+        LOG(error) << ddlerror.what();
+      }
+    }
+  }
+
+  sendData(ctx, mPedestalData);
+}
+
+int PedestalProcessorDevice::getCellAbsID(int supermoduleID, int column, int row) const
+{
+  auto [phishift, etashift] = mGeometry->ShiftOnlineToOfflineCellIndexes(supermoduleID, row, column);
+  int cellID = mGeometry->GetAbsCellIdFromCellIndexes(supermoduleID, phishift, etashift);
+  if (cellID > 17664 || cellID < 0) {
+    throw ModuleIndexException(cellID, column, row, etashift, phishift);
+  }
+  return cellID;
+}
+
+int PedestalProcessorDevice::geLEDMONAbsID(int supermoduleID, int moduleID) const
+{
+  if (moduleID >= o2::emcal::EMCAL_LEDREFS || moduleID < 0) {
+    throw ModuleIndexException(moduleID);
+  }
+  return supermoduleID * o2::emcal::EMCAL_LEDREFS + moduleID;
+}
+
+bool PedestalProcessorDevice::isLostTimeframe(framework::ProcessingContext& ctx) const
+{
+  constexpr auto originEMC = header::gDataOriginEMC;
+  o2::framework::InputSpec dummy{"dummy",
+                                 framework::ConcreteDataMatcher{originEMC,
+                                                                header::gDataDescriptionRawData,
+                                                                0xDEADBEEF}};
+  static size_t contDeadBeef = 0; // number of times 0xDEADBEEF was seen continuously
+  for (const auto& ref : o2::framework::InputRecordWalker(ctx.inputs(), {dummy})) {
+    const auto dh = o2::framework::DataRefUtils::getHeader(ref);
+    auto payloadSize = o2::framework::DataRefUtils::getPayloadSize(ref);
+    if (payloadSize == 0) {
+      auto maxWarn = o2::conf::VerbosityConfig::Instance().maxWarnDeadBeef;
+      if (++contDeadBeef <= maxWarn) {
+        LOGP(alarm, "Found input [{}/{}/{:#x}] TF#{} 1st_orbit:{} Payload {} : assuming no payload for all links in this TF{}",
+             dh->dataOrigin.str, dh->dataDescription.str, dh->subSpecification, dh->tfCounter, dh->firstTForbit, payloadSize,
+             contDeadBeef == maxWarn ? fmt::format(". {} such inputs in row received, stopping reporting", contDeadBeef) : "");
+      }
+      return true;
+    }
+  }
+  contDeadBeef = 0; // if good data, reset the counter
+  return false;
+}
+
+void PedestalProcessorDevice::sendData(framework::ProcessingContext& ctx, const PedestalProcessorData& data) const
+{
+  constexpr auto originEMC = o2::header::gDataOriginEMC;
+  ctx.outputs().snapshot(framework::Output{originEMC, "PEDDATA", 0}, data);
+}
+
+o2::framework::DataProcessorSpec o2::emcal::getPedestalProcessorDevice(bool askDistSTF)
+{
+  constexpr auto originEMC = o2::header::gDataOriginEMC;
+  std::vector inputs{{"stf", o2::framework::ConcreteDataTypeMatcher{originEMC, o2::header::gDataDescriptionRawData}, o2::framework::Lifetime::Timeframe}};
+  std::vector outputs;
+  outputs.emplace_back(originEMC, "PEDDATA", 0, o2::framework::Lifetime::Timeframe);
+  if (askDistSTF) {
+    inputs.emplace_back("stdDist", "FLP", "DISTSUBTIMEFRAME", 0, o2::framework::Lifetime::Timeframe);
+  }
+
+  return o2::framework::DataProcessorSpec{
+    "PedestalProcessor",
+    inputs,
+    outputs,
+    o2::framework::AlgorithmSpec{o2::framework::adaptFromTask()},
+    o2::framework::Options{}};
+}
\ No newline at end of file
diff --git a/Detectors/EMCAL/calibration/test/testPedestalProcessorData.cxx b/Detectors/EMCAL/calibration/test/testPedestalProcessorData.cxx
new file mode 100644
index 0000000000000..e591249fcea5b
--- /dev/null
+++ b/Detectors/EMCAL/calibration/test/testPedestalProcessorData.cxx
@@ -0,0 +1,155 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#define BOOST_TEST_MODULE Test_EMCAL_Calibration
+#define BOOST_TEST_MAIN
+#define BOOST_TEST_DYN_LINK
+
+#include 
+#include 
+#include "EMCALCalibration/PedestalProcessorData.h"
+#include 
+#include 
+#include 
+
+namespace o2
+{
+
+namespace emcal
+{
+
+const int NUMBERFECCHANNELS = 17664,
+          NUMBERLEDMONCHANNELS = 480;
+
+void compare(const PedestalProcessorData& testobject, const TProfile& refFECHG, const TProfile& refFECLG, const TProfile& refLEDMONHG, const TProfile& refLEDMONLG, bool testCount, bool verbose)
+{
+  const double PRECISION = FLT_EPSILON;
+  const double PRECISIONMEAN = FLT_EPSILON;
+  for (std::size_t ichan = 0; ichan < NUMBERFECCHANNELS; ichan++) {
+    auto [meanHG, rmsHG] = testobject.getValue(ichan, false, false);
+    auto [meanLG, rmsLG] = testobject.getValue(ichan, true, false);
+    if (verbose) {
+      std::cout << "Channel " << ichan << ", mean HG " << meanHG << ", RMS HG " << std::sqrt(rmsHG) << ", entries HG " << testobject.getEntriesForChannel(ichan, false, false) << " ref(mean " << refFECHG.GetBinContent(ichan + 1) << ", RMS " << refFECHG.GetBinError(ichan + 1) << ", entries " << refFECHG.GetBinEntries(ichan + 1) << ")" << std::endl;
+      std::cout << "Channel " << ichan << ", mean LG " << meanLG << ", RMS LG " << std::sqrt(rmsLG) << ", entries LG " << testobject.getEntriesForChannel(ichan, true, false) << " ref(mean " << refFECLG.GetBinContent(ichan + 1) << ", RMS " << refFECLG.GetBinError(ichan + 1) << ", entries " << refFECLG.GetBinEntries(ichan + 1) << ")" << std::endl;
+    }
+    BOOST_CHECK_LE(std::abs(meanHG - refFECHG.GetBinContent(ichan + 1)), PRECISIONMEAN);
+    BOOST_CHECK_LE(std::abs(meanLG - refFECLG.GetBinContent(ichan + 1)), PRECISIONMEAN);
+    BOOST_CHECK_LE(std::abs(std::sqrt(rmsHG) - refFECHG.GetBinError(ichan + 1)), PRECISION);
+    BOOST_CHECK_LE(std::abs(std::sqrt(rmsLG) - refFECLG.GetBinError(ichan + 1)), PRECISION);
+    if (testCount) {
+      BOOST_CHECK_EQUAL(testobject.getEntriesForChannel(ichan, false, false), refFECHG.GetBinEntries(ichan + 1));
+      BOOST_CHECK_EQUAL(testobject.getEntriesForChannel(ichan, true, false), refFECLG.GetBinEntries(ichan + 1));
+    }
+  }
+  for (std::size_t ichan = 0; ichan < NUMBERLEDMONCHANNELS; ichan++) {
+    auto [meanHG, rmsHG] = testobject.getValue(ichan, false, true);
+    auto [meanLG, rmsLG] = testobject.getValue(ichan, true, true);
+    if (verbose) {
+      std::cout << "LEDMON " << ichan << ", mean HG " << meanHG << ", RMS HG " << std::sqrt(rmsHG) << ", entries HG " << testobject.getEntriesForChannel(ichan, false, true) << " ref(mean " << refLEDMONHG.GetBinContent(ichan + 1) << ", RMS " << refLEDMONHG.GetBinError(ichan + 1) << ", entries " << refLEDMONHG.GetBinEntries(ichan + 1) << ")" << std::endl;
+      std::cout << "LEDMON " << ichan << ", mean LG " << meanLG << ", RMS LG " << std::sqrt(rmsLG) << ", entries LG " << testobject.getEntriesForChannel(ichan, true, true) << " ref(mean " << refLEDMONLG.GetBinContent(ichan + 1) << ", RMS " << refLEDMONLG.GetBinError(ichan + 1) << ", entries " << refLEDMONLG.GetBinEntries(ichan + 1) << ")" << std::endl;
+    }
+    BOOST_CHECK_LE(std::abs(meanHG - refLEDMONHG.GetBinContent(ichan + 1)), PRECISIONMEAN);
+    BOOST_CHECK_LE(std::abs(meanLG - refLEDMONLG.GetBinContent(ichan + 1)), PRECISIONMEAN);
+    BOOST_CHECK_LE(std::abs(std::sqrt(rmsHG) - refLEDMONHG.GetBinError(ichan + 1)), PRECISION);
+    BOOST_CHECK_LE(std::abs(std::sqrt(rmsLG) - refLEDMONLG.GetBinError(ichan + 1)), PRECISION);
+    if (testCount) {
+      BOOST_CHECK_EQUAL(testobject.getEntriesForChannel(ichan, false, true), refLEDMONHG.GetBinEntries(ichan + 1));
+      BOOST_CHECK_EQUAL(testobject.getEntriesForChannel(ichan, true, true), refLEDMONLG.GetBinEntries(ichan + 1));
+    }
+  }
+}
+
+BOOST_AUTO_TEST_CASE(testPedestalProcessorData)
+{
+  // we compare with RMS, so the
+  TProfile refFECHG("refFECHG", "Reference FEC HG", NUMBERFECCHANNELS, -0.5, NUMBERFECCHANNELS - 0.5, "s"),
+    refFECLG("refFECLG", "Reference FEC LG", NUMBERFECCHANNELS, -0.5, NUMBERFECCHANNELS - 0.5, "s"),
+    refLEDMONHG("refLEDMONHG", "Reference LEDMON HG", NUMBERLEDMONCHANNELS, -0.5, NUMBERLEDMONCHANNELS - 0.5, "s"),
+    refLEDMONLG("refLEDMONLG", "Reference LEDMON LG", NUMBERLEDMONCHANNELS, -0.5, NUMBERLEDMONCHANNELS - 0.5, "s");
+  refFECHG.Sumw2();
+  refFECLG.Sumw2();
+  refLEDMONHG.Sumw2();
+  refLEDMONLG.Sumw2();
+
+  PedestalProcessorData testobject;
+
+  const int NUMBERALTROSAMPLES = 15;
+  for (auto iev = 0; iev < 1000; iev++) {
+    for (int ichan = 0; ichan < NUMBERFECCHANNELS; ichan++) {
+      for (int isample = 0; isample < 15; isample++) {
+        // short adc_hg = static_cast(adcGeneratorHG(gen)),
+        //       adc_lg = static_cast(adcGeneratorLG(gen));
+        auto raw_hg = gRandom->Gaus(40, 6),
+             raw_lg = gRandom->Gaus(36, 10);
+        unsigned short adc_hg = static_cast(raw_hg > 0 ? raw_hg : 0),
+                       adc_lg = static_cast(raw_lg > 0 ? raw_lg : 0);
+        refFECHG.Fill(ichan, adc_hg);
+        refFECLG.Fill(ichan, adc_lg);
+        testobject.fillADC(adc_hg, ichan, false, false);
+        testobject.fillADC(adc_lg, ichan, true, false);
+      }
+    }
+  }
+  for (auto iev = 0; iev < 1000; iev++) {
+    for (int ichan = 0; ichan < NUMBERLEDMONCHANNELS; ichan++) {
+      for (int isample = 0; isample < 15; isample++) {
+        // short adc_hg = static_cast(adcGeneratorLEDMONHG(gen)),
+        //      adc_lg = static_cast(adcGeneratorLEDMONLG(gen));
+        auto raw_hg = gRandom->Gaus(40, 6),
+             raw_lg = gRandom->Gaus(36, 10);
+        unsigned short adc_hg = static_cast(raw_hg > 0 ? raw_hg : 0),
+                       adc_lg = static_cast(raw_lg > 0 ? raw_lg : 0);
+        refLEDMONHG.Fill(ichan, adc_hg);
+        refLEDMONLG.Fill(ichan, adc_lg);
+        testobject.fillADC(adc_hg, ichan, false, true);
+        testobject.fillADC(adc_lg, ichan, true, true);
+      }
+    }
+  }
+
+  // Compare channels
+  compare(testobject, refFECHG, refFECLG, refLEDMONHG, refLEDMONLG, true, false);
+
+  // Test operator+
+  auto testdoubled = testobject + testobject;
+  TProfile refFECHGdouble(refFECHG),
+    refFECLGdouble(refFECLG),
+    refLEDMONHGdouble(refLEDMONHG),
+    refLEDMONLGdouble(refLEDMONLG);
+  refFECHGdouble.Add(&refFECHG);
+  refFECLGdouble.Add(&refFECLG);
+  refLEDMONHGdouble.Add(&refLEDMONHG);
+  refLEDMONLGdouble.Add(&refLEDMONLG);
+  compare(testdoubled, refFECHGdouble, refFECLGdouble, refLEDMONHGdouble, refLEDMONLGdouble, false, false);
+
+  // Test reset function
+  auto testreset = testobject;
+  testreset.reset();
+  for (std::size_t ichan = 0; ichan < NUMBERFECCHANNELS; ichan++) {
+    auto [meanHG, rmsHG] = testreset.getValue(ichan, false, false);
+    auto [meanLG, rmsLG] = testreset.getValue(ichan, true, false);
+    BOOST_CHECK_EQUAL(meanHG, 0.);
+    BOOST_CHECK_EQUAL(meanLG, 0.);
+    BOOST_CHECK_EQUAL(testreset.getEntriesForChannel(ichan, false, false), 0);
+    BOOST_CHECK_EQUAL(testreset.getEntriesForChannel(ichan, true, false), 0);
+  }
+  for (std::size_t ichan = 0; ichan < NUMBERLEDMONCHANNELS; ichan++) {
+    auto [meanHG, rmsHG] = testreset.getValue(ichan, false, true);
+    auto [meanLG, rmsLG] = testreset.getValue(ichan, true, true);
+    BOOST_CHECK_EQUAL(meanHG, 0.);
+    BOOST_CHECK_EQUAL(meanLG, 0.);
+    BOOST_CHECK_EQUAL(testreset.getEntriesForChannel(ichan, false, true), 0);
+    BOOST_CHECK_EQUAL(testreset.getEntriesForChannel(ichan, true, true), 0);
+  }
+}
+
+} // namespace emcal
+
+} // namespace o2
\ No newline at end of file
diff --git a/Detectors/EMCAL/calibration/testWorkflow/EMCALChannelCalibratorSpec.h b/Detectors/EMCAL/calibration/testWorkflow/EMCALChannelCalibratorSpec.h
index aa54ad6aae756..2d6a9dc6c9d01 100644
--- a/Detectors/EMCAL/calibration/testWorkflow/EMCALChannelCalibratorSpec.h
+++ b/Detectors/EMCAL/calibration/testWorkflow/EMCALChannelCalibratorSpec.h
@@ -23,6 +23,7 @@
 #include "DetectorsCalibration/Utils.h"
 #include "DataFormatsEMCAL/TriggerRecord.h"
 #include "CommonUtils/MemFileHelper.h"
+#include "CommonConstants/Triggers.h"
 #include "Framework/Task.h"
 #include "Framework/ConfigParamRegistry.h"
 #include "Framework/ControlService.h"
@@ -31,8 +32,14 @@
 #include "CCDB/CcdbObjectInfo.h"
 #include "CommonUtils/NameConf.h"
 #include "DetectorsBase/GRPGeomHelper.h"
+#include "Framework/CCDBParamSpec.h"
+#include "DataFormatsCTP/Digits.h"
+#include "DataFormatsCTP/Configuration.h"
+
 // for time measurements
 #include 
+#include 
+#include 
 
 using namespace o2::framework;
 
@@ -41,13 +48,38 @@ namespace o2
 namespace calibration
 {
 
+class CalibInputDownsampler
+{
+ public:
+  CalibInputDownsampler() { initSeed(); }
+  ~CalibInputDownsampler() = default;
+
+  void setSamplingFraction(float samplingFraction) { mSamplingFraction = samplingFraction; }
+  bool acceptEvent()
+  {
+    auto rnr = mSampler(mRandomGenerator);
+    return rnr < mSamplingFraction;
+  }
+
+ private:
+  void initSeed()
+  {
+    auto now = std::chrono::system_clock::now().time_since_epoch().count();
+    std::seed_seq randomseed{uint32_t(now & 0xffffffff), uint32_t(now >> 32)};
+    mRandomGenerator.seed(randomseed);
+  }
+  std::mt19937_64 mRandomGenerator;
+  std::uniform_real_distribution mSampler{0, 1};
+  float mSamplingFraction = 1;
+};
+
 class EMCALChannelCalibDevice : public o2::framework::Task
 {
 
   using EMCALCalibParams = o2::emcal::EMCALCalibParams;
 
  public:
-  EMCALChannelCalibDevice(std::shared_ptr req) : mCCDBRequest(req) {}
+  EMCALChannelCalibDevice(std::shared_ptr req, bool params, std::string calibType, bool rejCalibTrg, bool rejL0Trig, bool applyGainCalib) : mCCDBRequest(req), mLoadCalibParamsFromCCDB(params), mCalibType(calibType), mRejectCalibTriggers(rejCalibTrg), mRejectL0Triggers(rejL0Trig), mApplyGainCalib(applyGainCalib) {}
 
   void init(o2::framework::InitContext& ic) final
   {
@@ -55,31 +87,24 @@ class EMCALChannelCalibDevice : public o2::framework::Task
 
     mCalibExtractor = std::make_shared();
 
-    if (EMCALCalibParams::Instance().calibType.find("time") != std::string::npos) { // time calibration
+    if (mCalibType.find("time") != std::string::npos) { // time calibration
       isBadChannelCalib = false;
       if (!mTimeCalibrator) {
-        mTimeCalibrator = std::make_unique>();
+        mTimeCalibrator = std::make_unique>();
       }
       mTimeCalibrator->SetCalibExtractor(mCalibExtractor);
-      mTimeCalibrator->setSlotLength(EMCALCalibParams::Instance().slotLength);
-      if (EMCALCalibParams::Instance().UpdateAtEndOfRunOnly) {
-        mBadChannelCalibrator->setUpdateAtTheEndOfRunOnly();
-      }
-
+      mTimeCalibrator->setSavedSlotAllowed(EMCALCalibParams::Instance().setSavedSlotAllowed_EMC);
+      mTimeCalibrator->setLoadAtSOR(EMCALCalibParams::Instance().setSavedSlotAllowedSOR_EMC);
+      mTimeCalibrator->setSaveFileName("emc-time-calib.root");
     } else { // bad cell calibration
       isBadChannelCalib = true;
       if (!mBadChannelCalibrator) {
-        mBadChannelCalibrator = std::make_unique>();
+        mBadChannelCalibrator = std::make_unique>();
       }
       mBadChannelCalibrator->SetCalibExtractor(mCalibExtractor);
-      mBadChannelCalibrator->setSlotLength(EMCALCalibParams::Instance().slotLength);
-      if (EMCALCalibParams::Instance().UpdateAtEndOfRunOnly) {
-        mBadChannelCalibrator->setUpdateAtTheEndOfRunOnly();
-      }
-      mBadChannelCalibrator->setIsTest(EMCALCalibParams::Instance().enableTestMode);
-      if (EMCALCalibParams::Instance().useScaledHistoForBadChannelMap) {
-        mBadChannelCalibrator->getCalibExtractor()->setUseScaledHistoForBadChannels(true);
-      }
+      mBadChannelCalibrator->setSavedSlotAllowed(EMCALCalibParams::Instance().setSavedSlotAllowed_EMC);
+      mBadChannelCalibrator->setLoadAtSOR(EMCALCalibParams::Instance().setSavedSlotAllowedSOR_EMC);
+      mBadChannelCalibrator->setSaveFileName("emc-channel-calib.root");
     }
   }
 
@@ -87,16 +112,155 @@ class EMCALChannelCalibDevice : public o2::framework::Task
   void finaliseCCDB(o2::framework::ConcreteDataMatcher& matcher, void* obj) final
   {
     o2::base::GRPGeomHelper::instance().finaliseCCDB(matcher, obj);
+    // check if calib params need to be updated
+    if (matcher == ConcreteDataMatcher("EMC", "EMCALCALIBPARAM", 0)) {
+      LOG(info) << "EMCal CalibParams updated";
+      EMCALCalibParams::Instance().printKeyValues(true, true);
+    }
+    if (matcher == ConcreteDataMatcher("EMC", "SCALEFACTORS", 0)) {
+      if (mBadChannelCalibrator && EMCALCalibParams::Instance().useScaledHisto_bc) {
+        LOG(info) << "Configuring scale factors for bad channel map";
+        mBadChannelCalibrator->getCalibExtractor()->setBCMScaleFactors(reinterpret_cast(obj));
+        mScaleFactorsInitialized = true;
+      }
+    }
+    if (mApplyGainCalib && matcher == ConcreteDataMatcher("EMC", "EMCGAINCALIB", 0)) {
+      if (mBadChannelCalibrator) {
+        LOG(info) << "Configuring gain calib factors for bad channel";
+        if (mBadChannelCalibrator->setGainCalibrationFactors(reinterpret_cast(obj))) {
+          mGainCalibFactorsInitialized = true;
+        }
+      }
+      if (mTimeCalibrator) {
+        LOG(info) << "Configuring gain calib factors for time calib";
+        if (mTimeCalibrator->setGainCalibrationFactors(reinterpret_cast(obj))) {
+          mGainCalibFactorsInitialized = true;
+        }
+      }
+    }
+    if (mRejectL0Triggers && matcher == ConcreteDataMatcher("CTP", "CTPCONFIG", 0)) {
+      // clear current class mask and prepare to fill in the updated values
+      // The trigger names are seperated by a ":" in one string in the calib params
+      mSelectedClassMasks.clear();
+      std::string strSelClassMasks = EMCALCalibParams::Instance().selectedClassMasks;
+      std::string delimiter = ":";
+      size_t pos = 0;
+      std::vector vSelMasks;
+      while ((pos = strSelClassMasks.find(delimiter)) != std::string::npos) {
+        vSelMasks.push_back(strSelClassMasks.substr(0, pos));
+        strSelClassMasks.erase(0, pos + delimiter.length());
+      }
+      vSelMasks.push_back(strSelClassMasks);
+
+      auto ctpconf = reinterpret_cast(obj);
+
+      for (auto& cls : ctpconf->getCTPClasses()) {
+        LOG(debug) << "CTP class: " << cls.name << "\t " << cls.classMask;
+
+        if (std::find(vSelMasks.begin(), vSelMasks.end(), cls.name) != vSelMasks.end()) {
+          mSelectedClassMasks.push_back(cls.classMask);
+          LOG(info) << "Setting selected class mask " << cls.name << " to bit " << cls.classMask;
+        }
+      }
+    }
   }
 
+  //_________________________________________________________________
   void run(o2::framework::ProcessingContext& pc) final
   {
     if (EMCALCalibParams::Instance().enableTimeProfiling) {
       timeMeas[0] = std::chrono::duration_cast(std::chrono::high_resolution_clock::now().time_since_epoch()).count();
     }
 
+    const auto& tinfo = pc.services().get();
+    if (tinfo.globalRunNumberChanged) { // new run is starting
+      mRunStopRequested = false;
+    }
+    if (mRunStopRequested) {
+      return;
+    }
+
     o2::base::GRPGeomHelper::instance().checkUpdates(pc);
-    o2::base::TFIDInfoHelper::fillTFIDInfo(pc, mTimeCalibrator->getCurrentTFInfo());
+    if (mTimeCalibrator) {
+      o2::base::TFIDInfoHelper::fillTFIDInfo(pc, mTimeCalibrator->getCurrentTFInfo());
+    } else if (mBadChannelCalibrator) {
+      o2::base::TFIDInfoHelper::fillTFIDInfo(pc, mBadChannelCalibrator->getCurrentTFInfo());
+    }
+
+    if (mLoadCalibParamsFromCCDB) {
+      // for reading the calib objects from the CCDB
+      pc.inputs().get("EMC_CalibParam");
+    }
+
+    if (mBadChannelCalibrator && EMCALCalibParams::Instance().useScaledHisto_bc && !mScaleFactorsInitialized) {
+      // Trigger reading the scale factors from the CCDB (Bad channel calib only)
+      pc.inputs().get("EMC_Scalefactors");
+    }
+
+    // prepare CTPConfiguration such that it can be loaded in finalise ccdb
+    if (mRejectL0Triggers) {
+      pc.inputs().get(getCTPConfigBinding());
+    }
+
+    if (!mIsConfigured) {
+      // configure calibrators (after calib params are loaded from the CCDB)
+      // long tsMS = o2::base::GRPGeomHelper::instance().getOrbitResetTimeMS() + pc.services().get().firstTForbit * o2::constants::lhc::LHCOrbitMUS / 1000; // this reads the ts from the data.
+      long tsMS = o2::ccdb::getCurrentTimestamp();
+      configureCalibrators(tsMS);
+      mIsConfigured = true;
+    }
+
+    if (mApplyGainCalib && !mGainCalibFactorsInitialized) {
+      // process dummy data with no cells to create a slot
+      std::vector cellDummyData(0);
+      if (isBadChannelCalib) {
+        mBadChannelCalibrator->process(cellDummyData);
+      } else {
+        mTimeCalibrator->process(cellDummyData);
+      }
+      // for reading the calib objects from the CCDB
+      pc.inputs().get(getGainCalibBinding());
+    }
+
+    float samplingFraction = isBadChannelCalib ? EMCALCalibParams::Instance().fractionEvents_bc : EMCALCalibParams::Instance().fractionEvents_tc;
+    if (samplingFraction < 1) {
+      if (!mDownsampler) {
+        mDownsampler = std::make_unique();
+      }
+      mDownsampler->setSamplingFraction(samplingFraction);
+    }
+
+    using ctpDigitsType = std::decay_t>(getCTPDigitsBinding()))>;
+    std::optional ctpDigits;
+    if (mRejectL0Triggers) {
+      ctpDigits = pc.inputs().get>(getCTPDigitsBinding());
+    }
+
+    // reset EOR behaviour
+    if (mTimeCalibrator) {
+      if (mTimeCalibrator->getSaveAtEOR())
+        mTimeCalibrator->setSaveAtEOR(false);
+    } else if (mBadChannelCalibrator) {
+      if (mBadChannelCalibrator->getSaveAtEOR())
+        mBadChannelCalibrator->setSaveAtEOR(false);
+    }
+
+    // prepare CTP information to reject EMCal triggers
+    uint64_t classMaskCTP = 0;
+    std::unordered_set numBCAccepted;
+    if (mRejectL0Triggers) {
+      for (auto& ctpDigit : *ctpDigits) {
+        // obtain trigger mask that belongs to the selected bc
+        classMaskCTP = ctpDigit.CTPClassMask.to_ulong();
+        // now check if min bias trigger is not in mask
+        for (const uint64_t& selectedClassMask : mSelectedClassMasks) {
+          if ((classMaskCTP & selectedClassMask) != 0) {
+            LOG(debug) << "classmask " << selectedClassMask << " added for the bc " << ctpDigit.intRecord.toLong() + EMCALCalibParams::Instance().bcShiftCTP;
+            numBCAccepted.insert(ctpDigit.intRecord.toLong() + EMCALCalibParams::Instance().bcShiftCTP);
+          }
+        }
+      }
+    }
 
     auto tfcounter = o2::header::get(pc.inputs().get(getCellBinding()).header)->startTime;
 
@@ -112,6 +276,26 @@ class EMCALChannelCalibDevice : public o2::framework::Task
       if (!trg.getNumberOfObjects()) {
         continue;
       }
+      // reject calibration trigger from the calibration
+      if (mRejectCalibTriggers) {
+        LOG(debug) << "Trigger: " << trg.getTriggerBits() << "   o2::trigger::Cal " << o2::trigger::Cal;
+        if (trg.getTriggerBits() & o2::trigger::Cal) {
+          LOG(debug) << "skipping triggered events due to wrong trigger (no Physics trigger)";
+          continue;
+        }
+      }
+
+      // reject all triggers that are not included in the classMask (typically only EMC min. bias should be accepted)
+      if (mRejectL0Triggers) {
+        if (numBCAccepted.find(trg.getBCData().toLong()) == numBCAccepted.end()) {
+          LOG(debug) << "correct trigger not found, rejecting event";
+          continue;
+        }
+      }
+
+      if (mDownsampler && !mDownsampler->acceptEvent()) {
+        continue;
+      }
       gsl::span eventData(data.data() + trg.getFirstEntry(), trg.getNumberOfObjects());
 
       // fast calibration
@@ -126,6 +310,28 @@ class EMCALChannelCalibDevice : public o2::framework::Task
         }
       }
     }
+    static bool firstCall = true;
+    if (firstCall) {
+      firstCall = false;
+      if (mTimeCalibrator) {
+        mTimeCalibrator->loadSavedSlot();
+      } else if (mBadChannelCalibrator) {
+        mBadChannelCalibrator->loadSavedSlot();
+      }
+    }
+
+    if (pc.transitionState() == TransitionHandlingState::Requested) {
+      LOG(debug) << "Run stop requested, finalizing";
+      mRunStopRequested = true;
+      if (isBadChannelCalib) {
+        mBadChannelCalibrator->setSaveAtEOR(true);
+        mBadChannelCalibrator->checkSlotsToFinalize(o2::calibration::INFINITE_TF);
+      } else {
+        mTimeCalibrator->setSaveAtEOR(true);
+        mTimeCalibrator->checkSlotsToFinalize(o2::calibration::INFINITE_TF);
+      }
+    }
+
     if (isBadChannelCalib) {
       sendOutput(pc.outputs());
     } else {
@@ -139,25 +345,45 @@ class EMCALChannelCalibDevice : public o2::framework::Task
 
   void endOfStream(o2::framework::EndOfStreamContext& ec) final
   {
+    if (mRunStopRequested) {
+      return;
+    }
     if (isBadChannelCalib) {
+      mBadChannelCalibrator->setSaveAtEOR(true);
       mBadChannelCalibrator->checkSlotsToFinalize(o2::calibration::INFINITE_TF);
       sendOutput(ec.outputs());
     } else {
+      mTimeCalibrator->setSaveAtEOR(true);
       mTimeCalibrator->checkSlotsToFinalize(o2::calibration::INFINITE_TF);
       sendOutput(ec.outputs());
     }
+    mRunStopRequested = true;
   }
 
   static const char* getCellBinding() { return "EMCCells"; }
   static const char* getCellTriggerRecordBinding() { return "EMCCellsTrgR"; }
+  static const char* getCTPDigitsBinding() { return "CTPDigits"; }
+  static const char* getCTPConfigBinding() { return "CTPConfig"; }
+  static const char* getGainCalibBinding() { return "EMCGainCalib"; }
 
  private:
-  std::unique_ptr> mBadChannelCalibrator;
-  std::unique_ptr> mTimeCalibrator;
-  std::shared_ptr mCalibExtractor;
-  std::shared_ptr mCCDBRequest;
-  bool isBadChannelCalib = true;
-  std::array timeMeas;
+  std::unique_ptr> mBadChannelCalibrator;     ///< Bad channel calibrator
+  std::unique_ptr> mTimeCalibrator; ///< Time calibrator
+  std::shared_ptr mCalibExtractor;                                                                     ///< Calibration postprocessing
+  std::shared_ptr mCCDBRequest;                                                                              ///< CCDB request for geometry
+  std::string mCalibType;                                                                                                              ///< Name of the calibration type
+  bool mIsConfigured = false;                                                                                                          ///< Configure status of calibrators
+  bool mScaleFactorsInitialized = false;                                                                                               ///< Scale factor init status
+  bool isBadChannelCalib = true;                                                                                                       ///< Calibration mode bad channel calib (false := time calib)
+  bool mLoadCalibParamsFromCCDB = true;                                                                                                ///< Switch for loading calib params from the CCDB
+  bool mRejectCalibTriggers = true;                                                                                                    ///! reject calibration triggers in the online calibration
+  bool mRejectL0Triggers = true;                                                                                                       ///! reject EMCal Gamma and Jet triggers in the online calibration
+  bool mApplyGainCalib = true;                                                                                                         ///! switch if gain calibration should be applied during filling of histograms or not
+  bool mGainCalibFactorsInitialized = false;                                                                                           ///! Gain calibration init status
+  bool mRunStopRequested = false;                                                                                                      ///< flag that the run has stopped
+  std::array timeMeas;                                                                                                      ///! Used for time measurement and holds the start and end time in the run function
+  std::vector mSelectedClassMasks = {};                                                                                      ///! EMCal minimum bias trigger bit. Only this bit will be used for calibration
+  std::unique_ptr mDownsampler;
 
   //________________________________________________________________
   template 
@@ -202,24 +428,68 @@ class EMCALChannelCalibDevice : public o2::framework::Task
       }
     }
   }
-};
+
+  /// \brief Configure calibrators from the calib params
+  void configureCalibrators(long ts)
+  {
+    auto currFill = o2::base::GRPGeomHelper::instance().getGRPLHCIF()->getFillNumber();
+    auto runtype = o2::base::GRPGeomHelper::instance().getGRPECS()->getRunType();
+    LOG(debug) << "currFill  " << currFill << "  runtype " << runtype;
+
+    if (mTimeCalibrator) {
+      LOG(info) << "Configuring time calibrator";
+      mTimeCalibrator->setSlotLength(EMCALCalibParams::Instance().slotLength_tc);
+      if (EMCALCalibParams::Instance().slotLength_tc == 0) {
+        // for infinite slot length do not check if there is enough statistics after every TF
+        mTimeCalibrator->setCheckIntervalInfiniteSlot(1e5);
+      }
+      if (EMCALCalibParams::Instance().UpdateAtEndOfRunOnly_tc) {
+        mTimeCalibrator->setUpdateAtTheEndOfRunOnly();
+      }
+      mTimeCalibrator->setSaveFileName("emc-time-calib-" + std::to_string(runtype) + ".root");
+      mTimeCalibrator->setFillNr(currFill);
+      mTimeCalibrator->setRunType(runtype);
+      mTimeCalibrator->setCurrTSInHours(static_cast(ts / o2::ccdb::CcdbObjectInfo::HOUR));
+    }
+    if (mBadChannelCalibrator) {
+      LOG(info) << "Configuring bad channel calibrator";
+      mBadChannelCalibrator->setSlotLength(EMCALCalibParams::Instance().slotLength_bc);
+      if (EMCALCalibParams::Instance().slotLength_bc == 0) {
+        // for infinite slot length do not check if there is enough statistics after every TF
+        mBadChannelCalibrator->setCheckIntervalInfiniteSlot(1e5);
+      }
+      if (EMCALCalibParams::Instance().UpdateAtEndOfRunOnly_bc) {
+        mBadChannelCalibrator->setUpdateAtTheEndOfRunOnly();
+      }
+      mBadChannelCalibrator->setIsTest(EMCALCalibParams::Instance().enableTestMode_bc);
+      mBadChannelCalibrator->setFillNr(currFill);
+      mBadChannelCalibrator->setRunType(runtype);
+      mBadChannelCalibrator->setSaveFileName("emc-channel-calib-" + std::to_string(runtype) + ".root");
+      mBadChannelCalibrator->setCurrTSInHours(static_cast(ts / o2::ccdb::CcdbObjectInfo::HOUR));
+    }
+  }
+}; // namespace calibration
 
 } // namespace calibration
 
 namespace framework
 {
 
-DataProcessorSpec getEMCALChannelCalibDeviceSpec()
+DataProcessorSpec getEMCALChannelCalibDeviceSpec(const std::string calibType, const bool loadCalibParamsFromCCDB, const bool rejectCalibTrigger, const bool rejectL0Trigger, const bool ctpcfgperrun, const bool applyGainCalib)
 {
   using device = o2::calibration::EMCALChannelCalibDevice;
   using clbUtils = o2::calibration::Utils;
   using EMCALCalibParams = o2::emcal::EMCALCalibParams;
+  using CalibDB = o2::emcal::CalibDB;
 
   std::vector outputs;
-  if (EMCALCalibParams::Instance().calibType.find("time") != std::string::npos) {                                                        // time calibration
-    outputs.emplace_back(ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBPayload, "EMC_TIMECALIB"}, Lifetime::Sporadic);   // This needs to match with the output!
-    outputs.emplace_back(ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBWrapper, "EMC_TIMECALIB"}, Lifetime::Sporadic);   // This needs to match with the output!
-  } else {                                                                                                                               // bad channel calibration
+  std::string processorName;
+  if (calibType.find("time") != std::string::npos) { // time calibration
+    processorName = "calib-emcalchannel-time";
+    outputs.emplace_back(ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBPayload, "EMC_TIMECALIB"}, Lifetime::Sporadic); // This needs to match with the output!
+    outputs.emplace_back(ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBWrapper, "EMC_TIMECALIB"}, Lifetime::Sporadic); // This needs to match with the output!
+  } else {                                                                                                                             // bad channel calibration
+    processorName = "calib-emcalchannel-badchannel";
     outputs.emplace_back(ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBPayload, "EMC_BADCHANNELS"}, Lifetime::Sporadic); // This needs to match with the output!
     outputs.emplace_back(ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBWrapper, "EMC_BADCHANNELS"}, Lifetime::Sporadic); // This needs to match with the output!
   }
@@ -227,22 +497,41 @@ DataProcessorSpec getEMCALChannelCalibDeviceSpec()
   std::vector inputs;
   inputs.emplace_back(device::getCellBinding(), o2::header::gDataOriginEMC, "CELLS", 0, o2::framework::Lifetime::Timeframe);
   inputs.emplace_back(device::getCellTriggerRecordBinding(), o2::header::gDataOriginEMC, "CELLSTRGR", 0, o2::framework::Lifetime::Timeframe);
+  // inputs.emplace_back("EMCTriggers", "EMC", "CELLSTRGR", 0, Lifetime::Timeframe)
+  // for loading the channelCalibParams from the ccdb
+  if (loadCalibParamsFromCCDB) {
+    inputs.emplace_back("EMC_CalibParam", o2::header::gDataOriginEMC, "EMCALCALIBPARAM", 0, Lifetime::Condition, ccdbParamSpec("EMC/Config/CalibParam"));
+  }
+  if (calibType.find("badchannel") != std::string::npos) {
+    inputs.emplace_back("EMC_Scalefactors", o2::header::gDataOriginEMC, "SCALEFACTORS", 0, Lifetime::Condition, ccdbParamSpec(CalibDB::getCDBPathChannelScaleFactors()));
+  }
+  if (applyGainCalib) {
+    inputs.emplace_back(device::getGainCalibBinding(), o2::header::gDataOriginEMC, "EMCGAINCALIB", 0, Lifetime::Condition, ccdbParamSpec("EMC/Calib/GainCalibFactors"));
+  }
+
+  // data request needed for rejection of EMCal trigger
+  if (rejectL0Trigger) {
+    inputs.emplace_back(device::getCTPConfigBinding(), "CTP", "CTPCONFIG", 0, Lifetime::Condition, ccdbParamSpec("CTP/Config/Config", ctpcfgperrun));
+    inputs.emplace_back(device::getCTPDigitsBinding(), "CTP", "DIGITS", 0, Lifetime::Timeframe);
+  }
+
   auto ccdbRequest = std::make_shared(true,                           // orbitResetTime
                                                                 true,                           // GRPECS=true
-                                                                false,                          // GRPLHCIF
+                                                                true,                           // GRPLHCIF
                                                                 false,                          // GRPMagField
                                                                 false,                          // askMatLUT
                                                                 o2::base::GRPGeomRequest::None, // geometry
                                                                 inputs);
+
   return DataProcessorSpec{
-    "calib-emcalchannel-calibration",
+    processorName,
     inputs,
     outputs,
-    AlgorithmSpec{adaptFromTask(ccdbRequest)},
+    AlgorithmSpec{adaptFromTask(ccdbRequest, loadCalibParamsFromCCDB, calibType, rejectCalibTrigger, rejectL0Trigger, applyGainCalib)},
     Options{}};
 }
 
 } // namespace framework
 } // namespace o2
 
-#endif
\ No newline at end of file
+#endif
diff --git a/Detectors/EMCAL/calibration/testWorkflow/emc-channel-calib-workflow.cxx b/Detectors/EMCAL/calibration/testWorkflow/emc-channel-calib-workflow.cxx
index 790cf5d22705e..2eb67276905aa 100644
--- a/Detectors/EMCAL/calibration/testWorkflow/emc-channel-calib-workflow.cxx
+++ b/Detectors/EMCAL/calibration/testWorkflow/emc-channel-calib-workflow.cxx
@@ -36,7 +36,13 @@ using namespace o2::emcal;
 void customize(std::vector& workflowOptions)
 {
   std::vector options{
-    {"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}}};
+    {"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}},
+    {"calibType", VariantType::String, "time", {"choose which calibration should be performed: time for tiem calibration, badchannel for bad channel calibration"}},
+    {"no-loadCalibParamsFromCCDB", VariantType::Bool, false, {"disabled by default such that calib params are taken from the ccdb. If enabled, calib params are taken from EMCALCalibParams.h directly"}},
+    {"no-rejectCalibTrigger", VariantType::Bool, false, {"disabled by default such that calib triggers are rejected. If enabled, calibration triggers (LED events etc.) also enter the calibration"}},
+    {"no-rejectL0Trigger", VariantType::Bool, false, {"disabled by default such that L0 triggers are rejected. If enabled, L0 triggers (Gamma trigger and jet trigger) also enter the calibration"}},
+    {"no-applyGainCalib", VariantType::Bool, false, {"if appplication of gain calibration should be disabled"}},
+    {"ctpconfig-run-independent", VariantType::Bool, false, {"Use CTP config w/o runNumber tag"}}};
 
   std::swap(workflowOptions, options);
 }
@@ -45,12 +51,19 @@ void customize(std::vector& workflowOptions)
 
 WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
 {
-  WorkflowSpec specs;
-  specs.emplace_back(getEMCALChannelCalibDeviceSpec());
 
   o2::conf::ConfigurableParam::updateFromString(cfgc.options().get("configKeyValues"));
+  std::string calibType = cfgc.options().get("calibType");
+  bool loadCalibParamsFromCCDB = !cfgc.options().get("no-loadCalibParamsFromCCDB");
+  bool rejectCalibTrigger = !cfgc.options().get("no-rejectCalibTrigger");
+  bool rejectL0Trigger = !cfgc.options().get("no-rejectL0Trigger");
+  bool ctpcfgperrun = !cfgc.options().get("ctpconfig-run-independent");
+  bool applyGainCalib = !cfgc.options().get("no-applyGainCalib");
+
+  WorkflowSpec specs;
+  specs.emplace_back(getEMCALChannelCalibDeviceSpec(calibType, loadCalibParamsFromCCDB, rejectCalibTrigger, rejectL0Trigger, ctpcfgperrun, applyGainCalib));
 
   // configure dpl timer to inject correct firstTForbit: start from the 1st orbit of TF containing 1st sampled orbit
   // o2::raw::HBFUtilsInitializer hbfIni(cfgc, specs);
   return specs;
-}
\ No newline at end of file
+}
diff --git a/Detectors/EMCAL/calibration/testWorkflow/emc-pedestal-calib-workflow.cxx b/Detectors/EMCAL/calibration/testWorkflow/emc-pedestal-calib-workflow.cxx
new file mode 100644
index 0000000000000..c624dbaa89784
--- /dev/null
+++ b/Detectors/EMCAL/calibration/testWorkflow/emc-pedestal-calib-workflow.cxx
@@ -0,0 +1,53 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// @file   emc-channel-calib-workflow.cxx
+/// @author Joshua Koenig
+/// @since  2024-03-25
+/// @brief  Basic workflow for EMCAL pedestal calibration
+
+#include "CommonUtils/ConfigurableParam.h"
+#include "EMCALCalibration/PedestalCalibDevice.h"
+#include "Framework/ConfigParamSpec.h"
+#include "Framework/Variant.h"
+#include "Framework/WorkflowSpec.h"
+#include "DetectorsRaw/HBFUtilsInitializer.h"
+
+#include 
+
+using namespace o2::framework;
+using namespace o2::emcal;
+
+// add workflow options, note that customization needs to be declared before
+// including Framework/runDataProcessing
+void customize(std::vector& workflowOptions)
+{
+  std::vector options{
+    {"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}},
+    {"dumpToFile", VariantType::Bool, false, {"if output (that goes to DCS ccdb) should be stored in txt file for local debugging"}},
+    {"addRunNumber", VariantType::Bool, false, {"if true, run number will be added to ccdb file as first element of the string"}}};
+  std::swap(workflowOptions, options);
+}
+
+#include "Framework/runDataProcessing.h"
+
+WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
+{
+
+  o2::conf::ConfigurableParam::updateFromString(cfgc.options().get("configKeyValues"));
+  bool dumpToFile = cfgc.options().get("dumpToFile");
+  bool addRunNumber = cfgc.options().get("addRunNumber");
+
+  WorkflowSpec specs;
+  specs.emplace_back(o2::emcal::getPedestalCalibDevice(dumpToFile, addRunNumber));
+
+  return specs;
+}
diff --git a/Detectors/EMCAL/calibration/testWorkflow/emc-pedestal-processor-workflow.cxx b/Detectors/EMCAL/calibration/testWorkflow/emc-pedestal-processor-workflow.cxx
new file mode 100644
index 0000000000000..efa4dc3f4afda
--- /dev/null
+++ b/Detectors/EMCAL/calibration/testWorkflow/emc-pedestal-processor-workflow.cxx
@@ -0,0 +1,54 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// @file   emc-channel-calib-workflow.cxx
+/// @author Hannah Bossi
+/// @since  2020-12-01
+/// @brief  Basic workflow for EMCAL bad channel calibration (adapted from tof-calib-workflow.cxx)
+
+#include "CommonUtils/ConfigurableParam.h"
+#include "EMCALCalibration/PedestalProcessorDevice.h"
+#include "Framework/ConfigParamSpec.h"
+#include "Framework/Variant.h"
+#include "Framework/WorkflowSpec.h"
+#include "DetectorsRaw/HBFUtilsInitializer.h"
+
+#include 
+
+using namespace o2::framework;
+using namespace o2::emcal;
+
+// add workflow options, note that customization needs to be declared before
+// including Framework/runDataProcessing
+void customize(std::vector& workflowOptions)
+{
+  std::vector options{
+    {"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}},
+    {"no-askdiststf", VariantType::Bool, false, {"Don't subscribe to FLP/DISTSUBTIMEFRAME (local testing)"}}};
+
+  std::swap(workflowOptions, options);
+}
+
+#include "Framework/runDataProcessing.h" // the main driver
+
+WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
+{
+
+  o2::conf::ConfigurableParam::updateFromString(cfgc.options().get("configKeyValues"));
+  bool askdiststf = !cfgc.options().get("no-askdiststf");
+
+  WorkflowSpec specs;
+  specs.emplace_back(getPedestalProcessorDevice(askdiststf));
+
+  // configure dpl timer to inject correct firstTForbit: start from the 1st orbit of TF containing 1st sampled orbit
+  // o2::raw::HBFUtilsInitializer hbfIni(cfgc, specs);
+  return specs;
+}
diff --git a/Detectors/EMCAL/doxymodules.h b/Detectors/EMCAL/doxymodules.h
index 9c2df257fc25c..90c822264f6cd 100644
--- a/Detectors/EMCAL/doxymodules.h
+++ b/Detectors/EMCAL/doxymodules.h
@@ -35,8 +35,13 @@
  * EMCAL calibration objects for
  * - Bad channel map
  * - Time calibration
+ * - Time slewing parameters
  * - Gain calibration
  * - Temperature calibration
+ * - FEE DCS parameters
+ * - Pedestal data
+ * In addition providing an interface convenient CCDB access and methods
+ * to recalibrate add cell level.
  */
 
 /**
@@ -44,8 +49,10 @@
  * @brief EMCAL bad channel calibration
  * @ingroup DetectorEMCAL
  *
- * Performs the EMCal bad channel calibration.
- *
+ * EMCAL calibrator performing
+ * - Bad channel calibration
+ * - Time calibration
+ * and corresponding workflows for calibration tasks.
  */
 
 /**
diff --git a/Detectors/EMCAL/reconstruction/CMakeLists.txt b/Detectors/EMCAL/reconstruction/CMakeLists.txt
index 0769e9b9ab51b..32aa4ccca67a5 100644
--- a/Detectors/EMCAL/reconstruction/CMakeLists.txt
+++ b/Detectors/EMCAL/reconstruction/CMakeLists.txt
@@ -12,12 +12,16 @@
 o2_add_library(EMCALReconstruction
         SOURCES src/RawReaderMemory.cxx
         src/RawBuffer.cxx
-        src/RawHeaderStream.cxx
         src/RawPayload.cxx
         src/AltroDecoder.cxx
         src/Bunch.cxx
         src/Channel.cxx
+        src/FastORTimeSeries.cxx
         src/RecoParam.cxx
+        src/RawDecodingError.cxx
+        src/STUDecoderError.cxx
+        src/ReconstructionErrors.cxx
+        src/RecoContainer.cxx
         src/CaloFitResults.cxx
         src/CaloRawFitter.cxx
         src/CaloRawFitterStandard.cxx
@@ -28,11 +32,13 @@ o2_add_library(EMCALReconstruction
         src/DigitReader.cxx
         src/CTFCoder.cxx
         src/CTFHelper.cxx
-        PUBLIC_LINK_LIBRARIES FairRoot::Base O2::Headers
+        src/StuDecoder.cxx
+        src/TRUDataHandler.cxx
+        PUBLIC_LINK_LIBRARIES O2::Headers
         AliceO2::InfoLogger
         O2::DataFormatsEMCAL
-        O2::DetectorsBase
         O2::DetectorsRaw
+        O2::DetectorsBase
         O2::EMCALBase
         O2::rANS
         Microsoft.GSL::GSL)
@@ -45,6 +51,7 @@ o2_target_root_dictionary(
         include/EMCALReconstruction/RawPayload.h
         include/EMCALReconstruction/Bunch.h
         include/EMCALReconstruction/Channel.h
+        include/EMCALReconstruction/FastORTimeSeries.h
         include/EMCALReconstruction/CaloFitResults.h
         include/EMCALReconstruction/CaloRawFitter.h
         include/EMCALReconstruction/CaloRawFitterStandard.h
@@ -54,6 +61,9 @@ o2_target_root_dictionary(
         include/EMCALReconstruction/ClusterizerTask.h
         include/EMCALReconstruction/DigitReader.h
         include/EMCALReconstruction/RecoParam.h
+        include/EMCALReconstruction/StuDecoder.h
+        include/EMCALReconstruction/TRUDataHandler.h
+        include/EMCALReconstruction/TRUDecodingErrors.h
 )
 
 o2_add_executable(rawreader-file
@@ -61,6 +71,48 @@ o2_add_executable(rawreader-file
         PUBLIC_LINK_LIBRARIES O2::EMCALReconstruction
         SOURCES run/rawReaderFile.cxx)
 
+o2_add_test(AltroDecoderError
+        SOURCES test/testAltroDecoderError.cxx
+        PUBLIC_LINK_LIBRARIES O2::EMCALReconstruction
+        COMPONENT_NAME emcal
+        LABELS emcal)
+
+o2_add_test(MinorAltroDecodingError
+        SOURCES test/testMinorAltroDecodingError.cxx
+        PUBLIC_LINK_LIBRARIES O2::EMCALReconstruction
+        COMPONENT_NAME emcal
+        LABELS emcal)
+
+o2_add_test(CaloRawFitterError
+        SOURCES test/testCaloRawFitterError.cxx
+        PUBLIC_LINK_LIBRARIES O2::EMCALReconstruction
+        COMPONENT_NAME emcal
+        LABELS emcal)
+
+o2_add_test(RawDecodingError
+        SOURCES test/testRawDecodingError.cxx
+        PUBLIC_LINK_LIBRARIES O2::EMCALReconstruction
+        COMPONENT_NAME emcal
+        LABELS emcal)
+
+o2_add_test(RecoContainer
+        SOURCES test/testRecoContainer.cxx
+        PUBLIC_LINK_LIBRARIES O2::EMCALReconstruction
+        COMPONENT_NAME emcal
+        LABELS emcal)
+
+o2_add_test(TRUDataHandler
+        SOURCES test/testTRUDataHandler.cxx
+        PUBLIC_LINK_LIBRARIES O2::EMCALReconstruction
+        COMPONENT_NAME emcal
+        LABELS emcal)
+
+o2_add_test(FastORTimeSeries
+        SOURCES test/testFastORTimeSeries.cxx
+        PUBLIC_LINK_LIBRARIES O2::EMCALReconstruction
+        COMPONENT_NAME emcal
+        LABELS emcal)
+
 o2_add_test_root_macro(macros/RawFitterTESTs.C
         PUBLIC_LINK_LIBRARIES O2::EMCALReconstruction O2::Headers
         LABELS emcal COMPILE_ONLY)
diff --git a/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/AltroDecoder.h b/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/AltroDecoder.h
index 87cac7012e5d2..be661a105395d 100644
--- a/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/AltroDecoder.h
+++ b/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/AltroDecoder.h
@@ -11,10 +11,12 @@
 #ifndef ALICEO2_EMCAL_ALTRODECODER_H
 #define ALICEO2_EMCAL_ALTRODECODER_H
 
+#include 
 #include 
 #include 
 #include 
 #include 
+#include 
 #include "EMCALBase/RCUTrailer.h"
 #include "EMCALReconstruction/Bunch.h"
 #include "EMCALReconstruction/Channel.h"
@@ -48,7 +50,10 @@ class AltroDecoderError : public std::exception
   ///
   /// Defining error code and error message. To be called when the
   /// exception is thrown
-  AltroDecoderError(ErrorType_t errtype, const char* message) : mErrorType(errtype), mErrorMessage(message) {}
+  ///
+  /// \param errtype Type of the error
+  /// \param message Error message related to the error
+  AltroDecoderError(ErrorType_t errtype, const std::string_view message) : mErrorType(errtype), mErrorMessage(message) {}
 
   /// \brief Destructor
   ~AltroDecoderError() noexcept override = default;
@@ -73,6 +78,69 @@ class AltroDecoderError : public std::exception
   /// \return Error type
   const ErrorType_t getErrorType() const noexcept { return mErrorType; }
 
+  /// \brief Get the name connected to the error type
+  ///
+  /// A single word descriptor i.e. used for object names
+  /// is produced.
+  ///
+  /// \param errortype Error type raising the exception (symbolic representation)
+  /// \return Name of the error type
+  static const char* getErrorTypeName(ErrorType_t errortype);
+
+  /// \brief Get the name connected to the error type
+  ///
+  /// A single word descriptor i.e. used for object names
+  /// is produced.
+  ///
+  /// \param errortype Error type raising the exception (numeric representation)
+  /// \return Name of the error type
+  static const char* getErrorTypeName(unsigned int errortype)
+  {
+    return getErrorTypeName(intToErrorType(errortype));
+  }
+
+  /// \brief Get the title connected to the error type
+  ///
+  /// A short description i.e. used for bin labels or histogam titles
+  /// is produced.
+  ///
+  /// \param errortype Error type raising the exception (symbolic representation)
+  /// \return Title of the error type
+  static const char* getErrorTypeTitle(ErrorType_t errortype);
+
+  /// \brief Get the title connected to the error type
+  ///
+  /// A short description i.e. used for bin labels or histogam titles
+  /// is produced.
+  ///
+  /// \param errortype Error type raising the exception (numeric representation)
+  /// \return Title of the error type
+  static const char* getErrorTypeTitle(unsigned int errortype)
+  {
+    return getErrorTypeTitle(intToErrorType(errortype));
+  }
+
+  /// \brief Get the description connected to the error type
+  ///
+  /// A detailed description i.e. used for error message on the stdout
+  /// is produced.
+  ///
+  /// \param errortype Error type raising the exception (symbolic representation)
+  /// \return Description connected to the error type
+  static const char* getErrorTypeDescription(ErrorType_t errortype);
+
+  /// \brief Get the description connected to the error type
+  ///
+  /// A detailed description i.e. used for error message on the stdout
+  /// is produced.
+  ///
+  /// \param errortype Error type raising the exception (numeric representation)
+  /// \return Description connected to the error type
+  static const char* getErrorTypeDescription(unsigned int errortype)
+  {
+    return getErrorTypeDescription(intToErrorType(errortype));
+  }
+
  private:
   ErrorType_t mErrorType;    ///< Code of the decoding error type
   std::string mErrorMessage; ///< Message connected to the error type
@@ -88,9 +156,13 @@ class MinorAltroDecodingError
   /// \brief Error codes connected with the ALTRO decoding
   enum class ErrorType_t {
     BUNCH_HEADER_NULL,            ///< Bunch header is 0
+    CHANNEL_HEADER,               ///< Channel header corruption
     CHANNEL_END_PAYLOAD_UNEXPECT, ///< Unexpected end of payload (channel or trailer word in bunch words)
     CHANNEL_PAYLOAD_EXCEED,       ///< Exceeding channel payload block
-    BUNCH_LENGTH_EXCEED           ///< Bunch length exceeding channel payload size
+    CHANNEL_ORDER,                ///< Channels not in increasing order
+    BUNCH_LENGTH_EXCEED,          ///< Bunch length exceeding channel payload size
+    BUNCH_LENGTH_ALLOW_EXCEED,    ///< Exceeds maximum allowed bunch length
+    BUNCH_STARTTIME               ///< Bunch start time exceeding
   };
 
   /// \brief Dummy constructor
@@ -139,7 +211,70 @@ class MinorAltroDecodingError
 
   /// \brief Get the number of error types handled by the AltroDecoderError
   /// \return Number of error types
-  static constexpr int getNumberOfErrorTypes() noexcept { return 2; }
+  static constexpr int getNumberOfErrorTypes() noexcept { return 8; }
+
+  /// \brief Get the name connected to the error type
+  ///
+  /// A single word descriptor i.e. used for object names
+  /// is produced.
+  ///
+  /// \param errortype Error type raising the exception (symbolic representation)
+  /// \return Name of the error type
+  static const char* getErrorTypeName(ErrorType_t errortype);
+
+  /// \brief Get the name connected to the error type
+  ///
+  /// A single word descriptor i.e. used for object names
+  /// is produced.
+  ///
+  /// \param errortype Error type raising the exception (numeric representation)
+  /// \return Name of the error type
+  static const char* getErrorTypeName(unsigned int errortype)
+  {
+    return getErrorTypeName(intToErrorType(errortype));
+  }
+
+  /// \brief Get the title connected to the error type
+  ///
+  /// A short description i.e. used for bin labels or histogam titles
+  /// is produced.
+  ///
+  /// \param errortype Error type raising the exception (symbolic representation)
+  /// \return Title of the error type
+  static const char* getErrorTypeTitle(ErrorType_t errortype);
+
+  /// \brief Get the title connected to the error type
+  ///
+  /// A short description i.e. used for bin labels or histogam titles
+  /// is produced.
+  ///
+  /// \param errortype Error type raising the exception (numeric representation)
+  /// \return Title of the error type
+  static const char* getErrorTypeTitle(unsigned int errortype)
+  {
+    return getErrorTypeTitle(intToErrorType(errortype));
+  }
+
+  /// \brief Get the description connected to the error type
+  ///
+  /// A detailed description i.e. used for error message on the stdout
+  /// is produced.
+  ///
+  /// \param errortype Error type raising the exception (symbolic representation)
+  /// \return Description connected to the error type
+  static const char* getErrorTypeDescription(ErrorType_t errortype);
+
+  /// \brief Get the description connected to the error type
+  ///
+  /// A detailed description i.e. used for error message on the stdout
+  /// is produced.
+  ///
+  /// \param errortype Error type raising the exception (numeric representation)
+  /// \return Description connected to the error type
+  static const char* getErrorTypeDescription(unsigned int errortype)
+  {
+    return getErrorTypeDescription(intToErrorType(errortype));
+  }
 
  private:
   ErrorType_t mErrorType;  ///< Type of the error
@@ -172,6 +307,13 @@ class AltroDecoder
   /// \brief Destructor
   ~AltroDecoder() = default;
 
+  /// \brief Set the max. allowed bunch length
+  /// \param maxBunchLength Max. allowed bunch lengths
+  ///
+  /// Rejects bunches in case the decoded bunch length or the
+  /// decoded start time exceeds the maximum allowed bunch length.
+  void setMaxBunchLength(int maxBunchLength) { mMaxBunchLength = maxBunchLength; }
+
   /// \brief Decode the ALTRO stream
   /// \throw AltroDecoderError if the RCUTrailer or ALTRO payload cannot be decoded
   ///
@@ -211,15 +353,57 @@ class AltroDecoder
   /// In case of failure an exception is thrown.
   void checkRCUTrailer();
 
+  /// \brief Check hardware address in channel header for consistency
+  /// \param hwaddress Hardware address to check
+  /// \return True if the channel is consistent (branch, FEC and Altro in expected range) - false otherwise
+  bool checkChannelHWAddress(int hwaddress);
+
   RawReaderMemory& mRawReader;                               ///< underlying raw reader
   RCUTrailer mRCUTrailer;                                    ///< RCU trailer
   std::vector mChannels;                            ///< vector of channels in the raw stream
   std::vector mMinorDecodingErrors; ///< Container for minor (non-crashing) errors
   bool mChannelsInitialized = false;                         ///< check whether the channels are initialized
+  unsigned int mMaxBunchLength = UINT_MAX;                   ///< Max bunch length
 
   ClassDefNV(AltroDecoder, 1);
 };
 
+/// \brief Stream operator of the AltroDecoderError
+///
+/// Printing error.what()
+///
+/// \param stream Stream to print on
+/// \param error Error to be displayed
+/// \return Stream after printing
+std::ostream& operator<<(std::ostream& stream, const AltroDecoderError& error);
+
+/// \brief Stream operator of AltroDecoderError's ErrorType_t
+///
+/// Prining name of the error type
+///
+/// \param stream Stream to print on
+/// \param error Error type to be displayed
+/// \return Stream after printing
+std::ostream& operator<<(std::ostream& stream, const AltroDecoderError::ErrorType_t& errortype);
+
+/// \brief Stream operator of the MinorAltroDecodingError
+///
+/// Printing error.what()
+///
+/// \param stream Stream to print on
+/// \param error Error to be displayed
+/// \return Stream after printing
+std::ostream& operator<<(std::ostream& stream, const MinorAltroDecodingError& error);
+
+/// \brief Stream operator of MinorAltroDecodingError's ErrorType_t
+///
+/// Prining name of the error type
+///
+/// \param stream Stream to print on
+/// \param error Error type to be displayed
+/// \return Stream after printing
+std::ostream& operator<<(std::ostream& stream, const MinorAltroDecodingError::ErrorType_t& errortype);
+
 } // namespace emcal
 
 } // namespace o2
diff --git a/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/CTFCoder.h b/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/CTFCoder.h
index e49763c69c479..706b9bf8138a6 100644
--- a/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/CTFCoder.h
+++ b/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/CTFCoder.h
@@ -23,7 +23,6 @@
 #include "DataFormatsEMCAL/CTF.h"
 #include "DetectorsCommonDataFormats/DetID.h"
 #include "DetectorsBase/CTFCoderBase.h"
-#include "rANS/rans.h"
 #include "EMCALReconstruction/CTFHelper.h"
 
 class TTree;
@@ -33,10 +32,10 @@ namespace o2
 namespace emcal
 {
 
-class CTFCoder : public o2::ctf::CTFCoderBase
+class CTFCoder final : public o2::ctf::CTFCoderBase
 {
  public:
-  CTFCoder(o2::ctf::CTFCoderBase::OpType op) : o2::ctf::CTFCoderBase(op, CTF::getNBlocks(), o2::detectors::DetID::EMC) {}
+  CTFCoder(o2::ctf::CTFCoderBase::OpType op, const std::string& ctfdictOpt = "none") : o2::ctf::CTFCoderBase(op, CTF::getNBlocks(), o2::detectors::DetID::EMC, 1.f, ctfdictOpt) {}
   ~CTFCoder() final = default;
 
   /// entropy-encode data to buffer with CTF
@@ -50,26 +49,51 @@ class CTFCoder : public o2::ctf::CTFCoderBase
   void createCoders(const std::vector& bufVec, o2::ctf::CTFCoderBase::OpType op) final;
 
  private:
+  template 
+  o2::ctf::CTFIOSize encode_impl(VEC& buff, const gsl::span& trigData, const gsl::span& cellData);
   void appendToTree(TTree& tree, CTF& ec);
   void readFromTree(TTree& tree, int entry, std::vector& trigVec, std::vector& cellVec);
+  void assignDictVersion(o2::ctf::CTFDictHeader& h) const final;
+  std::vector mTrgDataFilt;
+  std::vector mCellDataFilt;
 };
 
 /// entropy-encode clusters to buffer with CTF
 template 
 o2::ctf::CTFIOSize CTFCoder::encode(VEC& buff, const gsl::span& trigData, const gsl::span& cellData)
+{
+  if (mIRFrameSelector.isSet()) { // preselect data
+    mTrgDataFilt.clear();
+    mCellDataFilt.clear();
+    for (const auto& trig : trigData) {
+      if (mIRFrameSelector.check(trig.getBCData()) >= 0) {
+        mTrgDataFilt.push_back(trig);
+        auto cellIt = cellData.begin() + trig.getFirstEntry();
+        auto& trigC = mTrgDataFilt.back();
+        trigC.setDataRange((int)mCellDataFilt.size(), trig.getNumberOfObjects());
+        std::copy(cellIt, cellIt + trig.getNumberOfObjects(), std::back_inserter(mCellDataFilt));
+      }
+    }
+    return encode_impl(buff, mTrgDataFilt, mCellDataFilt);
+  }
+  return encode_impl(buff, trigData, cellData);
+}
+
+template 
+o2::ctf::CTFIOSize CTFCoder::encode_impl(VEC& buff, const gsl::span& trigData, const gsl::span& cellData)
 {
   using MD = o2::ctf::Metadata::OptStore;
   // what to do which each field: see o2::ctd::Metadata explanation
   constexpr MD optField[CTF::getNBlocks()] = {
-    MD::EENCODE, // BLC_bcIncTrig
-    MD::EENCODE, // BLC_orbitIncTrig
-    MD::EENCODE, // BLC_entriesTrig
-    MD::EENCODE, // BLC_towerID
-    MD::EENCODE, // BLC_time
-    MD::EENCODE, // BLC_energy
-    MD::EENCODE, // BLC_status
+    MD::EENCODE_OR_PACK, // BLC_bcIncTrig
+    MD::EENCODE_OR_PACK, // BLC_orbitIncTrig
+    MD::EENCODE_OR_PACK, // BLC_entriesTrig
+    MD::EENCODE_OR_PACK, // BLC_towerID
+    MD::EENCODE_OR_PACK, // BLC_time
+    MD::EENCODE_OR_PACK, // BLC_energy
+    MD::EENCODE_OR_PACK, // BLC_status
     // extra slot was added in the end
-    MD::EENCODE // BLC_trigger
+    MD::EENCODE_OR_PACK // BLC_trigger
   };
 
   CTFHelper helper(trigData, cellData);
@@ -83,11 +107,10 @@ o2::ctf::CTFIOSize CTFCoder::encode(VEC& buff, const gsl::spansetHeader(helper.createHeader());
   assignDictVersion(static_cast(ec->getHeader()));
-  ec->getANSHeader().majorVersion = 0;
-  ec->getANSHeader().minorVersion = 1;
+  ec->setANSHeader(mANSVersion);
   // at every encoding the buffer might be autoexpanded, so we don't work with fixed pointer ec
   o2::ctf::CTFIOSize iosize;
-#define ENCODEEMC(beg, end, slot, bits) CTF::get(buff.data())->encode(beg, end, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)].get(), getMemMarginFactor());
+#define ENCODEEMC(beg, end, slot, bits) CTF::get(buff.data())->encode(beg, end, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)], getMemMarginFactor());
   // clang-format off
   iosize += ENCODEEMC(helper.begin_bcIncTrig(),    helper.end_bcIncTrig(),     CTF::BLC_bcIncTrig,    0);
   iosize += ENCODEEMC(helper.begin_orbitIncTrig(), helper.end_orbitIncTrig(),  CTF::BLC_orbitIncTrig, 0);
@@ -113,12 +136,13 @@ o2::ctf::CTFIOSize CTFCoder::decode(const CTF::base& ec, VTRG& trigVec, VCELL& c
   const auto& header = ec.getHeader();
   checkDictVersion(static_cast(header));
   ec.print(getPrefix(), mVerbosity);
-  std::vector bcInc, entries, energy, cellTime, tower, trigger;
-  std::vector orbitInc;
+  std::vector orbitInc;
+  std::vector bcInc;
+  std::vector entries, energy, cellTime, tower, trigger;
   std::vector status;
 
   o2::ctf::CTFIOSize iosize;
-#define DECODEEMCAL(part, slot) ec.decode(part, int(slot), mCoders[int(slot)].get())
+#define DECODEEMCAL(part, slot) ec.decode(part, int(slot), mCoders[int(slot)])
   // clang-format off
   iosize += DECODEEMCAL(bcInc,       CTF::BLC_bcIncTrig);
   iosize += DECODEEMCAL(orbitInc,    CTF::BLC_orbitIncTrig);
@@ -142,10 +166,17 @@ o2::ctf::CTFIOSize CTFCoder::decode(const CTF::base& ec, VTRG& trigVec, VCELL& c
   trigVec.reserve(header.nTriggers);
   status.reserve(header.nCells);
 
+  Cell::EncoderVersion encodingversion = o2::emcal::Cell::EncoderVersion::EncodingV0;
+  if (header.majorVersion == 1 && header.minorVersion == 1) {
+    encodingversion = o2::emcal::Cell::EncoderVersion::EncodingV1;
+  } else if (header.majorVersion == 1 && header.minorVersion == 2) {
+    encodingversion = o2::emcal::Cell::EncoderVersion::EncodingV2;
+  }
+
   uint32_t firstEntry = 0, cellCount = 0;
   o2::InteractionRecord ir(header.firstBC, header.firstOrbit);
-
-  Cell cell;
+  bool checkIROK = (mBCShift == 0); // need to check if CTP offset correction does not make the local time negative ?
+  // Cell cell;
   TriggerRecord trg;
   for (uint32_t itrig = 0; itrig < header.nTriggers; itrig++) {
     // restore TrigRecord
@@ -155,14 +186,19 @@ o2::ctf::CTFIOSize CTFCoder::decode(const CTF::base& ec, VTRG& trigVec, VCELL& c
     } else {
       ir.bc += bcInc[itrig];
     }
-
+    if (checkIROK || canApplyBCShift(ir)) { // correction will be ok
+      checkIROK = true;
+    } else { // correction would make IR prior to mFirstTFOrbit, skip
+      cellCount += entries[itrig];
+      continue;
+    }
     firstEntry = cellVec.size();
+
     for (uint16_t ic = 0; ic < entries[itrig]; ic++) {
-      cell.setPacked(tower[cellCount], cellTime[cellCount], energy[cellCount], status[cellCount]);
-      cellVec.emplace_back(cell);
+      cellVec.emplace_back(tower[cellCount], energy[cellCount], cellTime[cellCount], status[cellCount], encodingversion);
       cellCount++;
     }
-    trg.setBCData(ir);
+    trg.setBCData(ir - mBCShift);
     trg.setDataRange(firstEntry, entries[itrig]);
     trg.setTriggerBitsCompressed(trigger[itrig]);
     trigVec.emplace_back(trg);
diff --git a/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/CTFHelper.h b/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/CTFHelper.h
index 5994a715b898d..b75dac976b47d 100644
--- a/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/CTFHelper.h
+++ b/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/CTFHelper.h
@@ -50,7 +50,7 @@ class CTFHelper
   class _Iter
   {
    public:
-    using difference_type = int64_t;
+    using difference_type = std::ptrdiff_t;
     using value_type = T;
     using pointer = const T*;
     using reference = const T&;
@@ -59,52 +59,98 @@ class CTFHelper
     _Iter(const gsl::span& data, bool end = false) : mData(data), mIndex(end ? data.size() : 0){};
     _Iter() = default;
 
-    const I& operator++()
+    inline I& operator++() noexcept
     {
       ++mIndex;
-      return (I&)(*this);
+      return static_cast(*this);
     }
 
-    const I& operator--()
+    inline I operator++(int)
+    {
+      I res = *(static_cast(this));
+      ++mIndex;
+      return res;
+    }
+
+    inline I& operator--() noexcept
     {
       mIndex--;
-      return (I&)(*this);
+      return static_cast(*this);
+    }
+
+    inline I operator--(int)
+    {
+      I res = *(static_cast(this));
+      --mIndex;
+      return res;
+    }
+
+    I& operator+=(difference_type i) noexcept
+    {
+      mIndex += i;
+      return static_cast(*this);
     }
 
-    difference_type operator-(const I& other) const { return mIndex - other.mIndex; }
+    I operator+(difference_type i) const
+    {
+      I res = *(const_cast(static_cast(this)));
+      return res += i;
+    }
 
-    difference_type operator-(size_t idx) const { return mIndex - idx; }
+    I& operator-=(difference_type i) noexcept
+    {
+      mIndex -= i;
+      return static_cast(*this);
+    }
 
-    const I& operator-(size_t idx)
+    I operator-(difference_type i) const
     {
-      mIndex -= idx;
-      return (I&)(*this);
+      I res = *(const_cast(static_cast(this)));
+      return res -= i;
     }
 
-    bool operator!=(const I& other) const { return mIndex != other.mIndex; }
-    bool operator==(const I& other) const { return mIndex == other.mIndex; }
-    bool operator>(const I& other) const { return mIndex > other.mIndex; }
-    bool operator<(const I& other) const { return mIndex < other.mIndex; }
+    difference_type operator-(const I& other) const noexcept { return mIndex - other.mIndex; }
+
+    inline friend I operator+(difference_type i, const I& iter) { return iter + i; };
+
+    bool operator!=(const I& other) const noexcept { return mIndex != other.mIndex; }
+    bool operator==(const I& other) const noexcept { return mIndex == other.mIndex; }
+    bool operator>(const I& other) const noexcept { return mIndex > other.mIndex; }
+    bool operator<(const I& other) const noexcept { return mIndex < other.mIndex; }
+    bool operator>=(const I& other) const noexcept { return mIndex >= other.mIndex; }
+    bool operator<=(const I& other) const noexcept { return mIndex <= other.mIndex; }
 
    protected:
     gsl::span mData{};
-    size_t mIndex = 0;
+    difference_type mIndex = 0;
   };
 
   //_______________________________________________
   // BC difference wrt previous if in the same orbit, otherwise the abs.value.
   // For the very 1st entry return 0 (diff wrt 1st BC in the CTF header)
-  class Iter_bcIncTrig : public _Iter
+  class Iter_bcIncTrig : public _Iter
   {
    public:
-    using _Iter::_Iter;
+    using _Iter::_Iter;
     value_type operator*() const
     {
       if (mIndex) {
         if (mData[mIndex].getBCData().orbit == mData[mIndex - 1].getBCData().orbit) {
-          return mData[mIndex].getBCData().bc - mData[mIndex - 1].getBCData().bc;
+          return value_type(mData[mIndex].getBCData().bc - mData[mIndex - 1].getBCData().bc);
+        } else {
+          return value_type(mData[mIndex].getBCData().bc);
+        }
+      }
+      return 0;
+    }
+    value_type operator[](difference_type i) const
+    {
+      size_t id = mIndex + i;
+      if (id) {
+        if (mData[id].getBCData().orbit == mData[id - 1].getBCData().orbit) {
+          return value_type(mData[id].getBCData().bc - mData[id - 1].getBCData().bc);
         } else {
-          return mData[mIndex].getBCData().bc;
+          return value_type(mData[id].getBCData().bc);
         }
       }
       return 0;
@@ -113,11 +159,16 @@ class CTFHelper
 
   //_______________________________________________
   // Orbit difference wrt previous. For the very 1st entry return 0 (diff wrt 1st BC in the CTF header)
-  class Iter_orbitIncTrig : public _Iter
+  class Iter_orbitIncTrig : public _Iter
   {
    public:
-    using _Iter::_Iter;
-    value_type operator*() const { return mIndex ? mData[mIndex].getBCData().orbit - mData[mIndex - 1].getBCData().orbit : 0; }
+    using _Iter::_Iter;
+    value_type operator*() const { return value_type(mIndex ? mData[mIndex].getBCData().orbit - mData[mIndex - 1].getBCData().orbit : 0); }
+    value_type operator[](difference_type i) const
+    {
+      size_t id = mIndex + i;
+      return value_type(id ? mData[id].getBCData().orbit - mData[id - 1].getBCData().orbit : 0);
+    }
   };
 
   //_______________________________________________
@@ -127,6 +178,7 @@ class CTFHelper
    public:
     using _Iter::_Iter;
     value_type operator*() const { return mData[mIndex].getNumberOfObjects(); }
+    value_type operator[](difference_type i) const { return mData[mIndex + i].getNumberOfObjects(); }
   };
 
   //_______________________________________________
@@ -136,6 +188,7 @@ class CTFHelper
    public:
     using _Iter::_Iter;
     value_type operator*() const { return mData[mIndex].getTriggerBitsCompressed(); }
+    value_type operator[](difference_type i) const { return mData[mIndex + i].getTriggerBitsCompressed(); }
   };
 
   //_______________________________________________
@@ -143,7 +196,8 @@ class CTFHelper
   {
    public:
     using _Iter::_Iter;
-    value_type operator*() const { return mData[mIndex].getPackedTowerID(); }
+    value_type operator*() const { return mData[mIndex].getTowerIDEncoded(); }
+    value_type operator[](difference_type i) const { return mData[mIndex + i].getTowerIDEncoded(); }
   };
 
   //_______________________________________________
@@ -151,7 +205,8 @@ class CTFHelper
   {
    public:
     using _Iter::_Iter;
-    value_type operator*() const { return mData[mIndex].getPackedTime(); }
+    value_type operator*() const { return mData[mIndex].getTimeStampEncoded(); }
+    value_type operator[](difference_type i) const { return mData[mIndex + i].getTimeStampEncoded(); }
   };
 
   //_______________________________________________
@@ -159,7 +214,8 @@ class CTFHelper
   {
    public:
     using _Iter::_Iter;
-    value_type operator*() const { return mData[mIndex].getPackedEnergy(); }
+    value_type operator*() const { return mData[mIndex].getEnergyEncoded(); }
+    value_type operator[](difference_type i) const { return mData[mIndex + i].getEnergyEncoded(); }
   };
 
   //_______________________________________________
@@ -167,7 +223,8 @@ class CTFHelper
   {
    public:
     using _Iter::_Iter;
-    value_type operator*() const { return mData[mIndex].getPackedCellStatus(); }
+    value_type operator*() const { return mData[mIndex].getCellTypeEncoded(); }
+    value_type operator[](difference_type i) const { return mData[mIndex + i].getCellTypeEncoded(); }
   };
 
   //<<< =========================== ITERATORS ========================================
diff --git a/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/CaloRawFitter.h b/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/CaloRawFitter.h
index ea0584650484d..e3086a23ea282 100644
--- a/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/CaloRawFitter.h
+++ b/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/CaloRawFitter.h
@@ -56,16 +56,87 @@ class CaloRawFitter
   /// \brief Create error message for a given error type
   /// \param fiterror Fit error type
   /// \return Error message connected to the error type
-  static std::string createErrorMessage(RawFitterError_t fiterror);
+  static std::string createErrorMessage(RawFitterError_t fiterror) { return getErrorTypeDescription(fiterror); }
 
   /// \brief Convert error type to numeric representation
   /// \param fiterror Fit error type
   /// \return Numeric representation of the raw fitter error
   static int getErrorNumber(RawFitterError_t fiterror);
 
+  /// \brief Convert numeric representation of error type to RawFitterError_t
+  ///
+  /// Expect the error code provided to be a valid error code.
+  ///
+  /// \param fiterror Numeric representation of fit error
+  /// \return Symbolic representation of the error code
+  static RawFitterError_t intToErrorType(unsigned int fiterror);
+
   /// \brief Get the number of raw fit error types supported
-  /// \return Number of error types (4)
-  static constexpr int getNumberOfErrorTypes() noexcept { return 4; }
+  /// \return Number of error types (5)
+  static constexpr int getNumberOfErrorTypes() noexcept { return 5; }
+
+  /// \brief Get the name connected to the fit error type
+  ///
+  /// A single word descriptor i.e. used for object names
+  /// is produced.
+  ///
+  /// \param errortype Error type raising the exception (symbolic representation)
+  /// \return Name of the fit error type
+  static const char* getErrorTypeName(RawFitterError_t fiterror);
+
+  /// \brief Get the name connected to the fit error type
+  ///
+  /// A single word descriptor i.e. used for object names
+  /// is produced.
+  ///
+  /// \param errortype Error type raising the exception (numeric representation)
+  /// \return Name of the fit error type
+  static const char* getErrorTypeName(unsigned int fiterror)
+  {
+    return getErrorTypeName(intToErrorType(fiterror));
+  }
+
+  /// \brief Get the title connected to the fit error type
+  ///
+  /// A short description i.e. used for bin labels or histogam titles
+  /// is produced.
+  ///
+  /// \param errortype Error type raising the exception (symbolic representation)
+  /// \return Title of the fit error type
+  static const char* getErrorTypeTitle(RawFitterError_t fiterror);
+
+  /// \brief Get the title connected to the fit error type
+  ///
+  /// A short description i.e. used for bin labels or histogam titles
+  /// is produced.
+  ///
+  /// \param errortype Error type raising the exception (numeric representation)
+  /// \return Title of the fit error type
+  static const char* getErrorTypeTitle(unsigned int fiterror)
+  {
+    return getErrorTypeTitle(intToErrorType(fiterror));
+  }
+
+  /// \brief Get the description connected to the fit error type
+  ///
+  /// A detailed description i.e. used for error message on the stdout
+  /// is produced.
+  ///
+  /// \param errortype Error type raising the exception (symbolic representation)
+  /// \return Description connected to the fit error type
+  static const char* getErrorTypeDescription(RawFitterError_t fiterror);
+
+  /// \brief Get the description connected to the fit error type
+  ///
+  /// A detailed description i.e. used for error message on the stdout
+  /// is produced.
+  ///
+  /// \param errortype Error type raising the exception (numeric representation)
+  /// \return Description connected to the fit error type
+  static const char* getErrorTypeDescription(unsigned int fiterror)
+  {
+    return getErrorTypeDescription(intToErrorType(fiterror));
+  }
 
   /// \brief Constructor
   CaloRawFitter(const char* name, const char* nameshort);
@@ -199,6 +270,12 @@ class CaloRawFitter
   ClassDefNV(CaloRawFitter, 1);
 }; // CaloRawFitter
 
+/// \brief Stream operator for CaloRawFitter's RawFitterError
+/// \param stream Stream to print on
+/// \param error Error code to be printed
+/// \return Stream after printing
+std::ostream& operator<<(std::ostream& stream, const CaloRawFitter::RawFitterError_t error);
+
 } // namespace emcal
 
 } // namespace o2
diff --git a/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/Clusterizer.h b/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/Clusterizer.h
index 994bfbfeea3d7..4d8620bd0db08 100644
--- a/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/Clusterizer.h
+++ b/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/Clusterizer.h
@@ -45,44 +45,112 @@ using ClusterIndex = int;
 template 
 class Clusterizer
 {
+  /// \struct cellWithE
+  /// \brief Wrapper structure to make cell sortable in energy
   struct cellWithE {
+
+    /// \brief Constructor
     cellWithE() : energy(0.), row(0), column(0) {}
+
+    /// \brief Constructor
+    /// \param e Energy (in GeV)
+    /// \param r Row number
+    /// \param c Column number
     cellWithE(float e, int r, int c) : energy(e), row(r), column(c) {}
-    // std::sort will require operator< to compile.
+
+    /// \brief Comparison lower operator comparing cells based on energy
+    ///
+    /// std::sort will require operator< to compile.
+    ///
+    /// \param rhs Cell to compare to
+    /// \return True if this cell is has a lower energy, false otherwise
     bool operator<(cellWithE const& rhs) const
     {
       return energy < rhs.energy;
     }
-    float energy;
-    int row;
-    int column;
+
+    float energy; ///< Energy (in GeV)
+    int row;      ///< Row number
+    int column;   ///< Column number
   };
 
+  /// \struct InputwithIndex
+  /// \brief Link of a cell object to a cluster index
   struct InputwithIndex {
-    const InputType* mInput;
-    ClusterIndex mIndex;
+    const InputType* mInput; ///< Input cell/digit object
+    ClusterIndex mIndex;     ///< index of the cluster
   };
 
  public:
+  /// \brief Main constructor
+  /// \param timeCut Max. time difference of cells in cluster in ns
+  /// \param timeMin Min. accepted cell time in ns
+  /// \param timeMax Max. accepted cell time in ns
+  /// \param gradientCut Min. gradient value allowed in cluster splitting
+  /// \param doEnergyGradientCut Apply gradient cut
+  /// \param thresholdSeedE Min. energy of seed cells in GeV
+  /// \param thresholdCellE Min. energy of associated cells in GeV
   Clusterizer(double timeCut, double timeMin, double timeMax, double gradientCut, bool doEnergyGradientCut, double thresholdSeedE, double thresholdCellE);
+
+  /// \brief Default constructor
   Clusterizer();
+
+  /// \brief Destructor
   ~Clusterizer() = default;
 
+  /// \brief Clear internal buffers of found clusters and cell indices
   void clear()
   {
     mFoundClusters.clear();
     mInputIndices.clear();
   }
+
+  /// \brief Initialize class member vars if not done in constructor
+  /// \param timeCut Max. time difference of cells in cluster in ns
+  /// \param timeMin Min. accepted cell time in ns
+  /// \param timeMax Max. accepted cell time in ns
+  /// \param gradientCut Min. gradient value allowed in cluster splitting
+  /// \param doEnergyGradientCut Apply gradient cut
+  /// \param thresholdSeedE Min. energy of seed cells in GeV
+  /// \param thresholdCellE Min. energy of associated cells in GeV
   void initialize(double timeCut, double timeMin, double timeMax, double gradientCut, bool doEnergyGradientCut, double thresholdSeedE, double thresholdCellE);
+
+  /// \brief Find clusters based on a give input collection.
+  ///
+  /// Start clustering from highest energy cell.
+  ///
+  /// \param inputArray Input collection of cells/digits
   void findClusters(const gsl::span& inputArray);
+
+  /// \brief Get list of found clusters
+  /// \return List of found clusters
   const std::vector* getFoundClusters() const { return &mFoundClusters; }
+
+  /// \brief Get list of found cell indices
+  /// \return List of found cell indices
   const std::vector* getFoundClustersInputIndices() const { return &mInputIndices; }
+
+  /// \brief Set EMCAL geometry
+  /// \param geometry Geometry pointer
   void setGeometry(Geometry* geometry) { mEMCALGeometry = geometry; }
+
+  /// \brief Get pointer to geometry
+  /// \return EMCAL geometry
   Geometry* getGeometry() { return mEMCALGeometry; }
 
  private:
-  void getClusterFromNeighbours(std::vector& clusterUnputs, int row, int column);
+  /// \brief Recursively search for neighbours (EMCAL)
+  /// \param[in,out] clusterInputs Cells/digits of prototype cluster
+  /// \param row Row number from neighbor search in recursion step
+  /// \param column Column number for neighbor search in recursion step
+  void getClusterFromNeighbours(std::vector& clusterInputs, int row, int column);
+
+  /// \brief Get row (phi) and column (eta) of a cell/digit, values corresponding to topology
+  /// \param input Input object (cell/digit)
+  /// \param[out] row Topological row
+  /// \param[out] column Topological column
   void getTopologicalRowColumn(const InputType& input, int& row, int& column);
+
   Geometry* mEMCALGeometry = nullptr;                             //! mSeedList;                 //!, NROWS> mInputMap; //!
+#include 
+#include "Rtypes.h"
+
+namespace o2::emcal
+{
+
+/// \class FastORTimeSeries
+/// \brief Container for FastOR time series
+/// \author Markus Fasel , Oak Ridge National Laboratory
+/// \ingroup EMCALreconstruction
+/// \since April 19, 2024
+///
+/// Time series are encoded in bunches in the raw data, which are usually time-reversed.
+/// The FastORTimeSeries handles the time series of all bunches in the readout window,
+/// in proper future-direction time order, correcting the time-reversal from the Fake-ALTRO.
+/// Consequently the ADC samples are expected in time-reversed format. The function
+/// calculateL1TimeSum calculates the timesum of the timeseries as 4-integral with respect to
+/// a given L0 time, which is expected at the end of the time integration range.
+class FastORTimeSeries
+{
+ public:
+  /// @brief Dummy constructor
+  FastORTimeSeries() = default;
+
+  /// \brief Construcor
+  /// \param maxsamples Maximum number of time samples
+  /// \param timesamples Time-reversed raw ADC samples
+  /// \param starttime Start time
+  FastORTimeSeries(int maxsamples, const gsl::span timesamples, uint8_t starttime)
+  {
+    setSize(maxsamples);
+    fillReversed(timesamples, starttime);
+  }
+
+  /// \brief Destructor
+  ~FastORTimeSeries() = default;
+
+  void setTimeSamples(const gsl::span timesamples, uint8_t starttime) { fillReversed(timesamples, starttime); }
+
+  /// \brief Calculate L0 timesum (4-integral of the ADC series) with respect to a given L0 time
+  /// \param l0time L0 time (end of the time series)
+  /// \return Timesum of the time series
+  uint16_t calculateL1TimeSum(uint8_t l0time) const;
+
+  /// \brief Access raw ADC values (in forward time order)
+  /// \return ADC values of the time series in forward time order
+  const gsl::span getADCs() const { return mTimeSamples; }
+
+  /// \brief Clear ADC samples in the time series
+  void clear();
+
+ private:
+  /// \brief Set the container size for the ADC samples
+  /// \param maxsamples Max. amount of samples to be handled
+  void setSize(int maxsamples);
+
+  /// \brief Fill the internal time samples in proper time order
+  /// \param timesamples Time-reversed time samples
+  /// \param starttime Start time
+  void fillReversed(const gsl::span timesamples, uint8_t starttime);
+
+  std::vector mTimeSamples; ///< Raw ADC time samples (in forward time order)
+
+  ClassDef(FastORTimeSeries, 1);
+};
+
+} // namespace o2::emcal
+
+#endif
\ No newline at end of file
diff --git a/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/RawDecodingError.h b/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/RawDecodingError.h
index 1f30b5848f91a..7d68b7bb7a03f 100644
--- a/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/RawDecodingError.h
+++ b/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/RawDecodingError.h
@@ -11,6 +11,8 @@
 #ifndef ALICEO2_EMCAL_RAWDECODINGERROR_H
 #define ALICEO2_EMCAL_RAWDECODINGERROR_H
 
+#include 
+#include 
 #include 
 
 namespace o2
@@ -42,7 +44,8 @@ class RawDecodingError : public std::exception
     HEADER_INVALID,     ///< Header in memory not belonging to requested superpage
     PAGE_START_INVALID, ///< Page position starting outside payload size
     PAYLOAD_INVALID,    ///< Payload in memory not belonging to requested superpage
-    TRAILER_DECODING    ///< Inconsistent trailer in memory (several trailer words missing the trailer marker)
+    TRAILER_DECODING,   ///< Inconsistent trailer in memory (several trailer words missing the trailer marker)
+    TRAILER_INCOMPLETE  ///< Incomplete trailer words (i.e. registers)
   };
 
   /// \brief Constructor
@@ -61,23 +64,7 @@ class RawDecodingError : public std::exception
   /// \return Error message of the exception
   const char* what() const noexcept override
   {
-    switch (mErrorType) {
-      case ErrorType_t::PAGE_NOTFOUND:
-        return "Page with requested index not found";
-      case ErrorType_t::HEADER_DECODING:
-        return "RDH of page cannot be decoded";
-      case ErrorType_t::PAYLOAD_DECODING:
-        return "Payload of page cannot be decoded";
-      case ErrorType_t::HEADER_INVALID:
-        return "Access to header not belonging to requested superpage";
-      case ErrorType_t::PAGE_START_INVALID:
-        return "Page decoding starting outside payload size";
-      case ErrorType_t::PAYLOAD_INVALID:
-        return "Access to payload not belonging to requested superpage";
-      case ErrorType_t::TRAILER_DECODING:
-        return "Inconsistent trailer in memory";
-    };
-    return "Undefined error";
+    return getErrorCodeDescription(mErrorType);
   }
 
   /// \brief Get the type identifier of the error handled with this exception
@@ -107,17 +94,171 @@ class RawDecodingError : public std::exception
         return 5;
       case ErrorType_t::TRAILER_DECODING:
         return 6;
+      case ErrorType_t::TRAILER_INCOMPLETE:
+        return 7;
     };
     // can never reach this, due to enum class
     // just to make Werror happy
     return -1;
   }
 
+  /// \brief Get the number of error codes
+  /// \return Number of error codes
+  static constexpr int getNumberOfErrorTypes() { return 8; }
+
+  static ErrorType_t intToErrorType(unsigned int errortype)
+  {
+    assert(errortype < getNumberOfErrorTypes());
+    static constexpr std::array errortypes = {{ErrorType_t::PAGE_NOTFOUND, ErrorType_t::HEADER_DECODING,
+                                                                                     ErrorType_t::PAYLOAD_DECODING, ErrorType_t::HEADER_INVALID,
+                                                                                     ErrorType_t::PAGE_START_INVALID, ErrorType_t::PAYLOAD_INVALID,
+                                                                                     ErrorType_t::TRAILER_DECODING, ErrorType_t::TRAILER_INCOMPLETE}};
+    return errortypes[errortype];
+  }
+
+  /// \brief Get name of error type
+  ///
+  /// A single word descriptor i.e. to be used in object names
+  /// is produced.
+  ///
+  /// \param errortype Error type raising the exception (symbolic representation)
+  /// \return Name of the error type
+  static const char* getErrorCodeNames(ErrorType_t errortype)
+  {
+    switch (errortype) {
+      case ErrorType_t::PAGE_NOTFOUND:
+        return "PageNotFound";
+      case ErrorType_t::HEADER_DECODING:
+        return "HeaderDecoding";
+      case ErrorType_t::PAYLOAD_DECODING:
+        return "PayloadDecoding";
+      case ErrorType_t::HEADER_INVALID:
+        return "HeaderCorruption";
+      case ErrorType_t::PAGE_START_INVALID:
+        return "PageStartInvalid";
+      case ErrorType_t::PAYLOAD_INVALID:
+        return "PayloadCorruption";
+      case ErrorType_t::TRAILER_DECODING:
+        return "TrailerDecoding";
+      case ErrorType_t::TRAILER_INCOMPLETE:
+        return "TrailerIncomplete";
+    };
+    return "Undefined error";
+  }
+
+  /// \brief Get name of error type
+  ///
+  /// A single word descriptor i.e. to be used in object names
+  /// is produced.
+  ///
+  /// \param errortype Error type raising the exception (numeric representation)
+  /// \return Name of the error type
+  static const char* getErrorCodeNames(unsigned int errortype)
+  {
+    return getErrorCodeNames(intToErrorType(errortype));
+  }
+
+  /// \brief Get title of error type
+  ///
+  /// A short description i.e. to be used in histogram titles
+  /// is produced.
+  ///
+  /// \param errortype Error type raising the exception (symbolic representation)
+  /// \return Title of the error type
+  static const char* getErrorCodeTitles(ErrorType_t errortype)
+  {
+    switch (errortype) {
+      case ErrorType_t::PAGE_NOTFOUND:
+        return "Page not found";
+      case ErrorType_t::HEADER_DECODING:
+        return "Header decoding";
+      case ErrorType_t::PAYLOAD_DECODING:
+        return "Payload decoding";
+      case ErrorType_t::HEADER_INVALID:
+        return "Header corruption";
+      case ErrorType_t::PAGE_START_INVALID:
+        return "Page start invalid";
+      case ErrorType_t::PAYLOAD_INVALID:
+        return "Payload corruption";
+      case ErrorType_t::TRAILER_DECODING:
+        return "Trailer decoding";
+      case ErrorType_t::TRAILER_INCOMPLETE:
+        return "Trailer incomplete";
+    };
+    return "Undefined error";
+  }
+
+  /// \brief Get title of error type
+  ///
+  /// A short description i.e. to be used in histogram titles
+  /// is produced.
+  ///
+  /// \param errortype Error type raising the exception (numeric representation)
+  /// \return Title of the error type
+  static const char* getErrorCodeTitles(unsigned int errortype)
+  {
+    return getErrorCodeTitles(intToErrorType(errortype));
+  }
+
+  /// \brief Get description of error type
+  ///
+  /// A dedicated description is created which can be used i.e. in the
+  /// what() function of the exception.
+  ///
+  /// \param errortype Error type raising the exceptio (symbolic representation)
+  /// \return Description text for the error type
+  static const char* getErrorCodeDescription(ErrorType_t errortype)
+  {
+    switch (errortype) {
+      case ErrorType_t::PAGE_NOTFOUND:
+        return "Page with requested index not found";
+      case ErrorType_t::HEADER_DECODING:
+        return "RDH of page cannot be decoded";
+      case ErrorType_t::PAYLOAD_DECODING:
+        return "Payload of page cannot be decoded";
+      case ErrorType_t::HEADER_INVALID:
+        return "Access to header not belonging to requested superpage";
+      case ErrorType_t::PAGE_START_INVALID:
+        return "Page decoding starting outside payload size";
+      case ErrorType_t::PAYLOAD_INVALID:
+        return "Access to payload not belonging to requested superpage";
+      case ErrorType_t::TRAILER_DECODING:
+        return "Inconsistent trailer in memory";
+      case ErrorType_t::TRAILER_INCOMPLETE:
+        return "Incomplete trailer";
+    };
+    return "Undefined error";
+  }
+
+  /// \brief Get description of error type
+  ///
+  /// A dedicated description is created which can be used i.e. in the
+  /// what() function of the exception.
+  ///
+  /// \param errortype Error type raising the exception (numeric representation)
+  /// \return Description text for the error type
+  static const char* getErrorCodeDescription(unsigned int errortype)
+  {
+    return getErrorCodeDescription(intToErrorType(errortype));
+  }
+
  private:
   ErrorType_t mErrorType; ///< Type of the error
   int mFecID;             ///< ID of the FEC responsible for the ERROR
 };
 
+/// \brief Streaming operator for RawDecodingError
+/// \param stream Stream to print on
+/// \param error Error to be printed
+/// \return Stream after printing
+std::ostream& operator<<(std::ostream& stream, const RawDecodingError& error);
+
+/// \brief Streaming operator for RawDecodingError's ErrorType_t
+/// \param stream Stream to print on
+/// \param error Error to be printed
+/// \return Stream after printing
+std::ostream& operator<<(std::ostream& stream, const RawDecodingError::ErrorType_t& error);
+
 } // namespace emcal
 
 } // namespace o2
diff --git a/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/RawHeaderStream.h b/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/RawHeaderStream.h
deleted file mode 100644
index 4f898d66934ad..0000000000000
--- a/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/RawHeaderStream.h
+++ /dev/null
@@ -1,39 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-/// \file RawHeaderStream.h
-/// \brief Input stream operators for raw header 4 and 5 from binary file
-///
-/// Helpers to define input stream operator for raw headers v4 and v5 from
-/// binary file input stream, used in RawReaderFile
-
-#ifndef ALICEO2_EMCAL_RAWHEADERSTREAM_H
-#define ALICEO2_EMCAL_RAWHEADERSTREAM_H
-
-#include 
-#include "Headers/RAWDataHeader.h"
-
-namespace o2
-{
-
-namespace emcal
-{
-
-std::istream& operator>>(std::istream& stream, o2::header::RAWDataHeaderV4& header);
-std::istream& operator>>(std::istream& stream, o2::header::RAWDataHeaderV5& header);
-
-std::ostream& operator<<(std::ostream& stream, const o2::header::RAWDataHeaderV4& header);
-std::ostream& operator<<(std::ostream& stream, const o2::header::RAWDataHeaderV5& header);
-
-} // namespace emcal
-
-} // namespace o2
-#endif
diff --git a/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/RawReaderMemory.h b/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/RawReaderMemory.h
index 843bda5f8398b..f726d1307fb90 100644
--- a/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/RawReaderMemory.h
+++ b/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/RawReaderMemory.h
@@ -11,11 +11,13 @@
 #ifndef ALICEO2_EMCAL_RAWREADERMEMORY_H
 #define ALICEO2_EMCAL_RAWREADERMEMORY_H
 
+#include 
 #include 
 #include 
 
 #include "EMCALBase/RCUTrailer.h"
 #include "EMCALReconstruction/RawBuffer.h"
+#include "EMCALReconstruction/RawDecodingError.h"
 #include "EMCALReconstruction/RawPayload.h"
 #include "Headers/RAWDataHeader.h"
 #include "Headers/RDHAny.h"
@@ -36,6 +38,46 @@ namespace emcal
 class RawReaderMemory
 {
  public:
+  /// \class MinorError
+  /// \brief Minor (non-crashing) raw decoding errors
+  ///
+  /// Minor errors share the same codes as major raw decoding errors,
+  /// however are not crashing.
+  class MinorError
+  {
+   public:
+    /// \brief Dummy constructor
+    MinorError() = default;
+
+    /// \brief Main constructor
+    /// \param errortype Type of the error
+    /// \param feeID ID of the FEE equipment
+    MinorError(RawDecodingError::ErrorType_t errortype, int feeID) : mErrorType(errortype), mFEEID(feeID) {}
+
+    /// \brief Destructor
+    ~MinorError() = default;
+
+    /// \brief Set the type of the error
+    /// \param errortype Type of the error
+    void setErrorType(RawDecodingError::ErrorType_t errortype) { mErrorType = errortype; }
+
+    /// \brief Set the ID of the FEE equipment
+    /// \param feeID ID of the FEE
+    void setFEEID(int feeID) { mFEEID = feeID; }
+
+    /// \brief Get type of the error
+    /// \return Type of the error
+    RawDecodingError::ErrorType_t getErrorType() const { return mErrorType; }
+
+    /// \brief Get ID of the FEE
+    /// \return ID of the FEE
+    int getFEEID() const { return mFEEID; }
+
+   private:
+    RawDecodingError::ErrorType_t mErrorType; ///< Type of the error
+    int mFEEID;                               ///< ID of the FEC responsible for the ERROR
+  };
+
   /// \brief Constructor
   RawReaderMemory(const gsl::span rawmemory);
 
@@ -58,8 +100,7 @@ class RawReaderMemory
   /// \brief Read next payload from the stream
   ///
   /// Read the next pages until the stop bit is found.
-  void
-    next();
+  void next();
 
   /// \brief Read the next page from the stream (single DMA page)
   /// \param resetPayload If true the raw payload is reset
@@ -85,6 +126,10 @@ class RawReaderMemory
   /// \return Raw Payload of the data until the stop bit is received.
   const RawPayload& getPayload() const { return mRawPayload; }
 
+  /// \brief Get minor (non-crashing) raw decoding errors
+  /// \return Minor raw decoding errors
+  gsl::span getMinorErrors() const { return mMinorErrors; }
+
   /// \brief Return size of the payload
   /// \return size of the payload
   int getPayloadSize() const { return mRawPayload.getPayloadSize(); }
@@ -103,11 +148,10 @@ class RawReaderMemory
   /// Rewind stream to the first entry
   void init();
 
-  /// \brief Check whether the current page is accepted
-  /// \param page Raw page to check
-  /// \return True if the page is accepted, false otherwise
-  bool acceptPage(const char* page) const;
-
+  /// \brief Decode raw header words
+  /// \param headerwords Headerwords
+  /// \return Decoded RDH
+  /// \throw RawDecodingError with code HEADER_DECODING if the payload does not correspond to an expected header
   o2::header::RDHAny decodeRawHeader(const void* headerwords);
 
  private:
@@ -123,6 +167,7 @@ class RawReaderMemory
   int mCurrentFEE = -1;                   ///< Current FEE in the data stream
   bool mRawHeaderInitialized = false;     ///< RDH for current page initialized
   bool mPayloadInitialized = false;       ///< Payload for current page initialized
+  std::vector mMinorErrors;   ///< Minor raw decoding errors
 
   ClassDefNV(RawReaderMemory, 1);
 };
diff --git a/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/RecoContainer.h b/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/RecoContainer.h
new file mode 100644
index 0000000000000..dc3821810c4a5
--- /dev/null
+++ b/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/RecoContainer.h
@@ -0,0 +1,379 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file RecoContainer.h
+/// \brief Reconstruction container for EMCAL Cells and LEDMONs
+/// \author Markus Fasel , Oak Ridge National Laboratory
+/// \since May 30, 2023
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#ifndef ALICEO2_EMCAL_RECOCONTAINER_H
+#define ALICEO2_EMCAL_RECOCONTAINER_H
+
+namespace o2::emcal
+{
+/// \struct RecCellInfo
+/// \brief Container class for cell information for merging
+///
+/// In case the energy is in the overlap region between the
+/// two digitizers 2 channels exist for the same cell. In this
+/// case the low gain cells are used above a certain threshold.
+/// In certain error cases the information from the other digitizer
+/// might be missing. Such cases must be fitered out, however this
+/// can be done only after all cells are processed. The overlap information
+/// needs to be propagated for the filtering but is not part of the
+/// final cell object
+struct RecCellInfo {
+  o2::emcal::Cell mCellData; ///< Cell information
+  bool mIsLGnoHG;            ///< Cell has only LG digits
+  bool mHGOutOfRange;        ///< Cell has only HG digits which are out of range
+  int mDDLID;                ///< DDL of the channel (for monitoring)
+  int mHWAddressLG;          ///< HW address of LG (for monitoring)
+  int mHWAddressHG;          ///< HW address of HG (for monitoring)
+};
+
+/// \class EventContainer
+/// \brief Containter of cells for a given event
+/// \author Markus Fasel , Oak Ridge National Laboratory
+/// \ingroup EMCALreconstruction
+/// \since May 30, 2023
+///
+/// The EventContainer is part of the reco container and keeps the cell / LEDMON / TRU
+/// data of a given event (trigger) defined by the BC and orbit in the raw data header.
+/// In case the gain type merging is activated the event container performs on-the-fly
+/// merging of high- and low-gain data for cells and LEDMONS keeping only high- or
+/// low-gain and preferring the high-gain due to better resolution if not saturated.
+///
+/// Error handling:
+/// The EventContainer can raise a TRUIndexException in case TRU information with an
+/// unexpected index (>= 52) is added / requested.
+class EventContainer
+{
+ public:
+  /// \class TRUIndexException
+  /// \brief Handler for access of TRU data with invalid TRU index
+  /// \ingroup EMCALreconstruction
+  class TRUIndexException final : public std::exception
+  {
+   public:
+    /// \brief Constructor
+    /// \param index TRU index raising the exception
+    TRUIndexException(std::size_t index);
+
+    /// \brief Destructor
+    ~TRUIndexException() noexcept final = default;
+
+    /// \brief Get the error message of the exception
+    /// \return Error message
+    const char* what() const noexcept final { return mMessage.data(); }
+
+    /// \brief Get the TRU index raising the exception
+    /// \return TRU index
+    std::size_t getIndex() const { return mIndex; }
+
+    /// \brief Print error message on stream
+    /// \param stream Stream to print on
+    void printStream(std::ostream& stream) const;
+
+   private:
+    std::size_t mIndex;   ///< TRU index raising the exception
+    std::string mMessage; ///< Buffer for error message
+  };
+
+  /// \brief Constructor
+  EventContainer() = default;
+
+  /// \brief Constructor, setting interaction record
+  /// \param currentIR Interaction record of the trigger
+  EventContainer(const o2::InteractionRecord& currentIR);
+
+  /// \brief Destructor
+  ~EventContainer() = default;
+
+  /// \brief Set trigger bits of the interaction
+  /// \param triggerbits Trigger bits
+  void setTriggerBits(uint64_t triggerbits) { mTriggerBits = triggerbits; }
+
+  /// \brief Set interaction record
+  /// \param currentIR Interaction record of the trigger
+  void setInteractionRecord(const o2::InteractionRecord& currentIR) { mInteractionRecord = currentIR; }
+
+  /// \brief Get trigger bits of the interaction
+  /// \return Trigger bits
+  uint64_t getTriggerBits() const { return mTriggerBits; }
+
+  /// \brief Get interaction record of the event
+  /// \return Interaction record connected to this event
+  const o2::InteractionRecord& getInteractionRecord() const { return mInteractionRecord; }
+
+  /// \brief Get cells in container
+  /// \return List of cells in container
+  const gsl::span getCells() const { return mCells; }
+
+  /// \brief Get LEDMONs in container
+  /// \return List of LEDMONs
+  const gsl::span getLEDMons() const { return mLEDMons; }
+
+  /// \brief Get the number of cells in the event
+  /// \return Number of cells
+  int getNumberOfCells() const { return mCells.size(); }
+
+  /// \brief Get the number of LEDMONs in the event
+  /// \return Number of LEDMONs
+  int getNumberOfLEDMONs() const { return mLEDMons.size(); }
+
+  /// \brief Read and write access TRU data of a given TRU
+  /// \param truIndex Index of the TRU
+  /// \return TRU data handler for the TRU
+  /// \throw TRUIndexException in case the TRU index is invalid (>= 52)
+  TRUDataHandler& getTRUData(std::size_t truIndex);
+
+  /// \brief Read-only access TRU data of a given TRU
+  /// \param truIndex Index of the TRU
+  /// \return TRU data handler for the TRU
+  /// \throw TRUIndexException in case the TRU index is invalid (>= 52)
+  const TRUDataHandler& readTRUData(std::size_t truIndex) const;
+
+  /// \brief Access to container with FastOR time series
+  /// \return Container with time series
+  const std::unordered_map& getTimeSeriesContainer() const { return mL0FastORs; }
+
+  /// \brief Add cell information to the event container
+  /// \param tower Tower ID
+  /// \param energy Cell energy
+  /// \param time Cell time
+  /// \param celltype Cell type (high gain or low gain)
+  /// \param hwaddress Hardware address
+  /// \param ddlID ID of the DDL
+  /// \param doMergeHGLG If true merge with existing HG/LG cell
+  ///
+  /// In case of merge mode the priory is given to the HG digitizer (better resolution).
+  /// As long as the energy is not in the saturation region (approx 16 GeV) the HG is selected,
+  /// otherwise the LG digit is used.
+  void setCell(int tower, double energy, double time, ChannelType_t celltype, int hwaddress, int ddlID, bool doMergeHGLG)
+  {
+    setCellCommon(tower, energy, time, celltype, false, hwaddress, ddlID, doMergeHGLG);
+  }
+
+  /// \brief Add LEDMON information to the event container
+  /// \param tower LEDMON ID
+  /// \param energy LEDMON energy
+  /// \param time LEDMON time
+  /// \param celltype LEDMON type (high gain or low gain)
+  /// \param hwaddress Hardware address
+  /// \param ddlID ID of the DDL
+  /// \param doMergeHGLG If true merge with existing HG/LG LEDMON
+  ///
+  /// In case of merge mode the priory is given to the HG digitizer (better resolution).
+  /// As long as the energy is not in the saturation region (approx 16 GeV) the HG is selected,
+  /// otherwise the LG digit is used.
+  void setLEDMONCell(int tower, double energy, double time, ChannelType_t celltype, int hwaddress, int ddlID, bool doMergeHGLG)
+  {
+    setCellCommon(tower, energy, time, celltype, true, hwaddress, ddlID, doMergeHGLG);
+  }
+
+  /// \brief Add bunch of time series to the container
+  /// \param fastORAbsID Absolute ID of the FastOR
+  /// \param starttime Start time of the bunch
+  /// \param timesamples Time samples of the bunch in time-reversed format
+  ///
+  /// In case a TimeSeries is already present for the given FastOR abs. ID in the container
+  /// the bunch is added to this, otherwise a new TimeSeries is added with the ADCs of the
+  /// bunch.
+  void setFastOR(uint16_t fastORAbsID, uint8_t starttime, const gsl::span timesamples);
+
+  /// \brief Sort Cells / LEDMONs in container according to tower / module ID
+  /// \param isLEDmon Switch between Cell and LEDMON
+  void sortCells(bool isLEDmon);
+
+ private:
+  /// \brief Common handler for adding cell/LEDMON information to the event container
+  /// \param tower Tower / LEDMON ID
+  /// \param energy Energy
+  /// \param time Time
+  /// \param celltype Digitizer type (high gain or low gain)
+  /// \param hwaddress Hardware address
+  /// \param ddlID ID of the DDL
+  /// \param doMergeHGLG Switch for merge mode
+  void setCellCommon(int tower, double energy, double time, ChannelType_t celltype, bool isLEDmon, int hwaddress, int ddlID, bool doMergeHGLG);
+
+  /// \brief Check whether the energy is in the saturation limit
+  /// \return True if the energy is in the saturation region, false otherwise
+  bool isCellSaturated(double energy) const;
+
+  /// \brief Initialize the TRU handlers
+  void initTRUs();
+
+  o2::InteractionRecord mInteractionRecord;                       ///< Interaction record of the event
+  uint64_t mTriggerBits = 0;                                      ///< Trigger bits of the event
+  std::vector mCells;                                ///< Container of cells in event
+  std::vector mLEDMons;                              ///< Container of LEDMONs in event
+  std::array mTRUData; ///< TRU status
+  std::unordered_map mL0FastORs;      ///< L0 FastOR time series
+};
+
+/// \class RecoContainer
+/// \brief Handler for cells/LEDMONS/Trigger data in timeframes
+/// \author Markus Fasel , Oak Ridge National Laboratory
+/// \ingroup EMCALreconstruction
+/// \since May 30, 2023
+///
+/// The RecoContainer handles the cell/LEDMON/trigger data of all events of a given
+/// timeframe during the reconstruction. Event data are handled internally via the
+/// EventContainer, where the RecoContainer provides access to.
+///
+/// Error handling:
+/// The RecoContainer can raise an InteractionNotFoundException in case read access
+/// is requested for an interaction based on the o2::InteractionRecord which is not
+/// for which no data was inserted into the container.
+class RecoContainer
+{
+ public:
+  /// \class InteractionNotFoundException
+  /// \brief Handling of access to trigger interaction record not present in container
+  class InteractionNotFoundException final : public std::exception
+  {
+   public:
+    /// \brief Constructor
+    /// \param currentIR Interaction record raising the exception
+    InteractionNotFoundException(const o2::InteractionRecord& currentIR) : mCurrentIR(currentIR)
+    {
+      mMessage = "Interaction record not found: Orbit " + std::to_string(mCurrentIR.orbit) + ", BC " + std::to_string(mCurrentIR.bc);
+    }
+
+    /// \brief Destructor
+    ~InteractionNotFoundException() noexcept final = default;
+
+    /// \brief Get error message of the exception
+    /// \return Error message
+    const char* what() const noexcept final
+    {
+      return mMessage.data();
+    };
+
+    /// \brief Get interaction record raising the exception
+    /// \return Interaction record
+    const o2::InteractionRecord& getInteractionRecord() const { return mCurrentIR; }
+
+   private:
+    o2::InteractionRecord mCurrentIR; ///< Interaction record raising the exception
+    std::string mMessage;             ///< Error message
+  };
+
+  /// \brief Constructor
+  RecoContainer() = default;
+
+  /// \brief Destructor
+  ~RecoContainer() = default;
+
+  /// \brief Get container for trigger
+  /// \param currentIR Interaction record of the trigger
+  /// \return Container for trigger (creating new container if not yet present)
+  EventContainer& getEventContainer(const o2::InteractionRecord& currentIR);
+
+  /// \brief Get container for trigger (read-only)
+  /// \param currentIR Interaction record of the trigger
+  /// \return Container for trigger
+  /// \throw InteractionNotFoundException if interaction record is not present
+  const EventContainer& getEventContainer(const o2::InteractionRecord& currentIR) const;
+
+  /// \brief Get sorted vector interaction records of triggers in container
+  /// \return Sorted vector of container
+  std::vector getOrderedInteractions() const;
+
+  /// \brief Get number of events in container
+  /// \return Number of events
+  std::size_t getNumberOfEvents() const { return mEvents.size(); }
+
+  /// \brief Clear container
+  void reset() { mEvents.clear(); }
+
+ private:
+  std::unordered_map mEvents; ///< Containers in event
+};
+
+/// \class RecoContainerReader
+/// \brief Iterator over reco containers
+/// \author Markus Fasel , Oak Ridge National Laboratory
+/// \ingroup EMCALreconstruction
+/// \since May 30, 2023
+///
+/// The RecoContainerReader iterates over the events stored in the RecoContainer in
+/// a time-ordered sequence. The function hasNext checks whether there are more
+/// events to iterate over, while nextEvent provides access to the next event.
+///
+/// Error handling:
+/// The RecoContainerReader can raise an InvalidAccessException in case access is tried to
+/// invalid data.
+class RecoContainerReader
+{
+ public:
+  /// \class InvalidAccessException
+  /// \brief Handling of access to objects beyond container boundary
+  class InvalidAccessException : public std::exception
+  {
+   public:
+    /// \brief Constructor
+    InvalidAccessException() = default;
+
+    /// \brief Destructor
+    ~InvalidAccessException() noexcept final = default;
+
+    /// \brief Create error message
+    /// \return Error message
+    const char* what() const noexcept final { return "Access to invalid element in reco container"; }
+  };
+
+  /// \brief Constructor
+  /// \param container Container to be iterated over
+  RecoContainerReader(RecoContainer& container);
+  RecoContainerReader(RecoContainer&& container) = delete;
+
+  /// \brief Destructor
+  ~RecoContainerReader() = default;
+
+  /// \brief Get the next event in container
+  /// \return Next event in reco container (ordered)
+  /// \throw InvalidAccessException
+  EventContainer& nextEvent();
+
+  /// \brief Check whehter there are more events in the container
+  /// \return True if the event is not the last event, false otherwise.
+  bool hasNext() const { return mCurrentEvent < mOrderedInteractions.size(); }
+
+  /// \brief Get the number of events in the container
+  /// \return Number of events
+  std::size_t getNumberOfEvents() const { return mDataContainer.getNumberOfEvents(); }
+
+ private:
+  RecoContainer& mDataContainer;                           ///< Reference to container which is iterated over
+  std::vector mOrderedInteractions; ///< Ordered list of the interaction records
+  std::size_t mCurrentEvent = 0;                           ///< Index of the current event in the sorted event container
+};
+
+} // namespace o2::emcal
+
+#endif
\ No newline at end of file
diff --git a/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/RecoParam.h b/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/RecoParam.h
index dd6bdd8800fce..c675f57b3506a 100644
--- a/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/RecoParam.h
+++ b/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/RecoParam.h
@@ -27,22 +27,56 @@ namespace emcal
 class RecoParam : public o2::conf::ConfigurableParamHelper
 {
  public:
+  /// \brief Destructor
   ~RecoParam() override = default;
 
+  /// \brief Get the average cell time shift
+  /// \return Average cell time shift
   double getCellTimeShiftNanoSec() const { return mCellTimeShiftNanoSec; }
+
+  /// \brief Get noise threshold for LGnoHG error
+  /// \return Noise threshold
   double getNoiseThresholdLGnoHG() const { return mNoiseThresholdLGnoHG; }
+
+  /// \brief Get the BC phase
+  /// \return BC phase
   int getPhaseBCmod4() const { return mPhaseBCmod4; }
 
+  /// \brief Get the max. allowed bunch length
+  /// \return Max. allowed bunch length
+  ///
+  /// In case of max. bunch length 0 the max. bunch length is auto-determined from
+  /// the RCU trailer
+  int getMaxAllowedBunchLength() const { return mMaxBunchLength; }
+
+  /// \brief Print current reconstruction parameters to stream
+  /// \param stream Stream to print on
   void PrintStream(std::ostream& stream) const;
 
  private:
   double mNoiseThresholdLGnoHG = 10.;  ///< Noise threshold applied to suppress LGnoHG error
   double mCellTimeShiftNanoSec = 470.; ///< Time shift applied on the cell time to center trigger peak around 0
   int mPhaseBCmod4 = 1;                ///< Rollback of the BC ID in the correction of the cell time for the BC mod 4
+  unsigned int mMaxBunchLength = 15;   ///< Max. allowed bunch length
 
   O2ParamDef(RecoParam, "EMCRecoParam");
 };
-std::ostream& operator<<(std::ostream& stream, const RecoParam& s);
+
+/// \brief Streaming operator for the reconstruction parameters
+/// \param stream Stream to print on
+/// \param par RecoParams to be printed
+/// \return Stream after printing the reco params
+std::ostream& operator<<(std::ostream& stream, const RecoParam& par);
 } // namespace emcal
+
+namespace framework
+{
+template 
+struct is_messageable;
+template <>
+struct is_messageable : std::true_type {
+};
+} // namespace framework
+
 } // namespace o2
 #endif
\ No newline at end of file
diff --git a/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/ReconstructionErrors.h b/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/ReconstructionErrors.h
new file mode 100644
index 0000000000000..c46ef63b6f3ac
--- /dev/null
+++ b/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/ReconstructionErrors.h
@@ -0,0 +1,325 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#ifndef ALICEO2_EMCAL_RECONSTRUCTIONERRORS_H
+#define ALICEO2_EMCAL_RECONSTRUCTIONERRORS_H
+
+namespace o2
+{
+
+namespace emcal
+{
+
+namespace reconstructionerrors
+{
+
+/// \class GeometryError_t
+/// \brief Errors appearing in geometry access obtaining the tower ID
+/// \ingroup EMCALreconstruction
+///
+/// Errors can appear during geometry access, either because the
+/// cell number provided by the geometry is negative or because
+/// the cell number exceeds the range of allowed cell indices
+enum class GeometryError_t {
+  CELL_RANGE_EXCEED,   ///< Requested cell value exceeding allowed cell range
+  CELL_INDEX_NEGATIVE, ///< Requested cell value negative
+  UNKNOWN_ERROR        ///< Unknown error code
+};
+
+/// \brief Get the number of geometry error codes
+/// \return Number of geometry error codes
+constexpr int getNumberOfGeometryErrorCodes() { return 2; }
+
+/// \brief Convert geometry error type into numberic representation
+/// \param errortype Geometry error type
+/// \return Error code connected to error type
+constexpr int getErrorCodeFromGeometryError(GeometryError_t errortype)
+{
+  switch (errortype) {
+    case GeometryError_t::CELL_RANGE_EXCEED:
+      return 0;
+    case GeometryError_t::CELL_INDEX_NEGATIVE:
+      return 1;
+    default:
+      return -1;
+  }
+}
+
+/// \brief Convert error code to geometry error type
+///
+/// Attention: Error code must be a valid error code, handled
+/// internally via assert.
+///
+/// \param errorcode Error code to be converted
+/// \return Error type connected to error code
+GeometryError_t getGeometryErrorFromErrorCode(unsigned int errorcode);
+
+/// \brief Get name of a given geometry error type
+///
+/// Name is a short single word descriptor used i.e. in
+/// object names.
+///
+/// \param errortype Error type of the geometry error
+/// \return Name connected to geometry error type
+const char* getGeometryErrorName(GeometryError_t errortype);
+
+/// \brief Get name of a given geometry error code
+///
+/// Name is a short single word descriptor used i.e. in
+/// object names. Attention: Error code must be a valid
+/// geomentry error code.
+///
+/// \param errorcode Error code of the geometry error
+/// \return Name connected to geometry error type
+const char* getGeometryErrorName(unsigned int errorcode);
+
+/// \brief Get title of a given geometry error type
+///
+/// Title is a short descriptor used i.e. in
+/// histogram titles.
+///
+/// \param errortype Error type of the geometry error
+/// \return Title connected to geometry error type
+const char* getGeometryErrorTitle(GeometryError_t errortype);
+
+/// \brief Get title of a given geometry error type
+///
+/// Title is a short descriptor used i.e. in
+/// histogram titles. Attention: Error code must
+/// be a valid geomentry error code.
+///
+/// \param errorcode Error code of the geometry error
+/// \return Title connected to geometry error type
+const char* getGeometryErrorTitle(unsigned int errorcode);
+
+/// \brief Get detailed description of a given geometry error type
+///
+/// Provides a long description to be used i.e. in error messages.
+///
+/// \param errortype Error type of the geometry error
+/// \return Detaied description connected to geometry error type
+const char* getGeometryErrorDescription(GeometryError_t errortype);
+
+/// \brief Get detailed description of a given geometry error type
+///
+/// Provides a long description to be used i.e. in error messages.
+/// Attention: Error code must be a valid geomentry error code.
+///
+/// \param errortype Error type of the geometry error
+/// \return Detaied description connected to geometry error type
+const char* getGeometryErrorDescription(unsigned int errorcode);
+
+/// \enum GainError_t
+/// \brief Errors appearing when merging gain types
+/// \ingroup EMCALreconstruction
+///
+/// Errors can appear when an expected gain type is missing,
+/// either because the HG is saturated and the LG is missing
+/// or because the LG is found for an ADC below HG/LG transition
+/// and the HG is missing.
+enum class GainError_t {
+  LGNOHG,       ///< LG found below HG/LG transition, HG missing
+  HGNOLG,       ///< HG saturated, LG missing
+  UNKNOWN_ERROR ///< Unknown error code
+};
+
+/// \brief Get the number of gain error codes
+/// \return Number of gain error codes
+constexpr int getNumberOfGainErrorCodes() { return 2; }
+
+/// \brief Convert gain error type into numberic representation
+/// \param errortype Gain error type
+/// \return Error code connected to error type
+constexpr int getErrorCodeFromGainError(GainError_t errortype)
+{
+  switch (errortype) {
+    case GainError_t::LGNOHG:
+      return 0;
+    case GainError_t::HGNOLG:
+      return 1;
+    default:
+      return -1;
+  };
+}
+
+/// \brief Convert error code to gain error type
+///
+/// Attention: Error code must be a valid error code, handled
+/// internally via assert.
+///
+/// \param errorcode Error code to be converted
+/// \return Error type connected to error code
+GainError_t getGainErrorFromErrorCode(unsigned int errorcode);
+
+/// \brief Get name of a given gain error type
+///
+/// Name is a short single word descriptor used i.e. in
+/// object names.
+///
+/// \param errortype Error type of the gain type error
+/// \return Name connected to gain type error
+const char* getGainErrorName(GainError_t errortype);
+
+/// \brief Get name of a given gain error code
+///
+/// Name is a short single word descriptor used i.e. in
+/// object names. Attention: Error code must be a valid
+/// gain type error code.
+///
+/// \param errorcode Error code of the gain type error
+/// \return Name connected to gain type error
+const char* getGainErrorName(unsigned int errorcode);
+
+/// \brief Get title of a given gain error type
+///
+/// Title is a short descriptor used i.e. in
+/// histogram titles.
+///
+/// \param errortype Error type of the gain type error
+/// \return Title connected to gain type error
+const char* getGainErrorTitle(GainError_t errortype);
+
+/// \brief Get title of a given gain error type
+///
+/// Title is a short descriptor used i.e. in
+/// histogram titles. Attention: Error code must
+/// be a valid gain type error code.
+///
+/// \param errorcode Error code of the gain type error
+/// \return Title connected to gain type error
+const char* getGainErrorTitle(unsigned int errorcode);
+
+/// \brief Get detailed description of a given gain error type
+///
+/// Provides a long description to be used i.e. in error messages.
+///
+/// \param errortype Error type of the gain type error
+/// \return Detaied description connected to gain type error
+const char* getGainErrorDescription(GainError_t errortype);
+
+/// \brief Get detailed description of a given gain error type
+///
+/// Provides a long description to be used i.e. in error messages.
+/// Attention: Error code must be a valid gain type error code.
+///
+/// \param errortype Error type of the gain type error
+/// \return Detaied description connected to gain type error
+const char* getGainErrorDescription(unsigned int errorcode);
+
+/// \enum TRUDecodingError_t
+/// \brief Errors related to TRU data decoding
+/// \ingroup EMCALReconstruction
+///
+/// TRU decoding errors appear during the decoding of the TRU data.
+/// They can be raised by the trigger mapping, in case an invalid
+/// FastOR index or TRU index is called, or by the TRU data handler if
+/// the patch index is outside range.
+enum class TRUDecodingError_t {
+  TRU_INDEX_INVALID,        ///< TRU index invalid
+  PATCH_INDEX_INVALID,      ///< Patch index outside range
+  FASTOR_INDEX_INVALID,     ///< FastOR index unknown
+  FASTOR_STARTTIME_INVALID, ///< FastOr stattime is larger than 14
+  UNKNOWN_ERROR             ///< Unknown error type
+};
+
+/// \brief Get the number of TRU error codes
+/// \return Number of TRU error codes
+constexpr int getNumberOfTRUErrorCodes() { return 3; }
+
+/// \brief Convert error code to TRU decoding error type
+///
+/// Attention: Error code must be a valid error code, handled
+/// internally via assert.
+///
+/// \param errorcode Error code to be converted
+/// \return Error type connected to error code
+TRUDecodingError_t getTRUDecodingErrorFromErrorCode(unsigned int errorcode);
+
+/// \brief Convert TRU decoding error type into numberic representation
+/// \param errortype TRU decoding error type
+/// \return Error code connected to error type
+constexpr int getErrorCodeFromTRUDecodingError(TRUDecodingError_t error)
+{
+  switch (error) {
+    case TRUDecodingError_t::TRU_INDEX_INVALID:
+      return 0;
+    case TRUDecodingError_t::PATCH_INDEX_INVALID:
+      return 1;
+    case TRUDecodingError_t::FASTOR_INDEX_INVALID:
+      return 2;
+    case TRUDecodingError_t::FASTOR_STARTTIME_INVALID:
+      return 3;
+    case TRUDecodingError_t::UNKNOWN_ERROR:
+      return -1;
+  }
+}
+
+/// \brief Get name of a given TRU decoding error type
+///
+/// Name is a short single word descriptor used i.e. in
+/// object names.
+///
+/// \param errortype Error type of the TRU decoding error
+/// \return Name connected to TRU decoding error type
+const char* getTRUDecodingErrorName(TRUDecodingError_t errortype);
+
+/// \brief Get name of a given TRU decoding error code
+///
+/// Name is a short single word descriptor used i.e. in
+/// object names. Attention: Error code must be a valid
+/// TRU decoding error code.
+///
+/// \param errorcode Error code of the TRU decoding error
+/// \return Name connected to TRU decoding error type
+const char* getTRUDecodingErrorName(unsigned int errorcode);
+
+/// \brief Get title of a given TRU decoding error type
+///
+/// Title is a short descriptor used i.e. in
+/// histogram titles.
+///
+/// \param errortype Error type of the TRU decoding error
+/// \return Title connected to TRU decoding error
+const char* getTRUDecodingErrorTitle(TRUDecodingError_t errortype);
+
+/// \brief Get title of a given TRU decoding error type
+///
+/// Title is a short descriptor used i.e. in
+/// histogram titles. Attention: Error code must
+/// be a valid TRU decoding error code.
+///
+/// \param errorcode Error code of the TRU decoding error
+/// \return Title connected to TRU decoding error
+const char* getTRUDecodingErrorTitle(unsigned int errorcode);
+
+/// \brief Get detailed description of a given TRU decoding error type
+///
+/// Provides a long description to be used i.e. in error messages.
+///
+/// \param errortype Error type of the TRU decoding error
+/// \return Detaied description connected to TRU decoding error
+const char* getTRUDecodingErrorErrorDescription(TRUDecodingError_t errortype);
+
+/// \brief Get detailed description of a given TRU decoding error type
+///
+/// Provides a long description to be used i.e. in error messages.
+/// Attention: Error code must be a valid TRU decoding error code.
+///
+/// \param errortype Error type of the TRU decoding error
+/// \return Detaied description connected to TRU decoding error
+const char* getTRUDecodingErrorErrorDescription(unsigned int errorcode);
+
+} // namespace reconstructionerrors
+
+} // namespace emcal
+
+} // namespace o2
+
+#endif // !ALICEO2_EMCAL_RECONSTRUCTIONERRORS_H
\ No newline at end of file
diff --git a/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/STUDecoderError.h b/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/STUDecoderError.h
new file mode 100644
index 0000000000000..1bebb3db4bf43
--- /dev/null
+++ b/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/STUDecoderError.h
@@ -0,0 +1,117 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#ifndef ALICEO2_EMCAL_STUDECODERERROR_H
+#define ALICEO2_EMCAL_STUDECODERERROR_H
+
+#include 
+#include 
+#include 
+
+namespace o2::emcal
+{
+
+/// \class STUDecoderError
+/// \brief Handling of STU reconstruction errors
+/// \ingroup EMCALreconstruction
+/// \since April 25, 2023
+///
+/// In order to distinguish different error types the STUDecoder error
+/// carries an error code which can be uniquely identified with a
+/// condition raising the excpetion. Source is always the DDL of the
+/// STU raising the exception.
+class STUDecoderError final : public std::exception
+{
+ public:
+  /// \enum ErrorCode_t
+  /// \brief Error codes of STU decoding
+  enum class ErrorCode_t {
+    PAGE_ERROR,                    ///< Page decoding failed (missing header)
+    WORD_UNEXPECTED,               ///< Word unexpected
+    INDEX_UNEXPECTED,              ///< Patch index unexpected
+    ADC_OVERFLOW,                  ///< ADC overflow
+    FEEID_UNEXPECTED,              ///< FeeID index unexpected
+    OLD_PAYLOAD_VERSION,           ///< unsupported old payload  version
+    FULL_PAYLOAD_SIZE_UNEXPECTED,  ///< full payload size unexpected
+    SHORT_PAYLOAD_SIZE_UNEXPECTED, ///< short payload size unexpected
+    UNKNOWN                        ///< Unknown error code (needed for conversion to int)
+  };
+
+  /// \brief Get integer representation of error code
+  /// \param errorcode Error code
+  /// \return Integer representation
+  static int errorCodeToInt(ErrorCode_t errorcode);
+
+  /// \brief Convert integer to error code
+  /// \param errorcode Error code as integer
+  /// \return Error code (symbolic) - UNKNOWN for invalid integer error codes
+  static ErrorCode_t intToErrorCode(int errorcode);
+
+  /// \brief Get the name of the error code
+  /// \param errorcode Error code
+  /// \return Name of the error code
+  static std::string getErrorCodeName(ErrorCode_t errorcode);
+
+  /// \brief Get the name of the error code
+  /// \param errorcode Error code (integer representation)
+  /// \return Name of the error code
+  static std::string getErrorCodeName(int errorcode) { return getErrorCodeName(intToErrorCode(errorcode)); }
+
+  /// \brief Get the title of the error code
+  /// \param errorcode Error code
+  /// \return Title of the error code
+  static std::string getErrorCodeTitle(ErrorCode_t errorcode);
+
+  /// \brief Get the title of the error code
+  /// \param errorcode Error code (integer representation)
+  /// \return Title of the error code
+  static std::string getErrorCodeTitle(int errorcode) { return getErrorCodeTitle(intToErrorCode(errorcode)); }
+
+  /// \brief Constructor
+  /// \param ddlID ID of the DDL for which the exception is raised
+  /// \param errcode Error code of the exception
+  STUDecoderError(int ddlID, ErrorCode_t errcode);
+
+  /// \brief Destructor
+  ~STUDecoderError() noexcept final = default;
+
+  /// \brief Access to error message
+  /// \return Error message
+  const char* what() const noexcept final { return mMessage.data(); }
+
+  /// \brief Get the ID of the DDL in which the exception is raised
+  /// \return ID of the DDL
+  int getDDLID() const noexcept { return mDDLId; }
+
+  /// \brief Get error code of the exception
+  /// \return Error code
+  ErrorCode_t getErrorCode() const noexcept { return mErrorCode; }
+
+  /// \brief Print details of the error on the stream
+  /// \param stream Stream to print on
+  ///
+  /// Helper function for streaming operator
+  void printStream(std::ostream& stream) const;
+
+ private:
+  int mDDLId;             ///< ID of the DDL raising the exception
+  ErrorCode_t mErrorCode; ///< Error code of the exception
+  std::string mMessage;   ///< Message buffer
+};
+
+/// \brief Streaming operator of STU decoding errors
+/// \param stream Stream to print the error on
+/// \param error Error to be streamed
+/// \return Stream after printing
+std::ostream& operator<<(std::ostream& stream, const STUDecoderError& error);
+
+} // namespace o2::emcal
+
+#endif
\ No newline at end of file
diff --git a/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/StuDecoder.h b/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/StuDecoder.h
new file mode 100755
index 0000000000000..2e26ff434b531
--- /dev/null
+++ b/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/StuDecoder.h
@@ -0,0 +1,142 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#ifndef ALICEO2_EMCAL_STUDECODER_H
+#define ALICEO2_EMCAL_STUDECODER_H
+
+#include 
+#include 
+#include 
+#include "EMCALReconstruction/RawReaderMemory.h"
+#include "DataFormatsEMCAL/Constants.h"
+
+namespace o2
+{
+namespace emcal
+{
+/// \class StuDecoder
+/// \brief Decoder of the EMCAL/DCAL STU data
+/// \ingroup EMCALreconstruction
+/// \author Martin Poghosyan , Oak Ridge National Laboratory
+/// \since Apr. 27, 2022
+///
+
+class StuDecoder
+{
+ public:
+  /// \brief Constructor
+  /// \param reader Raw reader instance to be decoded
+  StuDecoder(RawReaderMemory& reader);
+
+  /// \brief Destructor
+  ~StuDecoder() = default;
+
+  /// \brief Decode the STU stream
+  /// \throw StuDecoderError if the STU payload cannot be decoded
+  void decode();
+
+  int32_t getL1GammaHighThreshold() const { return mL1GammaHighThreshold; }
+  int32_t getL1JetHighThreshold() const { return mL1JetHighThreshold; }
+  int32_t getL1GammaLowThreshold() const { return mL1GammaLowThreshold; }
+  int32_t getL1JetLowThreshold() const { return mL1JetLowThreshold; }
+  int32_t getRho() const { return (mCFGword13 & 0x3FFFF); }
+  int32_t getFrameReceivedSTU() const { return ((mCFGword13 >> 18) & 0x3); }
+  int32_t getRegionEnable() const { return mRegionEnable; }
+  int32_t getFrameReceived() const { return mFrameReceived; }
+  int32_t getParchSize() const { return ((mCFGword16 >> 16) & 0xFFFF); }
+  int32_t getFWversion() const { return (mCFGword16 & 0xFFFF); }
+
+  STUtype_t getSTUtype() const { return mSTU; }
+  int getFeeID() const { return STUparam::FeeID[mSTU]; }
+  int getNumberOfTRUs() const { return STUparam::NTRU[mSTU]; }
+
+  std::vector getL1JetHighPatchIndices() const { return mL1JetHighPatchIndex; }
+  std::vector getL1JetLowPatchIndices() const { return mL1JetLowPatchIndex; }
+  std::vector getL1GammaHighPatchIndices() const { return mL1GammaHighPatchIndex; }
+  std::vector getL1GammaLowPatchIndics() const { return mL1GammaLowPatchIndex; }
+  std::vector getFastOrADCs() const { return mFastOrADC; }
+
+  int getNumberOfL1JetHighPatches() const { return mL1JetHighPatchIndex.size(); }
+  int getNumberOfL1JetLowPatches() const { return mL1JetLowPatchIndex.size(); }
+  int getNumberOfL1GammaHighPatches() const { return mL1GammaHighPatchIndex.size(); }
+  int getNumberOfL1GammaLowPatches() const { return mL1GammaLowPatchIndex.size(); }
+
+  int16_t getIndexOfL1JetHighPatch(int id) const { return mL1JetHighPatchIndex[id]; }
+  int16_t getIndexOfL1JetLowPatch(int id) const { return mL1JetLowPatchIndex[id]; }
+  int16_t getIndexOfL1GammaHighPatch(int id) const { return mL1GammaHighPatchIndex[id]; }
+  int16_t getIndexOfL1GammaLowPatch(int id) const { return mL1GammaLowPatchIndex[id]; }
+
+  int16_t getFastOrADC(int iTRU, int iCh) const { return mFastOrADC[iTRU + getNumberOfTRUs() * iCh]; }
+
+  std::tuple getL1GammaMaxPatch() const // std::tuple
+  {
+    return std::make_tuple(((mCFGWord0 >> 9) & 0x1F), ((mCFGWord0 >> 4) & 0x1F), (mCFGWord0 & 0xF));
+  }
+
+  bool isFullPayload() const { return mIsFullPayload; }
+  bool isL1GammaLowFired() const { return ((mCFGWord0 >> 16) & 0x1); }
+  bool isL1GammaHighFired() const { return ((mCFGWord0 >> 17) & 0x1); }
+  bool isL1JetLowFired() const { return ((mCFGWord0 >> 18) & 0x1); }
+  bool isL1JetHighFired() const { return ((mCFGWord0 >> 19) & 0x1); }
+  bool isMedianMode() const { return ((mCFGWord0 >> 20) & 0x1); }
+
+  void dumpSTUcfg() const;
+
+  int mDebug = -3;
+
+ private:
+  RawReaderMemory& mRawReader; ///< underlying raw reader
+
+  std::vector mL1JetHighPatchIndex;
+  std::vector mL1JetLowPatchIndex;
+  std::vector mL1GammaHighPatchIndex;
+  std::vector mL1GammaLowPatchIndex;
+  std::vector mFastOrADC;
+
+  // data from payload
+  int32_t mCFGWord0 = 0;             ///<
+  int32_t mCFGWord1 = 0;             ///<
+  int32_t mL0mask = 0;               ///<
+  int32_t mL1GammaHighThreshold = 0; ///<
+  int32_t mShortPayloadRate = 0;     ///<
+  int32_t mL0bits = 0;               ///<
+  int32_t mL1JetHighThreshold = 0;   ///<
+  int32_t mL1GammaLowThreshold = 0;  ///<
+  int32_t mL1JetLowThreshold = 0;    ///<
+  int32_t mCFGword13 = 0;            ///<
+  int32_t mRegionEnable = 0;         ///<
+  int32_t mFrameReceived = 0;        ///<
+  int32_t mCFGword16 = 0;            ///<
+
+  STUtype_t mSTU = STUtype_t::ESTU;
+  bool mIsFullPayload = true; ///<
+
+  void init();
+  void decodeL1JetPatchIndices(const uint32_t* buffer);
+  void decodeL1GammaPatchIndices(const uint32_t* buffer);
+  void decodeFastOrADC(const uint32_t* buffer);
+
+  int getCFGWords() { return STUparam::CFG_nWords[mSTU]; }
+  int getL1JetIndexWords() { return STUparam::L1JetIndex_nWords[mSTU]; }
+  int getL0indexWords() { return STUparam::L0index_nWords[mSTU]; }
+  int getL1GammaIndexWords() { return STUparam::L1GammaIndex_nWords[mSTU]; }
+  int getRawWords() { return STUparam::Raw_nWords[mSTU]; }
+  int getSubregionsEta() { return STUparam::SubregionsEta[mSTU]; }
+  int getSubregionsPhi() { return STUparam::SubregionsPhi[mSTU]; }
+  int getPaloadSizeFull() { return STUparam::PaloadSizeFull[mSTU]; }
+  int getPaloadSizeShort() { return STUparam::PaloadSizeShort[mSTU]; }
+
+  ClassDefNV(StuDecoder, 1);
+};
+
+} // namespace emcal
+} // namespace o2
+
+#endif
diff --git a/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/TRUDataHandler.h b/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/TRUDataHandler.h
new file mode 100644
index 0000000000000..811faf13a05ff
--- /dev/null
+++ b/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/TRUDataHandler.h
@@ -0,0 +1,185 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#ifndef ALICEO2_EMCAL_TRUDataHandler_H
+#define ALICEO2_EMCAL_TRUDataHandler_H
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "Rtypes.h"
+
+#include "EMCALBase/TriggerMappingV2.h"
+
+namespace o2::emcal
+{
+
+/// \class TRUDataHandler
+/// \brief Helper class to handle decoded TRU data during the reconstruction
+/// \author Markus Fasel , Oak Ridge National Laboratory
+/// \ingroup EMCALreconstruction
+/// \since April 19, 2024
+///
+/// The decoded TRU data contains the following information
+/// - Index of the TRU
+/// - Trigger time of the TRU
+/// - Fired or not
+/// - Index of fired patches with time the patch has fired
+/// The information is decoded in columns 96 to 105 of the FakeALTRO data. The
+/// class does not handle FastOR timesums (colums 0-96), they are handled by a
+/// separate class FastORTimeSeries.
+class TRUDataHandler
+{
+ public:
+  /// \class PatchIndexException
+  /// \brief Handler of errors related to invalid trigger patch IDs
+  /// \ingroup EMCALreconstruction
+  class PatchIndexException final : public std::exception
+  {
+   public:
+    /// \brief Constructor
+    /// \param index Patch index raising the exception
+    PatchIndexException(int8_t index);
+
+    /// \brief  Destructor
+    ~PatchIndexException() noexcept final = default;
+
+    /// \brief Get patch index raising the exception
+    /// \return Patch index
+    int8_t getIndex() const { return mIndex; }
+
+    /// \brief Access Error message
+    /// \return Error message
+    const char* what() const noexcept final
+    {
+      return mMessage.data();
+    }
+
+    /// \brief Print error on output stream
+    /// \param stream Stream to be printed to
+    void printStream(std::ostream& stream) const;
+
+   private:
+    int8_t mIndex = -1;   ///< Patch index rainsing the exception
+    std::string mMessage; ///< Buffer for error message
+  };
+
+  /// \brief Constructor
+  TRUDataHandler();
+
+  /// \brief Destructor
+  ~TRUDataHandler() = default;
+
+  /// \brief Reset handler
+  void reset();
+
+  /// \brief Set reconstructed trigger patch
+  /// \param index Index of the trigger patch in the TRU
+  /// \param time Decoded time of the patch
+  /// \throw PatchIndexException in case the patch index is invalid (>= 77)
+  void setPatch(unsigned int index, unsigned int time)
+  {
+    checkPatchIndex(index);
+    mPatchTimes[index] = time;
+  }
+
+  /// \brief Mark TRU as fired (containing at least one patch above threshold)
+  /// \param fired
+  void setFired(bool fired) { mL0Fired = fired; }
+
+  /// \brief Set the L0 time of the TRU
+  /// \param l0time L0 time of the TRU
+  void setL0time(int l0time) { mL0Time = l0time; }
+
+  /// \brief Set the index of the TRU (in global STU indexing scheme)
+  /// \param index Index of the TRU
+  void setTRUIndex(int index) { mTRUIndex = index; }
+
+  /// \brief Check whether the TRU was fired (at least one patch above threshold)
+  /// \return True if the TRU was fired, false otherwise
+  bool isFired() const { return mL0Fired; }
+
+  int8_t getL0time() const { return mL0Time; }
+
+  /// \brief Check whehther the patch at the given index has fired
+  /// \param index Index of the patch
+  /// \return True if the patch has fired, false otherwise
+  /// \throw PatchIndexException in case the patch index is invalid (>= 77)
+  bool hasPatch(unsigned int index) const
+  {
+    checkPatchIndex(index);
+    return mPatchTimes[index] < UCHAR_MAX;
+  }
+
+  /// \brief Get the trigger time of the trigger patch at a given index
+  /// \param index Index of the trigger patch
+  /// \return Reconstructed patch time (UCHAR_MAX in case the patch has not fired)
+  /// \throw PatchIndexException in case the patch index is invalid (>= 77)
+  uint8_t getPatchTime(unsigned int index) const
+  {
+    checkPatchIndex(index);
+    return mPatchTimes[index];
+  }
+
+  /// \brief Check whether the TRU has any patch fired
+  /// \return True if at least one fired patch was found, false otherwise
+  bool hasAnyPatch() const
+  {
+    for (int ipatch = 0; ipatch < mPatchTimes.size(); ipatch++) {
+      if (hasPatch(ipatch)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  /// \brief Get the index of the TRU in global (STU) index schemes
+  /// \return Index of the TRU
+  int getTRUIndex() const { return mTRUIndex; }
+
+  /// \brief Print TRU information to an output stream
+  /// \param stream Stream to print on
+  void printStream(std::ostream& stream) const;
+
+ private:
+  /// \brief Check whether the patch index is valid
+  /// \throw PatchIndexException in case the patch index is invalid (>= 77)
+  void checkPatchIndex(unsigned int index) const
+  {
+    if (index >= mPatchTimes.size()) {
+      throw PatchIndexException(index);
+    }
+  }
+
+  std::array mPatchTimes; ///< Patch times: In case the patch time is smaller than UCHAR_MAX then the patch has fired
+  bool mL0Fired = false;                                                      ///< TRU has fired
+  int8_t mL0Time = -1;                                                        ///< L0 time of the TRU
+  int8_t mTRUIndex = -1;                                                      ///< Index of the TRU
+  ClassDefNV(TRUDataHandler, 1);
+};
+
+/// \brief Output stream operator for the TRU data handler
+/// \param stream Stream to print on
+/// \param data TRU data to be streamed
+/// \return Stream after printing
+std::ostream& operator<<(std::ostream& stream, const TRUDataHandler& data);
+
+/// \brief Output stream operator of the PatchIndexException
+/// \param stream Stream to print on
+/// \param error Error to be streamed
+/// \return Stream after printing
+std::ostream& operator<<(std::ostream& stream, const TRUDataHandler::PatchIndexException& error);
+
+} // namespace o2::emcal
+#endif
diff --git a/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/TRUDecodingErrors.h b/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/TRUDecodingErrors.h
new file mode 100644
index 0000000000000..084465f84944f
--- /dev/null
+++ b/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/TRUDecodingErrors.h
@@ -0,0 +1,60 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef ALICEO2_EMCAL_TRUDECODINGERRORS_H
+#define ALICEO2_EMCAL_TRUDECODINGERRORS_H
+
+#include 
+#include 
+
+namespace o2
+{
+
+namespace emcal
+{
+
+/// \class FastOrStartTimeInvalidException
+/// \brief Handling of error if starttime is to large (>=14). This is most likely caused by a corrupted channel header where a FEC channel is identified as a TRU channel
+/// \ingroup EMCALbase
+class FastOrStartTimeInvalidException : public std::exception
+{
+ public:
+  /// \brief Constructor
+  /// \param l0size Size of the L0 patch
+  FastOrStartTimeInvalidException(int time) : std::exception(), mErrorMessage(), mStartTime(time)
+  {
+    mErrorMessage = "FastOr starttime invalid: " + std::to_string(time);
+  }
+
+  /// \brief Destructor
+  ~FastOrStartTimeInvalidException() noexcept final = default;
+
+  /// \brief Access to error message
+  /// \return Error message
+  const char* what() const noexcept final
+  {
+    return mErrorMessage.data();
+  }
+
+  /// \brief Get the size of the L0 patch
+  /// \return Size of the L0 patch
+  int getStartTime() const noexcept { return mStartTime; }
+
+ private:
+  std::string mErrorMessage; ///< Buffer for error message
+  int mStartTime;            ///< Size of the L0 patch
+};
+
+} // namespace emcal
+
+} // namespace o2
+
+#endif //  ALICEO2_EMCAL_TRUDECODINGERRORS_H
\ No newline at end of file
diff --git a/Detectors/EMCAL/reconstruction/run/rawReaderFile.cxx b/Detectors/EMCAL/reconstruction/run/rawReaderFile.cxx
index 3e27f6ce46c73..18c2a0a258e93 100644
--- a/Detectors/EMCAL/reconstruction/run/rawReaderFile.cxx
+++ b/Detectors/EMCAL/reconstruction/run/rawReaderFile.cxx
@@ -18,7 +18,7 @@
 #include "DetectorsRaw/RawFileReader.h"
 #include "EMCALReconstruction/AltroDecoder.h"
 #include "EMCALReconstruction/RawReaderMemory.h"
-#include "FairLogger.h"
+#include 
 
 namespace bpo = boost::program_options;
 //using namespace o2::emcal;
diff --git a/Detectors/EMCAL/reconstruction/src/AltroDecoder.cxx b/Detectors/EMCAL/reconstruction/src/AltroDecoder.cxx
index a74be009145d6..d555b4523cc49 100644
--- a/Detectors/EMCAL/reconstruction/src/AltroDecoder.cxx
+++ b/Detectors/EMCAL/reconstruction/src/AltroDecoder.cxx
@@ -8,14 +8,15 @@
 // In applying this license CERN does not waive the privileges and immunities
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
+#include 
 #include 
 #include 
 #include 
-#include 
-#include "InfoLogger/InfoLogger.hxx"
+#include 
 #include "DetectorsRaw/RDHUtils.h"
 #include "EMCALReconstruction/AltroDecoder.h"
 #include "EMCALReconstruction/RawReaderMemory.h"
+#include "Framework/Logger.h"
 
 using namespace o2::emcal;
 
@@ -41,9 +42,7 @@ void AltroDecoder::readRCUTrailer()
     gsl::span payloadwords(payloadwordsOrig.data(), payloadwordsOrig.size());
     mRCUTrailer.constructFromRawPayload(payloadwords);
   } catch (RCUTrailer::Error& e) {
-    AliceO2::InfoLogger::InfoLogger logger;
-    logger << e.what();
-    throw AltroDecoderError(AltroDecoderError::ErrorType_t::RCU_TRAILER_ERROR, (boost::format("RCU trailer decoding error: %s") % e.what()).str().data());
+    throw AltroDecoderError(AltroDecoderError::ErrorType_t::RCU_TRAILER_ERROR, fmt::format("{} {}", AltroDecoderError::getErrorTypeDescription(AltroDecoderError::ErrorType_t::RCU_TRAILER_ERROR), e.what()));
   }
 }
 
@@ -52,7 +51,7 @@ void AltroDecoder::checkRCUTrailer()
   int trailersize = mRCUTrailer.getTrailerSize();
   int buffersize = mRawReader.getPayload().getPayloadWords().size();
   if (trailersize > buffersize) {
-    throw AltroDecoderError(AltroDecoderError::ErrorType_t::RCU_TRAILER_SIZE_ERROR, (boost::format("Trailer size %d exceeding buffer size %d") % trailersize % buffersize).str().data());
+    throw AltroDecoderError(AltroDecoderError::ErrorType_t::RCU_TRAILER_SIZE_ERROR, fmt::format("{}: Trailer size {} exceeding buffer size {}", AltroDecoderError::getErrorTypeDescription(AltroDecoderError::ErrorType_t::RCU_TRAILER_SIZE_ERROR), trailersize, buffersize));
   }
 }
 
@@ -63,6 +62,7 @@ void AltroDecoder::readChannels()
   int currentpos = 0;
   auto& buffer = mRawReader.getPayload().getPayloadWords();
   auto maxpayloadsize = buffer.size() - mRCUTrailer.getTrailerSize();
+  int lastFEC = -1;
   while (currentpos < maxpayloadsize) {
     auto currentword = buffer[currentpos++];
     if (currentword >> 30 != 1) {
@@ -75,6 +75,23 @@ void AltroDecoder::readChannels()
     uint16_t payloadsize = (channelheader >> 16) & 0x3FF;
     bool badchannel = (channelheader >> 29) & 0x1;
 
+    // check hardware address for consistency
+    if (!checkChannelHWAddress(hwaddress)) {
+      // Inconsistent HW address information, channel header corrupted, payload must be skipped
+      mMinorDecodingErrors.emplace_back(MinorAltroDecodingError::ErrorType_t::CHANNEL_ORDER, channelheader, currentword);
+      continue;
+    }
+
+    int currentfec = 10 * Channel::getBranchIndexFromHwAddress(hwaddress) + Channel::getFecIndexFromHwAddress(hwaddress);
+    // std::cout << "Branch: " << Channel::getBranchIndexFromHwAddress(hwaddress) << ", FEC " << Channel::getFecIndexFromHwAddress(hwaddress) << " -> " << currentfec << " (channel " << Channel::getChannelIndexFromHwAddress(hwaddress) << " )" << std::endl;
+    if (currentfec < lastFEC) {
+      // FECs are ordered by the SRU, breaking the order is a clear sign of data corruption
+      // std::cout << "Error Fec, current " << currentfec << ", last " << lastFEC << std::endl;
+      mMinorDecodingErrors.emplace_back(MinorAltroDecodingError::ErrorType_t::CHANNEL_ORDER, channelheader, currentword);
+      continue;
+    }
+    lastFEC = currentfec;
+
     /// decode all words for channel
     bool foundChannelError = false;
     int numberofwords = (payloadsize + 2) / 3;
@@ -122,6 +139,18 @@ void AltroDecoder::readChannels()
         // we must break here as well, the bunch is cut and the pointer would be set to invalid memory
         break;
       }
+      // Raise minor decoding error in case the bunch length exceeds the maximum possible amount of samples
+      if ((unsigned int)bunchlength > mMaxBunchLength) {
+        mMinorDecodingErrors.emplace_back(MinorAltroDecodingError::ErrorType_t::BUNCH_LENGTH_ALLOW_EXCEED, channelheader, 0);
+        // Same as above: if the bunch length exceeds the maximum possible bunch length it will for sure conflict with the next bunch
+        break;
+      }
+      // Raise minor decoding error in case the start timne the maximum possible amount of samples (resulting in negative first sample index)
+      if ((unsigned int)starttime > mMaxBunchLength) {
+        mMinorDecodingErrors.emplace_back(MinorAltroDecodingError::ErrorType_t::BUNCH_STARTTIME, channelheader, 0);
+        // Also here we must break, out-of-bounds start time will create troubles in the raw fit
+        break;
+      }
       if (bunchlength == 0) {
         // skip bunches with bunch size 0, they don't contain any payload
         // Map error type to NULL header since header word is null
@@ -138,10 +167,29 @@ void AltroDecoder::readChannels()
   mChannelsInitialized = true;
 }
 
+bool AltroDecoder::checkChannelHWAddress(int hwaddress)
+{
+  unsigned int branch = Channel::getBranchIndexFromHwAddress(hwaddress),
+               fec = Channel::getFecIndexFromHwAddress(hwaddress),
+               altro = Channel::getAltroIndexFromHwAddress(hwaddress);
+  if (branch > 1) {
+    return false;
+  }
+  if (fec > 9) {
+    return false;
+  }
+  if (fec > 0) { // fec == 0 corresponds to trigger, which only has a fake altro information. Hence it's not checked
+    if (!(altro == 0 || altro == 2 || altro == 3 || altro == 4)) {
+      return false;
+    }
+  }
+  return true;
+}
+
 const RCUTrailer& AltroDecoder::getRCUTrailer() const
 {
   if (!mRCUTrailer.isInitialized()) {
-    throw AltroDecoderError(AltroDecoderError::ErrorType_t::RCU_TRAILER_ERROR, "RCU trailer was not initialized");
+    throw AltroDecoderError(AltroDecoderError::ErrorType_t::RCU_TRAILER_ERROR, fmt::format("{}: RCU trailer was not initialized", AltroDecoderError::getErrorTypeDescription(AltroDecoderError::ErrorType_t::RCU_TRAILER_ERROR)));
   }
   return mRCUTrailer;
 }
@@ -149,7 +197,7 @@ const RCUTrailer& AltroDecoder::getRCUTrailer() const
 const std::vector& AltroDecoder::getChannels() const
 {
   if (!mChannelsInitialized) {
-    throw AltroDecoderError(AltroDecoderError::ErrorType_t::CHANNEL_ERROR, "Channels not initizalized");
+    throw AltroDecoderError(AltroDecoderError::ErrorType_t::CHANNEL_ERROR, AltroDecoderError::getErrorTypeDescription(AltroDecoderError::ErrorType_t::CHANNEL_ERROR));
   }
   return mChannels;
 }
@@ -186,6 +234,7 @@ int AltroDecoderError::errorTypeToInt(AltroErrType errortype)
     case AltroErrType::CHANNEL_ERROR:
       errorNumber = 7;
       break;
+
     default:
       break;
   }
@@ -195,6 +244,7 @@ int AltroDecoderError::errorTypeToInt(AltroErrType errortype)
 
 AltroErrType AltroDecoderError::intToErrorType(int errornumber)
 {
+  assert(errornumber < getNumberOfErrorTypes());
 
   AltroErrType errorType;
 
@@ -230,23 +280,79 @@ AltroErrType AltroDecoderError::intToErrorType(int errornumber)
   return errorType;
 }
 
+const char* AltroDecoderError::getErrorTypeName(ErrorType_t errortype)
+{
+  switch (errortype) {
+    case AltroErrType::RCU_TRAILER_ERROR:
+      return "RCUTrailerError";
+    case AltroErrType::RCU_VERSION_ERROR:
+      return "RCUTrailerVersionError";
+    case AltroErrType::RCU_TRAILER_SIZE_ERROR:
+      return "RCUTrailerSizeError";
+    case AltroErrType::ALTRO_BUNCH_HEADER_ERROR:
+      return "BunchHeaderError";
+    case AltroErrType::ALTRO_BUNCH_LENGTH_ERROR:
+      return "BunchLengthError";
+    case AltroErrType::ALTRO_PAYLOAD_ERROR:
+      return "ALTROPayloadError";
+    case AltroErrType::ALTRO_MAPPING_ERROR:
+      return "ALTROMappingError";
+    case AltroErrType::CHANNEL_ERROR:
+      return "ChannelError";
+  };
+  return "";
+}
+
+const char* AltroDecoderError::getErrorTypeTitle(ErrorType_t errortype)
+{
+  switch (errortype) {
+    case AltroErrType::RCU_TRAILER_ERROR:
+      return "RCU Trailer";
+    case AltroErrType::RCU_VERSION_ERROR:
+      return "RCU Version";
+    case AltroErrType::RCU_TRAILER_SIZE_ERROR:
+      return "RCU Trailer Size";
+    case AltroErrType::ALTRO_BUNCH_HEADER_ERROR:
+      return "ALTRO Bunch Header";
+    case AltroErrType::ALTRO_BUNCH_LENGTH_ERROR:
+      return "ALTRO Bunch Length";
+    case AltroErrType::ALTRO_PAYLOAD_ERROR:
+      return "ALTRO Payload";
+    case AltroErrType::ALTRO_MAPPING_ERROR:
+      return "ALTRO Mapping";
+    case AltroErrType::CHANNEL_ERROR:
+      return "Channel";
+  };
+  return "";
+}
+
+const char* AltroDecoderError::getErrorTypeDescription(ErrorType_t errortype)
+{
+  switch (errortype) {
+    case AltroErrType::RCU_TRAILER_ERROR:
+      return "RCU trailer decoding error";
+    case AltroErrType::RCU_VERSION_ERROR:
+      return "Inconsistent RCU trailer version";
+    case AltroErrType::RCU_TRAILER_SIZE_ERROR:
+      return "Invalid RCU trailer size";
+    case AltroErrType::ALTRO_BUNCH_HEADER_ERROR:
+      return "Inconsistent bunch header";
+    case AltroErrType::ALTRO_BUNCH_LENGTH_ERROR:
+      return "Bunch length exceeding payload size";
+    case AltroErrType::ALTRO_PAYLOAD_ERROR:
+      return "Payload could not be decoded";
+    case AltroErrType::ALTRO_MAPPING_ERROR:
+      return "Invalid hardware address in ALTRO mapping";
+    case AltroErrType::CHANNEL_ERROR:
+      return "Channels not initizalized";
+  };
+  return "";
+}
+
 std::string MinorAltroDecodingError::what() const noexcept
 {
   std::stringstream result;
-  switch (mErrorType) {
-    case ErrorType_t::CHANNEL_END_PAYLOAD_UNEXPECT:
-      result << "Unexpected end of payload in altro channel payload!";
-      break;
-    case ErrorType_t::CHANNEL_PAYLOAD_EXCEED:
-      result << "Trying to access out-of-bound payload!";
-      break;
-    case ErrorType_t::BUNCH_HEADER_NULL:
-      result << "Bunch header 0 or not configured!";
-      break;
-    case ErrorType_t::BUNCH_LENGTH_EXCEED:
-      result << "Bunch length exceeding channel payload size!";
-      break;
-  };
+  result << getErrorTypeDescription(mErrorType);
   auto address = mChannelHeader & 0xFFF,
        payload = (mChannelHeader >> 16) & 0x3FF;
   bool good = (mChannelHeader >> 29) & 0x1;
@@ -271,12 +377,24 @@ int MinorAltroDecodingError::errorTypeToInt(MinorAltroErrType errortype)
     case MinorAltroErrType::CHANNEL_PAYLOAD_EXCEED:
       errorNumber = 1;
       break;
-    case MinorAltroErrType::BUNCH_HEADER_NULL:
+    case MinorAltroErrType::CHANNEL_ORDER:
       errorNumber = 2;
       break;
-    case MinorAltroErrType::BUNCH_LENGTH_EXCEED:
+    case MinorAltroErrType::CHANNEL_HEADER:
       errorNumber = 3;
       break;
+    case MinorAltroErrType::BUNCH_HEADER_NULL:
+      errorNumber = 4;
+      break;
+    case MinorAltroErrType::BUNCH_LENGTH_EXCEED:
+      errorNumber = 5;
+      break;
+    case MinorAltroErrType::BUNCH_LENGTH_ALLOW_EXCEED:
+      errorNumber = 6;
+      break;
+    case MinorAltroErrType::BUNCH_STARTTIME:
+      errorNumber = 7;
+      break;
   };
 
   return errorNumber;
@@ -284,7 +402,7 @@ int MinorAltroDecodingError::errorTypeToInt(MinorAltroErrType errortype)
 
 MinorAltroErrType MinorAltroDecodingError::intToErrorType(int errornumber)
 {
-
+  assert(errornumber < getNumberOfErrorTypes());
   MinorAltroErrType errorType;
 
   switch (errornumber) {
@@ -295,14 +413,119 @@ MinorAltroErrType MinorAltroDecodingError::intToErrorType(int errornumber)
       errorType = MinorAltroErrType::CHANNEL_PAYLOAD_EXCEED;
       break;
     case 2:
-      errorType = MinorAltroErrType::BUNCH_HEADER_NULL;
+      errorType = MinorAltroErrType::CHANNEL_ORDER;
       break;
     case 3:
+      errorType = MinorAltroErrType::CHANNEL_HEADER;
+      break;
+    case 4:
+      errorType = MinorAltroErrType::BUNCH_HEADER_NULL;
+      break;
+    case 5:
       errorType = MinorAltroErrType::BUNCH_LENGTH_EXCEED;
       break;
+    case 6:
+      errorType = MinorAltroErrType::BUNCH_LENGTH_ALLOW_EXCEED;
+      break;
+    case 7:
+      errorType = MinorAltroErrType::BUNCH_STARTTIME;
+      break;
     default:
       break;
   }
 
   return errorType;
-}
\ No newline at end of file
+}
+
+const char* MinorAltroDecodingError::getErrorTypeName(ErrorType_t errortype)
+{
+  switch (errortype) {
+    case MinorAltroErrType::CHANNEL_END_PAYLOAD_UNEXPECT:
+      return "ChannelEndPayloadUnexpected";
+    case MinorAltroErrType::CHANNEL_PAYLOAD_EXCEED:
+      return "ChannelPayloadExceed";
+    case MinorAltroErrType::CHANNEL_ORDER:
+      return "ChannelOrderError";
+    case MinorAltroErrType::CHANNEL_HEADER:
+      return "ChannelHeader";
+    case MinorAltroErrType::BUNCH_HEADER_NULL:
+      return "BunchHeaderNull";
+    case MinorAltroErrType::BUNCH_LENGTH_EXCEED:
+      return "BunchLengthExceed";
+    case MinorAltroErrType::BUNCH_LENGTH_ALLOW_EXCEED:
+      return "BunchLengthAllowExceed";
+    case MinorAltroErrType::BUNCH_STARTTIME:
+      return "BunchStarttimeExceed";
+  };
+  return "";
+}
+
+const char* MinorAltroDecodingError::getErrorTypeTitle(ErrorType_t errortype)
+{
+  switch (errortype) {
+    case MinorAltroErrType::CHANNEL_END_PAYLOAD_UNEXPECT:
+      return "Channel end unexpected";
+    case MinorAltroErrType::CHANNEL_PAYLOAD_EXCEED:
+      return "Channel exceed";
+    case MinorAltroErrType::CHANNEL_ORDER:
+      return "FEC order";
+    case MinorAltroErrType::CHANNEL_HEADER:
+      return "Channel header invalid";
+    case MinorAltroErrType::BUNCH_HEADER_NULL:
+      return "Bunch header null";
+    case MinorAltroErrType::BUNCH_LENGTH_EXCEED:
+      return "Bunch length exceed";
+    case MinorAltroErrType::BUNCH_LENGTH_ALLOW_EXCEED:
+      return "Bunch length impossible";
+    case MinorAltroErrType::BUNCH_STARTTIME:
+      return "Bunch starttime exceed";
+  };
+  return "";
+}
+
+const char* MinorAltroDecodingError::getErrorTypeDescription(ErrorType_t errortype)
+{
+  switch (errortype) {
+    case MinorAltroErrType::CHANNEL_END_PAYLOAD_UNEXPECT:
+      return "Unexpected end of payload in altro channel payload!";
+    case MinorAltroErrType::CHANNEL_PAYLOAD_EXCEED:
+      return "Trying to access out-of-bound payload!";
+    case MinorAltroErrType::CHANNEL_ORDER:
+      return "Invalid FEC order";
+    case MinorAltroErrType::CHANNEL_HEADER:
+      return "Invalid channel header";
+    case MinorAltroErrType::BUNCH_HEADER_NULL:
+      return "Bunch header 0 or not configured!";
+    case MinorAltroErrType::BUNCH_LENGTH_EXCEED:
+      return "Bunch length exceeding channel payload size!";
+    case MinorAltroErrType::BUNCH_LENGTH_ALLOW_EXCEED:
+      return "Bunch length exceeding max. possible bunch size!";
+    case MinorAltroErrType::BUNCH_STARTTIME:
+      return "Bunch start time outside range!";
+  };
+  return "";
+}
+
+std::ostream& o2::emcal::operator<<(std::ostream& stream, const AltroDecoderError& error)
+{
+  stream << error.what();
+  return stream;
+}
+
+std::ostream& o2::emcal::operator<<(std::ostream& stream, const AltroDecoderError::ErrorType_t& errortype)
+{
+  stream << AltroDecoderError::getErrorTypeName(errortype);
+  return stream;
+}
+
+std::ostream& o2::emcal::operator<<(std::ostream& stream, const MinorAltroDecodingError& error)
+{
+  stream << error.what();
+  return stream;
+}
+
+std::ostream& o2::emcal::operator<<(std::ostream& stream, const MinorAltroDecodingError::ErrorType_t& errortype)
+{
+  stream << MinorAltroDecodingError::getErrorTypeName(errortype);
+  return stream;
+}
diff --git a/Detectors/EMCAL/reconstruction/src/CTFCoder.cxx b/Detectors/EMCAL/reconstruction/src/CTFCoder.cxx
index 6c63a6f26fd9e..5f3b04f600192 100644
--- a/Detectors/EMCAL/reconstruction/src/CTFCoder.cxx
+++ b/Detectors/EMCAL/reconstruction/src/CTFCoder.cxx
@@ -41,10 +41,11 @@ void CTFCoder::createCoders(const std::vector& bufVec, o2::ctf::CTFCoderBa
 {
   const auto ctf = CTF::getImage(bufVec.data());
   // just to get types
-  uint16_t bcInc = 0, entries = 0, cellTime = 0, energy = 0, tower = 0, trigger = 0;
-  uint32_t orbitInc = 0;
+  int16_t bcInc = 0;
+  int32_t orbitInc = 0;
+  uint16_t entries = 0, cellTime = 0, energy = 0, tower = 0, trigger = 0;
   uint8_t status = 0;
-#define MAKECODER(part, slot) createCoder(op, ctf.getFrequencyTable(slot), int(slot))
+#define MAKECODER(part, slot) createCoder(op, std::get>(ctf.getDictionary(slot, mANSVersion)), int(slot))
   // clang-format off
   MAKECODER(bcInc,      CTF::BLC_bcIncTrig);
   MAKECODER(orbitInc,   CTF::BLC_orbitIncTrig);
@@ -57,3 +58,14 @@ void CTFCoder::createCoders(const std::vector& bufVec, o2::ctf::CTFCoderBa
   MAKECODER(trigger,    CTF::BLC_trigger);
   // clang-format on
 }
+
+///___________________________________________________________________________________
+void CTFCoder::assignDictVersion(o2::ctf::CTFDictHeader& h) const
+{
+  if (mExtHeader.isValidDictTimeStamp()) {
+    h = mExtHeader;
+  } else {
+    h.majorVersion = 1;
+    h.minorVersion = 2;
+  }
+}
\ No newline at end of file
diff --git a/Detectors/EMCAL/reconstruction/src/CaloFitResults.cxx b/Detectors/EMCAL/reconstruction/src/CaloFitResults.cxx
index 7a129f387fa44..4f81166761da0 100644
--- a/Detectors/EMCAL/reconstruction/src/CaloFitResults.cxx
+++ b/Detectors/EMCAL/reconstruction/src/CaloFitResults.cxx
@@ -12,7 +12,7 @@
 /// \file CaloFitResults.cxx
 /// \author Hadi Hassan (hadi.hassan@cern.ch)
 
-#include "FairLogger.h"
+#include 
 #include "TMath.h"
 
 #include "EMCALReconstruction/CaloFitResults.h"
diff --git a/Detectors/EMCAL/reconstruction/src/CaloRawFitter.cxx b/Detectors/EMCAL/reconstruction/src/CaloRawFitter.cxx
index 23f6758512850..9cdb6cab50cf8 100644
--- a/Detectors/EMCAL/reconstruction/src/CaloRawFitter.cxx
+++ b/Detectors/EMCAL/reconstruction/src/CaloRawFitter.cxx
@@ -11,13 +11,14 @@
 
 /// \file CaloRawFitter.cxx
 /// \author Hadi Hassan (hadi.hassan@cern.ch)
+#include 
 #include 
 #include 
 
 // ROOT sytem
 #include "TMath.h"
 
-#include "FairLogger.h"
+#include 
 #include "DataFormatsEMCAL/Constants.h"
 #include "EMCALReconstruction/Bunch.h"
 #include "EMCALReconstruction/CaloFitResults.h"
@@ -25,25 +26,6 @@
 
 using namespace o2::emcal;
 
-std::string CaloRawFitter::createErrorMessage(CaloRawFitter::RawFitterError_t errorcode)
-{
-  switch (errorcode) {
-    case RawFitterError_t::SAMPLE_UNINITIALIZED:
-      return "Sample for fit not initialzied or bunch length is 0";
-    case RawFitterError_t::FIT_ERROR:
-      return "Fit of the raw bunch was not successful";
-    case RawFitterError_t::CHI2_ERROR:
-      return "Chi2 of the fit could not be determined";
-    case RawFitterError_t::BUNCH_NOT_OK:
-      return "Calo bunch could not be selected";
-    case RawFitterError_t::LOW_SIGNAL:
-      return "No ADC value above threshold found";
-  };
-  // Silence compiler warnings for false positives
-  // can never enter here due to usage of enum class
-  return "Unknown error code";
-}
-
 int CaloRawFitter::getErrorNumber(CaloRawFitter::RawFitterError_t fiterror)
 {
   switch (fiterror) {
@@ -63,6 +45,86 @@ int CaloRawFitter::getErrorNumber(CaloRawFitter::RawFitterError_t fiterror)
   return -1;
 }
 
+CaloRawFitter::RawFitterError_t CaloRawFitter::intToErrorType(unsigned int fiterror)
+{
+  assert(fiterror < getNumberOfErrorTypes());
+  switch (fiterror) {
+    case 0:
+      return CaloRawFitter::RawFitterError_t::SAMPLE_UNINITIALIZED;
+    case 1:
+      return CaloRawFitter::RawFitterError_t::FIT_ERROR;
+    case 2:
+      return CaloRawFitter::RawFitterError_t::CHI2_ERROR;
+    case 3:
+      return CaloRawFitter::RawFitterError_t::BUNCH_NOT_OK;
+    case 4:
+      return CaloRawFitter::RawFitterError_t::LOW_SIGNAL;
+  };
+  // Silence the compiler warning for false positives
+  // Since we catch invalid codes via the assert we can
+  // never reach here. Since it is an enum class we need
+  // to choose one error code here. Pick the first error
+  // code instead.
+  return CaloRawFitter::RawFitterError_t::SAMPLE_UNINITIALIZED;
+}
+
+const char* CaloRawFitter::getErrorTypeName(CaloRawFitter::RawFitterError_t fiterror)
+{
+  switch (fiterror) {
+    case CaloRawFitter::RawFitterError_t::SAMPLE_UNINITIALIZED:
+      return "SampleUninitalized";
+    case CaloRawFitter::RawFitterError_t::FIT_ERROR:
+      return "NoConvergence";
+    case CaloRawFitter::RawFitterError_t::CHI2_ERROR:
+      return "Chi2Error";
+    case CaloRawFitter::RawFitterError_t::BUNCH_NOT_OK:
+      return "BunchRejected";
+    case CaloRawFitter::RawFitterError_t::LOW_SIGNAL:
+      return "LowSignal";
+  };
+  // Silence compiler warnings for false positives
+  // can never enter here due to usage of enum class
+  return "Unknown";
+}
+
+const char* CaloRawFitter::getErrorTypeTitle(CaloRawFitter::RawFitterError_t fiterror)
+{
+  switch (fiterror) {
+    case CaloRawFitter::RawFitterError_t::SAMPLE_UNINITIALIZED:
+      return "sample uninitalized";
+    case CaloRawFitter::RawFitterError_t::FIT_ERROR:
+      return "No convergence";
+    case CaloRawFitter::RawFitterError_t::CHI2_ERROR:
+      return "Chi2 error";
+    case CaloRawFitter::RawFitterError_t::BUNCH_NOT_OK:
+      return "Bunch rejected";
+    case CaloRawFitter::RawFitterError_t::LOW_SIGNAL:
+      return "Low signal";
+  };
+  // Silence compiler warnings for false positives
+  // can never enter here due to usage of enum class
+  return "Unknown";
+}
+
+const char* CaloRawFitter::getErrorTypeDescription(CaloRawFitter::RawFitterError_t fiterror)
+{
+  switch (fiterror) {
+    case CaloRawFitter::RawFitterError_t::SAMPLE_UNINITIALIZED:
+      return "Sample for fit not initialzied or bunch length is 0";
+    case CaloRawFitter::RawFitterError_t::FIT_ERROR:
+      return "Fit of the raw bunch was not successful";
+    case CaloRawFitter::RawFitterError_t::CHI2_ERROR:
+      return "Chi2 of the fit could not be determined";
+    case CaloRawFitter::RawFitterError_t::BUNCH_NOT_OK:
+      return "Calo bunch could not be selected";
+    case CaloRawFitter::RawFitterError_t::LOW_SIGNAL:
+      return "No ADC value above threshold found";
+  };
+  // Silence compiler warnings for false positives
+  // can never enter here due to usage of enum class
+  return "Unknown error code";
+}
+
 // Default constructor
 CaloRawFitter::CaloRawFitter(const char* name, const char* nameshort) : mMinTimeIndex(-1),
                                                                         mMaxTimeIndex(-1),
@@ -331,3 +393,9 @@ std::tuple CaloRawFitter::preFit
 
   return std::make_tuple(nsamples, bunchindex, peakADC, adcMAX, indexMaxADCRReveresed, pedestal, first, last);
 }
+
+std::ostream& o2::emcal::operator<<(std::ostream& stream, const CaloRawFitter::RawFitterError_t error)
+{
+  stream << CaloRawFitter::getErrorTypeName(error);
+  return stream;
+}
diff --git a/Detectors/EMCAL/reconstruction/src/CaloRawFitterGamma2.cxx b/Detectors/EMCAL/reconstruction/src/CaloRawFitterGamma2.cxx
index 5fa26df31f511..cf5ba2dc6cd60 100644
--- a/Detectors/EMCAL/reconstruction/src/CaloRawFitterGamma2.cxx
+++ b/Detectors/EMCAL/reconstruction/src/CaloRawFitterGamma2.cxx
@@ -12,7 +12,7 @@
 /// \file CaloRawFitterGamma2.cxx
 /// \author Martin Poghosyan (Martin.Poghosyan@cern.ch)
 
-#include "FairLogger.h"
+#include 
 #include 
 #include 
 
diff --git a/Detectors/EMCAL/reconstruction/src/CaloRawFitterStandard.cxx b/Detectors/EMCAL/reconstruction/src/CaloRawFitterStandard.cxx
index 6aa221588c50a..7899fafc0fe44 100644
--- a/Detectors/EMCAL/reconstruction/src/CaloRawFitterStandard.cxx
+++ b/Detectors/EMCAL/reconstruction/src/CaloRawFitterStandard.cxx
@@ -12,7 +12,7 @@
 /// \file CaloRawFitterStandard.cxx
 /// \author Hadi Hassan (hadi.hassan@cern.ch)
 
-#include "FairLogger.h"
+#include 
 #include 
 
 // ROOT sytem
diff --git a/Detectors/EMCAL/reconstruction/src/Clusterizer.cxx b/Detectors/EMCAL/reconstruction/src/Clusterizer.cxx
index d85ab333d79b6..96541aaff09f4 100644
--- a/Detectors/EMCAL/reconstruction/src/Clusterizer.cxx
+++ b/Detectors/EMCAL/reconstruction/src/Clusterizer.cxx
@@ -13,29 +13,23 @@
 /// \brief Implementation of the EMCAL clusterizer
 #include 
 #include 
-#include "FairLogger.h" // for LOG
+#include  // for LOG
 #include "EMCALReconstruction/Clusterizer.h"
 
 using namespace o2::emcal;
 
-///
-/// Constructor
 //____________________________________________________________________________
 template 
 Clusterizer::Clusterizer(double timeCut, double timeMin, double timeMax, double gradientCut, bool doEnergyGradientCut, double thresholdSeedE, double thresholdCellE) : mSeedList(), mInputMap(), mCellMask(), mTimeCut(timeCut), mTimeMin(timeMin), mTimeMax(timeMax), mGradientCut(gradientCut), mDoEnergyGradientCut(doEnergyGradientCut), mThresholdSeedEnergy(thresholdSeedE), mThresholdCellEnergy(thresholdCellE)
 {
 }
 
-///
-/// Default constructor
 //____________________________________________________________________________
 template 
 Clusterizer::Clusterizer() : mSeedList(), mInputMap(), mCellMask(), mTimeCut(0), mTimeMin(0), mTimeMax(0), mGradientCut(0), mDoEnergyGradientCut(false), mThresholdSeedEnergy(0), mThresholdCellEnergy(0)
 {
 }
 
-///
-/// Initialize class member vars if not done in constructor
 //____________________________________________________________________________
 template 
 void Clusterizer::initialize(double timeCut, double timeMin, double timeMax, double gradientCut, bool doEnergyGradientCut, double thresholdSeedE, double thresholdCellE)
@@ -49,8 +43,6 @@ void Clusterizer::initialize(double timeCut, double timeMin, double t
   mThresholdCellEnergy = thresholdCellE;
 }
 
-///
-/// Recursively search for neighbours (EMCAL)
 //____________________________________________________________________________
 template 
 void Clusterizer::getClusterFromNeighbours(std::vector& clusterInputs, int row, int column)
@@ -76,21 +68,19 @@ void Clusterizer::getClusterFromNeighbours(std::vectorgetEnergy() > mInputMap[row][column].mInput->getEnergy() + mGradientCut)) {
-          if (not(TMath::Abs(mInputMap[row + rowDiffs[dir]][column + colDiffs[dir]].mInput->getTimeStamp() - mInputMap[row][column].mInput->getTimeStamp()) > mTimeCut)) {
-            getClusterFromNeighbours(clusterInputs, row + rowDiffs[dir], column + colDiffs[dir]);
-            // Add the cell/digit to the current cluster -- if we end up here, the selected cluster fulfills the condition
-            clusterInputs.emplace_back(mInputMap[row + rowDiffs[dir]][column + colDiffs[dir]]);
-          }
+        if (mDoEnergyGradientCut && (mInputMap[row + rowDiffs[dir]][column + colDiffs[dir]].mInput->getEnergy() > mInputMap[row][column].mInput->getEnergy() + mGradientCut)) {
+          continue;
+        }
+        if (not(TMath::Abs(mInputMap[row + rowDiffs[dir]][column + colDiffs[dir]].mInput->getTimeStamp() - mInputMap[row][column].mInput->getTimeStamp()) > mTimeCut)) {
+          getClusterFromNeighbours(clusterInputs, row + rowDiffs[dir], column + colDiffs[dir]);
+          // Add the cell/digit to the current cluster -- if we end up here, the selected cluster fulfills the condition
+          clusterInputs.emplace_back(mInputMap[row + rowDiffs[dir]][column + colDiffs[dir]]);
         }
       }
     }
   }
 }
 
-///
-/// Get row (phi) and column (eta) of a cell/digit, values corresponding to topology
-///
 //____________________________________________________________________________
 template 
 void Clusterizer::getTopologicalRowColumn(const InputType& input, int& row, int& column)
@@ -119,8 +109,6 @@ void Clusterizer::getTopologicalRowColumn(const InputType& input, int
   }
 }
 
-///
-/// Return number of found clusters. Start clustering from highest energy cell.
 //____________________________________________________________________________
 template 
 void Clusterizer::findClusters(const gsl::span& inputArray)
diff --git a/Detectors/EMCAL/reconstruction/src/ClusterizerTask.cxx b/Detectors/EMCAL/reconstruction/src/ClusterizerTask.cxx
index 4d938462eb4b4..9385e8b43aabe 100644
--- a/Detectors/EMCAL/reconstruction/src/ClusterizerTask.cxx
+++ b/Detectors/EMCAL/reconstruction/src/ClusterizerTask.cxx
@@ -17,7 +17,7 @@
 #include 
 #include 
 #include 
-#include "FairLogger.h"      // for LOG
+#include  // for LOG
 #include "FairRootManager.h" // for FairRootManager
 #include "EMCALReconstruction/ClusterizerTask.h"
 #include "DataFormatsEMCAL/Cluster.h"
diff --git a/Detectors/EMCAL/reconstruction/src/DigitReader.cxx b/Detectors/EMCAL/reconstruction/src/DigitReader.cxx
index 67350690a44c8..3e6fb8ba41c90 100644
--- a/Detectors/EMCAL/reconstruction/src/DigitReader.cxx
+++ b/Detectors/EMCAL/reconstruction/src/DigitReader.cxx
@@ -14,7 +14,7 @@
 
 #include "CommonUtils/RootChain.h"
 #include "EMCALReconstruction/DigitReader.h"
-#include "FairLogger.h" // for LOG
+#include  // for LOG
 
 using namespace o2::emcal;
 using o2::emcal::Cell;
diff --git a/Detectors/EMCAL/reconstruction/src/EMCALReconstructionLinkDef.h b/Detectors/EMCAL/reconstruction/src/EMCALReconstructionLinkDef.h
index 4a6e835b25871..ee39958db5add 100644
--- a/Detectors/EMCAL/reconstruction/src/EMCALReconstructionLinkDef.h
+++ b/Detectors/EMCAL/reconstruction/src/EMCALReconstructionLinkDef.h
@@ -24,10 +24,14 @@
 #pragma link C++ class o2::emcal::CaloRawFitter + ;
 #pragma link C++ class o2::emcal::CaloRawFitterStandard + ;
 #pragma link C++ class o2::emcal::CaloRawFitterGamma2 + ;
+#pragma link C++ class o2::emcal::StuDecoder + ;
+#pragma link C++ class o2::emcal::FastORTimeSeries + ;
+#pragma link C++ class o2::emcal::TRUDataHandler + ;
 
 #pragma link C++ class o2::emcal::RecoParam + ;
+#pragma link C++ class o2::conf::ConfigurableParamHelper < o2::emcal::RecoParam> + ;
 
-//#pragma link C++ namespace o2::emcal+;
+// #pragma link C++ namespace o2::emcal+;
 #pragma link C++ class o2::emcal::ClusterizerParameters + ;
 #pragma link C++ class o2::emcal::Clusterizer < o2::emcal::Cell> + ;
 #pragma link C++ class o2::emcal::Clusterizer < o2::emcal::Digit> + ;
diff --git a/Detectors/EMCAL/reconstruction/src/FastORTimeSeries.cxx b/Detectors/EMCAL/reconstruction/src/FastORTimeSeries.cxx
new file mode 100644
index 0000000000000..67ada70d6008c
--- /dev/null
+++ b/Detectors/EMCAL/reconstruction/src/FastORTimeSeries.cxx
@@ -0,0 +1,50 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include 
+#include 
+#include "EMCALReconstruction/FastORTimeSeries.h"
+#include "EMCALReconstruction/TRUDecodingErrors.h"
+
+using namespace o2::emcal;
+
+void FastORTimeSeries::fillReversed(const gsl::span samples, uint8_t starttime)
+{
+  if (starttime >= 14) {
+    throw FastOrStartTimeInvalidException(starttime);
+  }
+  if (starttime + 1 < samples.size()) {
+    throw FastOrStartTimeInvalidException(static_cast(starttime) - static_cast(samples.size()) + 1);
+  }
+  for (std::size_t isample = 0; isample < samples.size(); isample++) {
+    mTimeSamples[starttime - isample] = samples[isample];
+  }
+}
+
+uint16_t FastORTimeSeries::calculateL1TimeSum(uint8_t l0time) const
+{
+  uint16_t timesum = 0;
+  int firstbin = l0time - 4; // Include sample before the L0 time
+  for (int isample = firstbin; isample < firstbin + 4; isample++) {
+    timesum += mTimeSamples[isample];
+  }
+  return timesum;
+}
+
+void FastORTimeSeries::setSize(int maxsamples)
+{
+  mTimeSamples.resize(maxsamples);
+}
+
+void FastORTimeSeries::clear()
+{
+  std::fill(mTimeSamples.begin(), mTimeSamples.end(), 0);
+}
\ No newline at end of file
diff --git a/Detectors/EMCAL/reconstruction/src/RawBuffer.cxx b/Detectors/EMCAL/reconstruction/src/RawBuffer.cxx
index c358b3169fb7f..d66a9e0d5e387 100644
--- a/Detectors/EMCAL/reconstruction/src/RawBuffer.cxx
+++ b/Detectors/EMCAL/reconstruction/src/RawBuffer.cxx
@@ -11,7 +11,7 @@
 #include 
 #include 
 #include 
-#include 
+#include 
 #include "EMCALReconstruction/RawBuffer.h"
 
 using namespace o2::emcal;
diff --git a/Detectors/EMCAL/reconstruction/src/RawDecodingError.cxx b/Detectors/EMCAL/reconstruction/src/RawDecodingError.cxx
new file mode 100644
index 0000000000000..db0bcb1b74269
--- /dev/null
+++ b/Detectors/EMCAL/reconstruction/src/RawDecodingError.cxx
@@ -0,0 +1,25 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include 
+#include 
+
+std::ostream& o2::emcal::operator<<(std::ostream& stream, const o2::emcal::RawDecodingError& error)
+{
+  stream << error.what();
+  return stream;
+}
+
+std::ostream& o2::emcal::operator<<(std::ostream& stream, const o2::emcal::RawDecodingError::ErrorType_t& error)
+{
+  stream << o2::emcal::RawDecodingError::getErrorCodeNames(error);
+  return stream;
+}
\ No newline at end of file
diff --git a/Detectors/EMCAL/reconstruction/src/RawHeaderStream.cxx b/Detectors/EMCAL/reconstruction/src/RawHeaderStream.cxx
deleted file mode 100644
index a046656f3a67e..0000000000000
--- a/Detectors/EMCAL/reconstruction/src/RawHeaderStream.cxx
+++ /dev/null
@@ -1,107 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-#include 
-#include 
-#include "EMCALReconstruction/RawHeaderStream.h"
-
-std::istream& o2::emcal::operator>>(std::istream& stream, o2::header::RAWDataHeaderV4& header)
-{
-  //std::cout << "called, 10 words" << std::endl;
-  using wordtype = uint64_t;
-  constexpr int RAWHEADERWORDS = sizeof(header) / sizeof(wordtype);
-  wordtype message[RAWHEADERWORDS];
-  auto address = reinterpret_cast(message);
-  for (int i = 0; i < RAWHEADERWORDS; i++) {
-    stream.read(address + i * sizeof(wordtype) / sizeof(char), sizeof(message[i]));
-    //std::cout << "Word " << i << ":  " << std::hex << message[i] << std::dec << std::endl;
-  }
-  header.word0 = message[0];
-  header.word1 = message[1];
-  header.word2 = message[2];
-  header.word3 = message[3];
-  header.word4 = message[4];
-  header.word5 = message[5];
-  header.word6 = message[6];
-  header.word7 = message[7];
-  return stream;
-}
-
-std::istream& o2::emcal::operator>>(std::istream& stream, o2::header::RAWDataHeaderV5& header)
-{
-  //std::cout << "called, 10 words" << std::endl;
-  using wordtype = uint64_t;
-  constexpr int RAWHEADERWORDS = sizeof(header) / sizeof(wordtype);
-  wordtype message[RAWHEADERWORDS];
-  auto address = reinterpret_cast(message);
-  for (int i = 0; i < RAWHEADERWORDS; i++) {
-    stream.read(address + i * sizeof(wordtype) / sizeof(char), sizeof(message[i]));
-    //std::cout << "Word " << i << ":  " << std::hex << message[i] << std::dec << std::endl;
-  }
-  header.word0 = message[0];
-  header.word1 = message[1];
-  header.word2 = message[2];
-  header.word3 = message[3];
-  header.word4 = message[4];
-  header.word5 = message[5];
-  header.word6 = message[6];
-  header.word7 = message[7];
-  return stream;
-}
-
-std::ostream& o2::emcal::operator<<(std::ostream& stream, const o2::header::RAWDataHeaderV4& header)
-{
-  stream << "Raw data header V4:\n"
-         << "  Word0 " << header.word0 << " (0x" << std::hex << header.word0 << std::dec << ")\n"
-         << "    Version:         " << header.version << "\n"
-         << "    Header size:     " << header.headerSize << "\n"
-         << "    Block length:    " << header.blockLength << "\n"
-         << "    FEE ID:          " << header.feeId << "\n"
-         << "    Priority:        " << header.priority << "\n"
-         << "  Word1 " << header.word1 << " (0x" << std::hex << header.word1 << std::dec << ")\n"
-         << "    Offset to next:  " << header.offsetToNext << "\n"
-         << "    Payload size (B):" << header.memorySize << "\n"
-         << "    Packet counter:  " << static_cast(header.packetCounter) << "\n"
-         << "    Link ID:         " << static_cast(header.linkID) << "\n"
-         << "    Card ID:         " << header.cruID << "\n"
-         << "    Endpoint:        " << static_cast(header.endPointID) << "\n"
-         << "  Word2 " << header.word2 << " (0x" << std::hex << header.word2 << std::dec << ")\n"
-         << "    Trigger orbit:   " << header.triggerOrbit << "\n"
-         << "    Heartbeat orbit: " << header.heartbeatOrbit << "\n"
-         << "  Word3 " << header.word3 << " (0x" << std::hex << header.word3 << std::dec << ")\n"
-         << "  Word4 " << header.word4 << " (0x" << std::hex << header.word4 << std::dec << ")\n"
-         << "    Trigger BC:      " << header.triggerBC << "\n"
-         << "    Heartbeat BC:    " << header.heartbeatBC << "\n"
-         << "    Trigger Type:    " << header.triggerType << "\n"
-         << "  Word5 " << header.word5 << " (0x" << std::hex << header.word5 << std::dec << ")\n"
-         << "  Word6 " << header.word6 << " (0x" << std::hex << header.word6 << std::dec << ")\n"
-         << "    Detector Field:  " << header.detectorField << "\n"
-         << "    PAR:             " << header.par << "\n"
-         << "    STOP:            " << header.stop << "\n"
-         << "    Page count:      " << header.pageCnt << "\n"
-         << "  Word7 " << header.word7 << " (0x" << std::hex << header.word7 << std::dec << ")\n"
-         << "End header\n";
-  return stream;
-}
-
-std::ostream& o2::emcal::operator<<(std::ostream& stream, const o2::header::RAWDataHeaderV5& header)
-{
-  stream << "Raw data header V5:\n"
-         << "   Word0 " << header.word0 << " (0x" << std::hex << header.word0 << std::dec << ")\n"
-         << "   Word1 " << header.word1 << " (0x" << std::hex << header.word1 << std::dec << ")\n"
-         << "   Word2 " << header.word2 << " (0x" << std::hex << header.word2 << std::dec << ")\n"
-         << "   Word3 " << header.word3 << " (0x" << std::hex << header.word3 << std::dec << ")\n"
-         << "   Word4 " << header.word4 << " (0x" << std::hex << header.word4 << std::dec << ")\n"
-         << "   Word5 " << header.word5 << " (0x" << std::hex << header.word5 << std::dec << ")\n"
-         << "   Word6 " << header.word6 << " (0x" << std::hex << header.word6 << std::dec << ")\n"
-         << "   Word7 " << header.word7 << " (0x" << std::hex << header.word7 << std::dec << ")\n"
-         << "End header\n";
-  return stream;
-}
\ No newline at end of file
diff --git a/Detectors/EMCAL/reconstruction/src/RawReaderMemory.cxx b/Detectors/EMCAL/reconstruction/src/RawReaderMemory.cxx
index a4cdabd0e953f..8ccbc5d63104a 100644
--- a/Detectors/EMCAL/reconstruction/src/RawReaderMemory.cxx
+++ b/Detectors/EMCAL/reconstruction/src/RawReaderMemory.cxx
@@ -8,12 +8,13 @@
 // In applying this license CERN does not waive the privileges and immunities
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
-
+#include 
 #include 
 #include 
 #include "EMCALReconstruction/RawReaderMemory.h"
-#include "EMCALReconstruction/RawDecodingError.h"
 #include "DetectorsRaw/RDHUtils.h"
+#include "Headers/RAWDataHeader.h"
+#include "Headers/DAQID.h"
 
 using namespace o2::emcal;
 
@@ -33,14 +34,20 @@ void RawReaderMemory::setRawMemory(const gsl::span rawmemory)
 o2::header::RDHAny RawReaderMemory::decodeRawHeader(const void* payloadwords)
 {
   auto headerversion = RDHDecoder::getVersion(payloadwords);
-  if (headerversion == 4) {
-    return o2::header::RDHAny(*reinterpret_cast(payloadwords));
-  } else if (headerversion == 5) {
-    return o2::header::RDHAny(*reinterpret_cast(payloadwords));
-  } else if (headerversion == 6) {
-    return o2::header::RDHAny(*reinterpret_cast(payloadwords));
+  if (headerversion < RDHDecoder::getVersion() || headerversion > RDHDecoder::getVersion()) {
+    throw RawDecodingError(RawDecodingError::ErrorType_t::HEADER_DECODING, mCurrentFEE);
+  }
+  if (!RDHDecoder::checkRDH(payloadwords, false, true)) {
+    // Header size != to 64 and 0 in reserved words indicate that the header is a fake header
+    throw RawDecodingError(RawDecodingError::ErrorType_t::HEADER_DECODING, mCurrentFEE);
   }
-  throw RawDecodingError(RawDecodingError::ErrorType_t::HEADER_DECODING, mCurrentFEE);
+  if (RDHDecoder::getVersion(payloadwords) >= 6) {
+    // If header version is >= 6 check that the source ID is EMCAL
+    if (RDHDecoder::getSourceID(payloadwords) != o2::header::DAQID::EMC) {
+      throw RawDecodingError(RawDecodingError::ErrorType_t::HEADER_DECODING, mCurrentFEE);
+    }
+  }
+  return {*reinterpret_cast(payloadwords)};
 }
 
 void RawReaderMemory::init()
@@ -55,6 +62,7 @@ void RawReaderMemory::next()
 {
   mRawPayload.reset();
   mCurrentTrailer.reset();
+  mMinorErrors.clear();
   bool isDataTerminated = false;
   do {
     nextPage(false);
@@ -121,9 +129,31 @@ void RawReaderMemory::nextPage(bool doResetPayload)
   }
   if (mCurrentPosition + RDHDecoder::getMemorySize(mRawHeader) > mRawMemoryBuffer.size()) {
     // Payload incomplete
+    // Set to offset to next or buffer size, whatever is smaller
+    if (mCurrentPosition + RDHDecoder::getOffsetToNext(mRawHeader) < mRawMemoryBuffer.size()) {
+      mCurrentPosition += RDHDecoder::getOffsetToNext(mRawHeader);
+      if (mCurrentPosition + 64 < mRawMemoryBuffer.size()) {
+        // check if the page continues with a RDH
+        // if next page is not a header decodeRawHeader will throw a RawDecodingError
+        auto nextheader = decodeRawHeader(mRawMemoryBuffer.data() + mCurrentPosition);
+        if (RDHDecoder::getVersion(nextheader) != RDHDecoder::getVersion(mRawHeader)) {
+          // Next header has different header version - clear indication that it is a fake header
+          throw RawDecodingError(RawDecodingError::ErrorType_t::HEADER_DECODING, mCurrentFEE);
+        }
+      }
+    } else {
+      mCurrentPosition = mRawMemoryBuffer.size();
+    }
+    // RDHDecoder::printRDH(mRawHeader);
     throw RawDecodingError(RawDecodingError::ErrorType_t::PAYLOAD_DECODING, mCurrentFEE);
   } else if (mCurrentPosition + RDHDecoder::getHeaderSize(mRawHeader) > mRawMemoryBuffer.size()) {
     // Start position of the payload is outside the payload range
+    // Set to offset to next or buffer size, whatever is smaller
+    if (mCurrentPosition + RDHDecoder::getOffsetToNext(mRawHeader) < mRawMemoryBuffer.size()) {
+      mCurrentPosition += RDHDecoder::getOffsetToNext(mRawHeader);
+    } else {
+      mCurrentPosition = mRawMemoryBuffer.size();
+    }
     throw RawDecodingError(RawDecodingError::ErrorType_t::PAGE_START_INVALID, mCurrentFEE);
   } else {
     mRawBuffer.readFromMemoryBuffer(gsl::span(mRawMemoryBuffer.data() + mCurrentPosition + RDHDecoder::getHeaderSize(mRawHeader), RDHDecoder::getMemorySize(mRawHeader) - RDHDecoder::getHeaderSize(mRawHeader)));
@@ -143,7 +173,7 @@ void RawReaderMemory::nextPage(bool doResetPayload)
       // The trailer is only decoded if the DDL is in the range of SRU DDLs. STU pages are propagated
       // 1-1 without trailer parsing
       auto lastword = *(mRawBuffer.getDataWords().rbegin());
-      if (lastword >> 30 == 3) {
+      if (RCUTrailer::checkLastTrailerWord(lastword)) {
         // lastword is a trailer word
         // decode trailer and chop
         try {
@@ -154,7 +184,17 @@ void RawReaderMemory::nextPage(bool doResetPayload)
             mCurrentTrailer.setPayloadSize(mCurrentTrailer.getPayloadSize() + trailer.getPayloadSize());
           }
           payloadWithoutTrailer = gsl::span(mRawBuffer.getDataWords().data(), mRawBuffer.getDataWords().size() - trailer.getTrailerSize());
+          if (trailer.getTrailerWordCorruptions()) {
+            mMinorErrors.emplace_back(RawDecodingError::ErrorType_t::TRAILER_INCOMPLETE, mCurrentFEE);
+          }
         } catch (RCUTrailer::Error& e) {
+          // We must forward the position in such cases in order to not end up in an infinity loop
+          if (mCurrentPosition + RDHDecoder::getOffsetToNext(mRawHeader) < mRawMemoryBuffer.size()) {
+            mCurrentPosition += RDHDecoder::getOffsetToNext(mRawHeader);
+          } else {
+            mCurrentPosition = mRawMemoryBuffer.size();
+          }
+          // Page is not consistent - must be skipped
           throw RawDecodingError(RawDecodingError::ErrorType_t::TRAILER_DECODING, mCurrentFEE);
         }
       } else {
diff --git a/Detectors/EMCAL/reconstruction/src/RecoContainer.cxx b/Detectors/EMCAL/reconstruction/src/RecoContainer.cxx
new file mode 100644
index 0000000000000..062887287f86b
--- /dev/null
+++ b/Detectors/EMCAL/reconstruction/src/RecoContainer.cxx
@@ -0,0 +1,183 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include 
+#include 
+#include 
+#include 
+
+using namespace o2::emcal;
+
+EventContainer::EventContainer(const o2::InteractionRecord& currentIR) : mInteractionRecord(currentIR) { initTRUs(); }
+
+void EventContainer::initTRUs()
+{
+  for (auto index = 0; index < mTRUData.size(); index++) {
+    mTRUData[index].setTRUIndex(index);
+    mTRUData[index].setL0time(INT8_MAX);
+  }
+}
+
+void EventContainer::setCellCommon(int tower, double energy, double time, ChannelType_t celltype, bool isLEDmon, int hwaddress, int ddlID, bool doMergeHGLG)
+{
+  auto updateCell = [energy, time, celltype](Cell& cell) {
+    cell.setEnergy(energy);
+    cell.setTimeStamp(time);
+    cell.setType(celltype);
+  };
+  auto& datacontainer = isLEDmon ? mLEDMons : mCells;
+  if (doMergeHGLG) {
+    auto found = std::find_if(datacontainer.begin(), datacontainer.end(), [&tower](const o2::emcal::RecCellInfo& testcell) -> bool { return testcell.mCellData.getTower() == tower; });
+    if (found != datacontainer.end()) {
+      // Cell already existing, store LG if HG is larger then the overflow cut
+      if (celltype == o2::emcal::ChannelType_t::LOW_GAIN) {
+        found->mHWAddressLG = hwaddress;
+        found->mHGOutOfRange = false; // LG is found so it can replace the HG if the HG is out of range
+        if (found->mCellData.getHighGain()) {
+          if (isCellSaturated(found->mCellData.getEnergy())) {
+            // High gain digit has energy above overflow cut, use low gain instead
+            updateCell(found->mCellData);
+          }
+          found->mIsLGnoHG = false;
+        }
+      } else {
+        // new channel would be HG: use that if it is below overflow cut
+        // as the channel existed before it must have been a LG channel,
+        // which would be used in case the HG is out-of-range
+        found->mIsLGnoHG = false;
+        found->mHGOutOfRange = false;
+        found->mHWAddressHG = hwaddress;
+        if (!isCellSaturated(energy)) {
+          updateCell(found->mCellData);
+        }
+      }
+    } else {
+      // New cell
+      bool lgNoHG = false;       // Flag for filter of cells which have only low gain but no high gain
+      bool hgOutOfRange = false; // Flag if only a HG is present which is out-of-range
+      int hwAddressLG = -1,      // Hardware address of the LG of the tower (for monitoring)
+        hwAddressHG = -1;        // Hardware address of the HG of the tower (for monitoring)
+      if (celltype == o2::emcal::ChannelType_t::LOW_GAIN) {
+        lgNoHG = true;
+        hwAddressLG = hwaddress;
+      } else {
+        // High gain cell: Flag as low gain if above threshold
+        hgOutOfRange = isCellSaturated(energy);
+        hwAddressHG = hwaddress;
+      }
+      datacontainer.push_back({o2::emcal::Cell(tower, energy, time, celltype),
+                               lgNoHG,
+                               hgOutOfRange,
+                               ddlID, hwAddressLG, hwAddressHG});
+    }
+  } else {
+    // No merge of HG/LG cells (usually MC where either
+    // of the two is simulated)
+    int hwAddressLG = celltype == ChannelType_t::LOW_GAIN ? hwaddress : -1,
+        hwAddressHG = celltype == ChannelType_t::HIGH_GAIN ? hwaddress : -1;
+    // New cell
+    datacontainer.push_back({o2::emcal::Cell(tower, energy, time, celltype),
+                             false,
+                             false,
+                             ddlID, hwAddressLG, hwAddressHG});
+  }
+}
+
+void EventContainer::setFastOR(uint16_t fastORAbsID, uint8_t starttime, const gsl::span timesamples)
+{
+  auto found = mL0FastORs.find(fastORAbsID);
+  if (found != mL0FastORs.end()) {
+    found->second.setTimeSamples(timesamples, starttime);
+  } else {
+    mL0FastORs[fastORAbsID] = FastORTimeSeries(14, timesamples, starttime);
+  }
+}
+
+void EventContainer::sortCells(bool isLEDmon)
+{
+  auto& dataContainer = isLEDmon ? mLEDMons : mCells;
+  std::sort(dataContainer.begin(), dataContainer.end(), [](const RecCellInfo& lhs, const RecCellInfo& rhs) -> bool { return lhs.mCellData.getTower() < rhs.mCellData.getTower(); });
+}
+
+bool EventContainer::isCellSaturated(double energy) const
+{
+  return energy / o2::emcal::constants::EMCAL_ADCENERGY > o2::emcal::constants::OVERFLOWCUT;
+}
+
+TRUDataHandler& EventContainer::getTRUData(std::size_t index)
+{
+  if (index >= mTRUData.size()) {
+    throw TRUIndexException(index);
+  }
+  return mTRUData[index];
+}
+
+const TRUDataHandler& EventContainer::readTRUData(std::size_t index) const
+{
+  if (index >= mTRUData.size()) {
+    throw TRUIndexException(index);
+  }
+  return mTRUData[index];
+}
+
+EventContainer::TRUIndexException::TRUIndexException(std::size_t index) : mIndex(index), mMessage()
+{
+  mMessage = "Invalid TRU index " + std::to_string(index);
+}
+
+EventContainer& RecoContainer::getEventContainer(const o2::InteractionRecord& currentIR)
+{
+  auto found = mEvents.find(currentIR);
+  if (found != mEvents.end()) {
+    return found->second;
+  }
+  // interaction not found, create new container
+  auto res = mEvents.insert({currentIR, EventContainer(currentIR)});
+  return res.first->second;
+}
+
+const EventContainer& RecoContainer::getEventContainer(const o2::InteractionRecord& currentIR) const
+{
+  auto found = mEvents.find(currentIR);
+  if (found == mEvents.end()) {
+    throw InteractionNotFoundException(currentIR);
+  }
+  return found->second;
+}
+
+std::vector RecoContainer::getOrderedInteractions() const
+{
+  std::vector result;
+  for (auto& entry : mEvents) {
+    result.emplace_back(entry.first);
+  }
+  std::sort(result.begin(), result.end(), std::less<>());
+  return result;
+}
+
+RecoContainerReader::RecoContainerReader(RecoContainer& container) : mDataContainer(container)
+{
+  mOrderedInteractions = mDataContainer.getOrderedInteractions();
+}
+
+EventContainer& RecoContainerReader::nextEvent()
+{
+  if (!hasNext()) {
+    throw InvalidAccessException();
+  }
+  int currentID = mCurrentEvent;
+  mCurrentEvent++;
+  try {
+    return mDataContainer.getEventContainer(mOrderedInteractions[currentID]);
+  } catch (RecoContainer::InteractionNotFoundException& e) {
+    throw InvalidAccessException();
+  }
+}
\ No newline at end of file
diff --git a/Detectors/EMCAL/reconstruction/src/RecoParam.cxx b/Detectors/EMCAL/reconstruction/src/RecoParam.cxx
index e6f7cf3fbfa22..f80721ba9b88e 100644
--- a/Detectors/EMCAL/reconstruction/src/RecoParam.cxx
+++ b/Detectors/EMCAL/reconstruction/src/RecoParam.cxx
@@ -16,9 +16,9 @@ O2ParamImpl(o2::emcal::RecoParam);
 
 using namespace o2::emcal;
 
-std::ostream& operator<<(std::ostream& stream, const o2::emcal::RecoParam& s)
+std::ostream& o2::emcal::operator<<(std::ostream& stream, const o2::emcal::RecoParam& par)
 {
-  s.PrintStream(stream);
+  par.PrintStream(stream);
   return stream;
 }
 
@@ -28,5 +28,6 @@ void RecoParam::PrintStream(std::ostream& stream) const
          << "\n============================================="
          << "\nTime offset (ns):                 " << mCellTimeShiftNanoSec
          << "\nNoise threshold HGLG suppression: " << mNoiseThresholdLGnoHG
-         << "\nPhase in BC mod 4 correction:     " << mPhaseBCmod4;
+         << "\nPhase in BC mod 4 correction:     " << mPhaseBCmod4
+         << "\nMax number of ALTRO samples:      " << mMaxBunchLength;
 }
\ No newline at end of file
diff --git a/Detectors/EMCAL/reconstruction/src/ReconstructionErrors.cxx b/Detectors/EMCAL/reconstruction/src/ReconstructionErrors.cxx
new file mode 100644
index 0000000000000..6f1a5f5bd4f2d
--- /dev/null
+++ b/Detectors/EMCAL/reconstruction/src/ReconstructionErrors.cxx
@@ -0,0 +1,233 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#include 
+#include "EMCALReconstruction/ReconstructionErrors.h"
+
+namespace o2
+{
+namespace emcal
+{
+
+namespace reconstructionerrors
+{
+
+GeometryError_t getGeometryErrorFromErrorCode(unsigned int errorcode)
+{
+  assert(errorcode < getNumberOfGeometryErrorCodes());
+  switch (errorcode) {
+    case 0:
+      return GeometryError_t::CELL_RANGE_EXCEED;
+    case 1:
+      return GeometryError_t::CELL_INDEX_NEGATIVE;
+    default:
+      return GeometryError_t::UNKNOWN_ERROR;
+  }
+}
+
+const char* getGeometryErrorName(GeometryError_t errortype)
+{
+  switch (errortype) {
+    case GeometryError_t::CELL_RANGE_EXCEED:
+      return "CellRangeExceed";
+    case GeometryError_t::CELL_INDEX_NEGATIVE:
+      return "CellIndexNegative";
+    default:
+      return "UnknownError";
+  }
+}
+
+const char* getGeometryErrorName(unsigned int errorcode)
+{
+  return getGeometryErrorName(getGeometryErrorFromErrorCode(errorcode));
+}
+
+const char* getGeometryErrorTitle(GeometryError_t errortype)
+{
+  switch (errortype) {
+    case GeometryError_t::CELL_RANGE_EXCEED:
+      return "Cell ID outside range";
+    case GeometryError_t::CELL_INDEX_NEGATIVE:
+      return "Cell ID corrupted";
+    default:
+      return "UnknownError";
+  };
+}
+
+const char* getGeometryErrorTitle(unsigned int errorcode)
+{
+  return getGeometryErrorTitle(getGeometryErrorFromErrorCode(errorcode));
+}
+
+const char* getGeometryErrorDescription(GeometryError_t errortype)
+{
+  switch (errortype) {
+    case GeometryError_t::CELL_RANGE_EXCEED:
+      return "Cell index exceeding valid rage";
+    case GeometryError_t::CELL_INDEX_NEGATIVE:
+      return "Cell index corrupted (i.e. negative)";
+    default:
+      return "UnknownError";
+  };
+}
+
+const char* getGeometryErrorDescription(unsigned int errorcode)
+{
+  return getGeometryErrorDescription(getGeometryErrorFromErrorCode(errorcode));
+}
+
+GainError_t getGainErrorFromErrorCode(unsigned int errorcode)
+{
+  assert(errorcode < getNumberOfGainErrorCodes());
+  switch (errorcode) {
+    case 0:
+      return GainError_t::LGNOHG;
+    case 1:
+      return GainError_t::HGNOLG;
+    default:
+      return GainError_t::UNKNOWN_ERROR;
+  };
+}
+
+const char* getGainErrorName(GainError_t errortype)
+{
+  switch (errortype) {
+    case GainError_t::LGNOHG:
+      return "HGnoLG";
+    case GainError_t::HGNOLG:
+      return "LGnoHG";
+    default:
+      return "UnknownError";
+  };
+}
+
+const char* getGainErrorName(unsigned int errorcode)
+{
+  return getGainErrorName(getGainErrorFromErrorCode(errorcode));
+}
+
+const char* getGainErrorTitle(GainError_t errortype)
+{
+  switch (errortype) {
+    case GainError_t::LGNOHG:
+      return "High Gain missing";
+    case GainError_t::HGNOLG:
+      return "Low Gain missing";
+    default:
+      return "UnknownError";
+  };
+}
+
+const char* getGainErrorTitle(unsigned int errorcode)
+{
+  return getGainErrorTitle(getGainErrorFromErrorCode(errorcode));
+}
+
+const char* getGainErrorDescription(GainError_t errortype)
+{
+  switch (errortype) {
+    case GainError_t::LGNOHG:
+      return "HG missing for LG below HGLG transition";
+    case GainError_t::HGNOLG:
+      return "LG not found for saturated HG";
+    default:
+      return "UnknownError";
+  };
+}
+
+const char* getGainErrorDescription(unsigned int errorcode)
+{
+  return getGainErrorDescription(getGainErrorFromErrorCode(errorcode));
+}
+
+TRUDecodingError_t getTRUDecodingErrorFromErrorCode(unsigned int errorcode)
+{
+  switch (errorcode) {
+    case 0:
+      return TRUDecodingError_t::TRU_INDEX_INVALID;
+    case 1:
+      return TRUDecodingError_t::PATCH_INDEX_INVALID;
+    case 2:
+      return TRUDecodingError_t::FASTOR_INDEX_INVALID;
+    case 3:
+      return TRUDecodingError_t::FASTOR_STARTTIME_INVALID;
+    default:
+      return TRUDecodingError_t::UNKNOWN_ERROR;
+  }
+}
+
+const char* getTRUDecodingErrorName(TRUDecodingError_t errortype)
+{
+  switch (errortype) {
+    case TRUDecodingError_t::TRU_INDEX_INVALID:
+      return "TRUIndexInvalid";
+    case TRUDecodingError_t::PATCH_INDEX_INVALID:
+      return "PatchIndexInvalid";
+    case TRUDecodingError_t::FASTOR_INDEX_INVALID:
+      return "FastORIndexInvalid";
+    case TRUDecodingError_t::FASTOR_STARTTIME_INVALID:
+      return "FastORStartTimeInvalid";
+    default:
+      return "UnknownError";
+  }
+}
+
+const char* getTRUDecodingErrorName(unsigned int errorcode)
+{
+  return getTRUDecodingErrorName(getTRUDecodingErrorFromErrorCode(errorcode));
+}
+
+const char* getTRUDecodingErrorTitle(TRUDecodingError_t errortype)
+{
+  switch (errortype) {
+    case TRUDecodingError_t::TRU_INDEX_INVALID:
+      return "TRU index invalid";
+    case TRUDecodingError_t::PATCH_INDEX_INVALID:
+      return "Patch index invalid";
+    case TRUDecodingError_t::FASTOR_INDEX_INVALID:
+      return "FastOR index invalid";
+    case TRUDecodingError_t::FASTOR_STARTTIME_INVALID:
+      return "FastOR starttime invalid";
+    default:
+      return "Unknown error";
+  }
+}
+
+const char* getTRUDecodingErrorTitle(unsigned int errortype)
+{
+  return getTRUDecodingErrorTitle(getTRUDecodingErrorFromErrorCode(errortype));
+}
+
+const char* getTRUDecodingErrorErrorDescription(TRUDecodingError_t errortype)
+{
+  switch (errortype) {
+    case TRUDecodingError_t::TRU_INDEX_INVALID:
+      return "TRU index is invalid";
+    case TRUDecodingError_t::PATCH_INDEX_INVALID:
+      return "Patch index is invalid";
+    case TRUDecodingError_t::FASTOR_INDEX_INVALID:
+      return "FastOR index is invalid";
+    case TRUDecodingError_t::FASTOR_STARTTIME_INVALID:
+      return "FastOR starttime is invalid";
+    default:
+      return "Unknown error";
+  }
+}
+
+const char* getTRUDecodingErrorErrorDescription(unsigned int errorcode)
+{
+  return getTRUDecodingErrorErrorDescription(getTRUDecodingErrorFromErrorCode(errorcode));
+}
+
+} // namespace reconstructionerrors
+
+} // namespace emcal
+
+} // namespace o2
\ No newline at end of file
diff --git a/Detectors/EMCAL/reconstruction/src/STUDecoderError.cxx b/Detectors/EMCAL/reconstruction/src/STUDecoderError.cxx
new file mode 100644
index 0000000000000..bc88dc02199fa
--- /dev/null
+++ b/Detectors/EMCAL/reconstruction/src/STUDecoderError.cxx
@@ -0,0 +1,110 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#include 
+#include 
+#include "EMCALReconstruction/STUDecoderError.h"
+
+using namespace o2::emcal;
+
+STUDecoderError::STUDecoderError(int ddlID, ErrorCode_t errcode) : mDDLId(ddlID),
+                                                                   mErrorCode(errcode)
+{
+  mMessage = "STU decoding error of type " + getErrorCodeTitle(mErrorCode) + " in DDL " + std::to_string(mDDLId);
+}
+
+void STUDecoderError::printStream(std::ostream& stream) const
+{
+}
+
+std::ostream& o2::emcal::operator<<(std::ostream& stream, const STUDecoderError& error)
+{
+  error.printStream(stream);
+  return stream;
+}
+
+int STUDecoderError::errorCodeToInt(ErrorCode_t errorcode)
+{
+  switch (errorcode) {
+    case ErrorCode_t::PAGE_ERROR:
+      return 0;
+    case ErrorCode_t::WORD_UNEXPECTED:
+      return 1;
+    case ErrorCode_t::INDEX_UNEXPECTED:
+      return 2;
+    case ErrorCode_t::ADC_OVERFLOW:
+      return 3;
+    case ErrorCode_t::FEEID_UNEXPECTED:
+      return 4;
+    case ErrorCode_t::OLD_PAYLOAD_VERSION:
+      return 5;
+    case ErrorCode_t::FULL_PAYLOAD_SIZE_UNEXPECTED:
+      return 6;
+    case ErrorCode_t::SHORT_PAYLOAD_SIZE_UNEXPECTED:
+      return 7;
+    default:
+      return -1;
+  }
+}
+STUDecoderError::ErrorCode_t STUDecoderError::intToErrorCode(int errorcode)
+{
+  static constexpr std::size_t NUMERRORCODES = 8;
+  static constexpr std::array errorcodes = {{ErrorCode_t::PAGE_ERROR, ErrorCode_t::WORD_UNEXPECTED, ErrorCode_t::INDEX_UNEXPECTED, ErrorCode_t::ADC_OVERFLOW, ErrorCode_t::FEEID_UNEXPECTED, ErrorCode_t::OLD_PAYLOAD_VERSION, ErrorCode_t::FULL_PAYLOAD_SIZE_UNEXPECTED, ErrorCode_t::SHORT_PAYLOAD_SIZE_UNEXPECTED}};
+  if (errorcode < 0 || errorcode >= NUMERRORCODES) {
+    return ErrorCode_t::UNKNOWN;
+  }
+  return errorcodes[errorcode];
+}
+std::string STUDecoderError::getErrorCodeName(ErrorCode_t errorcode)
+{
+  switch (errorcode) {
+    case ErrorCode_t::PAGE_ERROR:
+      return "PageDecoding";
+    case ErrorCode_t::WORD_UNEXPECTED:
+      return "WordUnexpected";
+    case ErrorCode_t::INDEX_UNEXPECTED:
+      return "IndexUnexpected";
+    case ErrorCode_t::ADC_OVERFLOW:
+      return "ADCOverflow";
+    case ErrorCode_t::FEEID_UNEXPECTED:
+      return "FeeIdUnexpected";
+    case ErrorCode_t::OLD_PAYLOAD_VERSION:
+      return "OldPayloadVersion";
+    case ErrorCode_t::FULL_PAYLOAD_SIZE_UNEXPECTED:
+      return "FullPayloadSizeUnexpected";
+    case ErrorCode_t::SHORT_PAYLOAD_SIZE_UNEXPECTED:
+      return "ShortPayloadSizeUnexpected";
+    default:
+      return "Unknown";
+  }
+}
+std::string STUDecoderError::getErrorCodeTitle(ErrorCode_t errorcode)
+{
+  switch (errorcode) {
+    case ErrorCode_t::PAGE_ERROR:
+      return "page decoding";
+    case ErrorCode_t::WORD_UNEXPECTED:
+      return "unexpected word";
+    case ErrorCode_t::INDEX_UNEXPECTED:
+      return "invalid patch index";
+    case ErrorCode_t::ADC_OVERFLOW:
+      return "ADC overflow";
+    case ErrorCode_t::FEEID_UNEXPECTED:
+      return "invalid FeeID";
+    case ErrorCode_t::OLD_PAYLOAD_VERSION:
+      return "unsupported old payload version";
+    case ErrorCode_t::FULL_PAYLOAD_SIZE_UNEXPECTED:
+      return "unexpected full payload size";
+    case ErrorCode_t::SHORT_PAYLOAD_SIZE_UNEXPECTED:
+      return "unexpected short payload size";
+    default:
+      return "Unknown";
+  }
+}
\ No newline at end of file
diff --git a/Detectors/EMCAL/reconstruction/src/StuDecoder.cxx b/Detectors/EMCAL/reconstruction/src/StuDecoder.cxx
new file mode 100755
index 0000000000000..c0bd615a03c56
--- /dev/null
+++ b/Detectors/EMCAL/reconstruction/src/StuDecoder.cxx
@@ -0,0 +1,245 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#include 
+#include 
+#include 
+#include 
+#include "InfoLogger/InfoLogger.hxx"
+#include "DetectorsRaw/RDHUtils.h"
+#include "EMCALReconstruction/RawReaderMemory.h"
+#include "EMCALReconstruction/StuDecoder.h"
+#include "EMCALReconstruction/STUDecoderError.h"
+
+using namespace o2::emcal;
+using namespace o2::emcal::STUparam;
+
+StuDecoder::StuDecoder(RawReaderMemory& reader) : mRawReader(reader)
+{
+}
+
+void StuDecoder::init()
+{
+
+  auto& header = mRawReader.getRawHeader();
+  auto feeID = o2::raw::RDHUtils::getFEEID(header);
+
+  if (!((feeID == FeeID[STUtype_t::ESTU]) || (feeID == FeeID[STUtype_t::DSTU]))) {
+    throw STUDecoderError(feeID, STUDecoderError::ErrorCode_t::FEEID_UNEXPECTED);
+  }
+
+  mSTU = STUtype_t::ESTU;
+  if (feeID == FeeID[STUtype_t::DSTU]) {
+    mSTU = STUtype_t::DSTU;
+  }
+
+  //  STU payload structure is described in JIRA-EMCAL-562
+  auto& buffer = mRawReader.getPayload().getPayloadWords();
+  bool vbit = ((buffer[0] >> 31) & 0x1); // payload version control bit
+  if (!vbit) {                           // old payloads version cannot be decoded with this method; discard 2022 early data
+    throw STUDecoderError(feeID, STUDecoderError::ErrorCode_t::OLD_PAYLOAD_VERSION);
+  }
+
+  mIsFullPayload = (buffer[1] & 0x1);
+
+  auto payloadsize = mRawReader.getPayload().getPayloadWords().size();
+
+  if (mIsFullPayload) {
+    if (payloadsize != getPaloadSizeFull()) {
+      //      std::cout << "ERROR: Wrong payload size (=" << std::dec << payloadsize << ") for FullPaload \n";
+      throw STUDecoderError(feeID, STUDecoderError::ErrorCode_t::FULL_PAYLOAD_SIZE_UNEXPECTED);
+    }
+  }
+
+  else {
+    if (payloadsize != getPaloadSizeShort()) {
+      //      std::cout << "ERROR: Wrong payload size (=" << std::dec << payloadsize << ") for ShortPaload \n";
+      throw STUDecoderError(feeID, STUDecoderError::ErrorCode_t::SHORT_PAYLOAD_SIZE_UNEXPECTED);
+    }
+  }
+
+  if (mDebug >= 3) {
+    std::cout << "==== paylaod size ===\n";
+    int currentpos = 0;
+    while ((currentpos < buffer.size())) {
+      auto currentword = buffer[currentpos++];
+      std::cout << std::dec << (currentpos - 1) << ")   0x" << std::hex << currentword << std::endl;
+    }
+  }
+}
+
+void StuDecoder::decode()
+{
+
+  init();
+
+  auto& buffer = mRawReader.getPayload().getPayloadWords();
+
+  mCFGWord0 = buffer[0];
+  mCFGWord1 = buffer[1];
+  mL0mask = buffer[2];
+  mL1GammaHighThreshold = buffer[3];
+  mShortPayloadRate = buffer[4];
+  mL0bits = buffer[5];
+  mL1JetHighThreshold = buffer[6];
+  mL1GammaLowThreshold = buffer[9];
+  mL1JetLowThreshold = buffer[12];
+  mCFGword13 = buffer[13];
+  mRegionEnable = buffer[14];
+  mFrameReceived = buffer[15];
+  mCFGword16 = buffer[16];
+
+  // decode L1Jet High/Low indices
+  int offset = getCFGWords();
+  decodeL1JetPatchIndices(&buffer[offset]);
+
+  // decode L1Gamma High/Low indices
+  offset += (2 * getL1JetIndexWords() + getL0indexWords());
+  decodeL1GammaPatchIndices(&buffer[offset]);
+
+  if (!isFullPayload()) {
+    return;
+  }
+
+  // decode FastOR data
+  offset += (2 * getL1GammaIndexWords());
+  decodeFastOrADC(&buffer[offset]);
+
+  //  std::cout << "lastWord = 0x" << std::hex << buffer[offset + getRawWords()] << std::endl;
+  return;
+}
+
+void StuDecoder::decodeL1JetPatchIndices(const uint32_t* buffer)
+{
+  int offset = getL1JetIndexWords(); // offset for Jet Low threshold
+
+  if (mDebug >= 2) {
+    for (int i = 0; i < offset; i++) {
+      std::cout << std::dec << i << ")  0x" << std::hex << buffer[i] << "   <- Jet-high-word\n";
+    }
+    for (int i = 0; i < offset; i++) {
+      std::cout << std::dec << i << ")  0x" << std::hex << buffer[offset + i] << "   <- Jet-low-word\n";
+    }
+  }
+
+  int nSubregionEta = getSubregionsEta();
+  int nSubregionPhi = getSubregionsPhi();
+  bool isFired = false;
+
+  int jetSize = 2 + getParchSize(); // 2 for 2x2-patch, 4 for 4x4=patch
+
+  for (Int_t ix = 0; ix < nSubregionEta - (jetSize - 1); ix++) {
+    uint32_t row_JetHigh = buffer[ix];
+    uint32_t row_JetLow = buffer[ix + offset];
+    for (Int_t iy = 0; iy < nSubregionPhi - (jetSize - 1); iy++) {
+      isFired = (mSTU == STUtype_t::ESTU) ? (row_JetHigh & (1 << (nSubregionPhi - jetSize - iy))) : (row_JetHigh & (1 << iy));
+      if (isFired) {
+        mL1JetHighPatchIndex.push_back(((ix << 8) & 0xFF00) | (iy & 0xFF));
+      }
+      isFired = (mSTU == STUtype_t::ESTU) ? (row_JetLow & (1 << (nSubregionPhi - jetSize - iy))) : (row_JetLow & (1 << iy));
+      if (isFired) {
+        mL1JetLowPatchIndex.push_back(((ix << 8) & 0xFF00) | (iy & 0xFF));
+      }
+    }
+  }
+
+  return;
+}
+
+void StuDecoder::decodeL1GammaPatchIndices(const uint32_t* buffer)
+{
+  const int offset = getL1GammaIndexWords(); // offset for Gamma Low threshold
+
+  if (mDebug >= 2) {
+    for (int i = 0; i < offset; i++) {
+      std::cout << std::dec << i << ")  0x" << std::hex << buffer[i] << "   <- Gamma-high-word\n";
+    }
+    for (int i = 0; i < offset; i++) {
+      std::cout << std::dec << i << ")  0x" << std::hex << buffer[offset + i] << "   <- Gamma-low-word\n";
+    }
+  }
+
+  const int nTRU = getNumberOfTRUs();
+  const int nTRU2 = getNumberOfTRUs() / 2;
+
+  unsigned short gammaHigh[nTRU][TRUparam::NchannelsOverPhi];
+  unsigned short gammaLow[nTRU][TRUparam::NchannelsOverPhi];
+
+  for (Int_t iphi = 0; iphi < TRUparam::NchannelsOverPhi / 2; iphi++) {
+    for (Int_t itru = 0; itru < nTRU2; itru++) {
+      gammaHigh[2 * itru][2 * iphi] = (buffer[iphi * nTRU2 + itru] >> 0 & 0xFF);
+      gammaHigh[2 * itru][2 * iphi + 1] = (buffer[iphi * nTRU2 + itru] >> 8 & 0xFF);
+      gammaHigh[2 * itru + 1][2 * iphi] = (buffer[iphi * nTRU2 + itru] >> 16 & 0xFF);
+      gammaHigh[2 * itru + 1][2 * iphi + 1] = (buffer[iphi * nTRU2 + itru] >> 24 & 0xFF);
+
+      gammaLow[2 * itru][2 * iphi] = (buffer[offset + iphi * nTRU2 + itru] >> 0 & 0xFF);
+      gammaLow[2 * itru][2 * iphi + 1] = (buffer[offset + iphi * nTRU2 + itru] >> 8 & 0xFF);
+      gammaLow[2 * itru + 1][2 * iphi] = (buffer[offset + iphi * nTRU2 + itru] >> 16 & 0xFF);
+      gammaLow[2 * itru + 1][2 * iphi + 1] = (buffer[offset + iphi * nTRU2 + itru] >> 24 & 0xFF);
+    }
+  }
+
+  for (Int_t iphi = 0; iphi < TRUparam::NchannelsOverPhi; iphi++) {
+    for (Int_t ieta = 0; ieta < TRUparam::NchannelsOverEta; ieta++) {
+      // loop over TRUs of Full or 2/3-size SMs
+      for (Int_t itru = 0; itru < (nTRU - 2); itru++) {
+        if ((gammaHigh[itru][iphi] >> ieta) & 0x1) {
+          mL1GammaHighPatchIndex.push_back(((iphi << 10) & 0x7C00) | ((ieta << 5) & 0x03E0) | ((itru << 0) & 0x001F));
+        }
+        if ((gammaLow[itru][iphi] >> ieta) & 0x1) {
+          mL1GammaLowPatchIndex.push_back(((iphi << 10) & 0x7C00) | ((ieta << 5) & 0x03E0) | ((itru << 0) & 0x001F));
+        }
+      }
+      // loop over TRUs of 1/3-size SMs
+      for (Int_t itru = (nTRU - 2); itru < nTRU; itru++) {
+        short iphi_tmp = (iphi % 2 + 2 * (int)(iphi / 6));
+        short ieta_tmp = (ieta + 8 * ((int)(iphi / 2) % 3));
+        if ((gammaHigh[itru][iphi] >> ieta) & 0x1) {
+          mL1GammaHighPatchIndex.push_back(((iphi_tmp << 10) & 0x7C00) | ((ieta_tmp << 5) & 0x03E0) | ((itru << 0) & 0x001F));
+        }
+        if ((gammaLow[itru][iphi] >> ieta) & 0x1) {
+          mL1GammaLowPatchIndex.push_back(((iphi_tmp << 10) & 0x7C00) | ((ieta_tmp << 5) & 0x03E0) | ((itru << 0) & 0x001F));
+        }
+      }
+    }
+  }
+
+  return;
+}
+
+void StuDecoder::decodeFastOrADC(const uint32_t* buffer)
+{
+  for (int i = 0; i < getRawWords(); i++) {
+    if (mDebug >= 2) {
+      std::cout << std::dec << i << ")  0x" << std::hex << buffer[i] << "   <- FasrOR-word\n";
+    }
+    mFastOrADC.push_back(int16_t(buffer[i] & 0xFFFF));
+    mFastOrADC.push_back(int16_t(buffer[i] >> 16) & 0xFFFF);
+  }
+
+  return;
+}
+
+void StuDecoder::dumpSTUcfg() const
+{
+
+  std::cout << "L1GammaHighThreshold:" << std::dec << getL1GammaHighThreshold() << std::endl;
+  std::cout << "L1JetHighThreshold  :" << std::dec << getL1JetHighThreshold() << std::endl;
+  std::cout << "L1GammaLowThreshold :" << std::dec << getL1GammaLowThreshold() << std::endl;
+  std::cout << "L1JetLowThreshold   :" << std::dec << getL1JetLowThreshold() << std::endl;
+  std::cout << "Rho                 :" << std::dec << getRho() << std::endl;
+  std::cout << "FrameReceivedSTU    :" << std::dec << getFrameReceivedSTU() << std::endl;
+  std::cout << "RegionEnable        :" << std::hex << "0x" << getRegionEnable() << std::endl;
+  std::cout << "FrameReceived       :" << std::dec << getFrameReceived() << std::endl;
+  std::cout << "ParchSize           :" << std::dec << getParchSize() << std::endl;
+  std::cout << "FWversion           :" << std::hex << "0x" << getFWversion() << std::endl;
+
+  std::cout << "\n";
+}
diff --git a/Detectors/EMCAL/reconstruction/src/TRUDataHandler.cxx b/Detectors/EMCAL/reconstruction/src/TRUDataHandler.cxx
new file mode 100644
index 0000000000000..d6d9bd3535d5c
--- /dev/null
+++ b/Detectors/EMCAL/reconstruction/src/TRUDataHandler.cxx
@@ -0,0 +1,67 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include 
+#include "EMCALReconstruction/TRUDataHandler.h"
+
+using namespace o2::emcal;
+
+TRUDataHandler::TRUDataHandler()
+{
+  reset();
+}
+
+void TRUDataHandler::reset()
+{
+  mTRUIndex = -1;
+  mL0Fired = false;
+  mL0Time = -1;
+  std::fill(mPatchTimes.begin(), mPatchTimes.end(), UCHAR_MAX);
+}
+
+void TRUDataHandler::printStream(std::ostream& stream) const
+{
+  std::string patchstring;
+  for (auto index = 0; index < mPatchTimes.size(); index++) {
+    if (hasPatch(index)) {
+      if (patchstring.length()) {
+        patchstring += ", ";
+      }
+      patchstring += std::to_string(index);
+    }
+  }
+  if (!patchstring.length()) {
+    patchstring = "-";
+  }
+  stream << "TRU: " << static_cast(mTRUIndex) << ", time " << static_cast(mL0Time) << ", fired: " << (mL0Fired ? "yes" : "no") << ", patches: " << patchstring;
+}
+
+TRUDataHandler::PatchIndexException::PatchIndexException(int8_t index) : mIndex(index), mMessage()
+{
+  mMessage = "Invalid patch index " + std::to_string(index);
+}
+
+void TRUDataHandler::PatchIndexException::printStream(std::ostream& stream) const
+{
+  stream << what();
+}
+
+std::ostream& o2::emcal::operator<<(std::ostream& stream, const TRUDataHandler& data)
+{
+  data.printStream(stream);
+  return stream;
+}
+
+std::ostream& o2::emcal::operator<<(std::ostream& stream, const TRUDataHandler::PatchIndexException& error)
+{
+  error.printStream(stream);
+  return stream;
+}
\ No newline at end of file
diff --git a/Detectors/EMCAL/reconstruction/test/testAltroDecoderError.cxx b/Detectors/EMCAL/reconstruction/test/testAltroDecoderError.cxx
new file mode 100644
index 0000000000000..ad68cf171e6f8
--- /dev/null
+++ b/Detectors/EMCAL/reconstruction/test/testAltroDecoderError.cxx
@@ -0,0 +1,79 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#define BOOST_TEST_MODULETest EMCAL Reconstruction
+#define BOOST_TEST_MAIN
+#define BOOST_TEST_DYN_LINK
+#include 
+#include 
+#include 
+
+namespace o2
+{
+namespace emcal
+{
+
+void testThrow(AltroDecoderError::ErrorType_t errortype)
+{
+  throw AltroDecoderError(errortype, AltroDecoderError::getErrorTypeDescription(errortype));
+}
+
+BOOST_AUTO_TEST_CASE(AltroDecoderError_test)
+{
+  BOOST_CHECK_EQUAL(AltroDecoderError::getNumberOfErrorTypes(), 8);
+  std::array errornames = {{"RCUTrailerError",
+                                            "RCUTrailerVersionError",
+                                            "RCUTrailerSizeError",
+                                            "BunchHeaderError",
+                                            "BunchLengthError",
+                                            "ALTROPayloadError",
+                                            "ALTROMappingError",
+                                            "ChannelError"}},
+                             errortitles = {{"RCU Trailer",
+                                             "RCU Version",
+                                             "RCU Trailer Size",
+                                             "ALTRO Bunch Header",
+                                             "ALTRO Bunch Length",
+                                             "ALTRO Payload",
+                                             "ALTRO Mapping",
+                                             "Channel"}},
+                             errordescriptions = {{"RCU trailer decoding error",
+                                                   "Inconsistent RCU trailer version",
+                                                   "Invalid RCU trailer size",
+                                                   "Inconsistent bunch header",
+                                                   "Bunch length exceeding payload size",
+                                                   "Payload could not be decoded",
+                                                   "Invalid hardware address in ALTRO mapping",
+                                                   "Channels not initizalized"}};
+  std::array errortypes = {{AltroDecoderError::ErrorType_t::RCU_TRAILER_ERROR,
+                                                               AltroDecoderError::ErrorType_t::RCU_VERSION_ERROR,
+                                                               AltroDecoderError::ErrorType_t::RCU_TRAILER_SIZE_ERROR,
+                                                               AltroDecoderError::ErrorType_t::ALTRO_BUNCH_HEADER_ERROR,
+                                                               AltroDecoderError::ErrorType_t::ALTRO_BUNCH_LENGTH_ERROR,
+                                                               AltroDecoderError::ErrorType_t::ALTRO_PAYLOAD_ERROR,
+                                                               AltroDecoderError::ErrorType_t::ALTRO_MAPPING_ERROR,
+                                                               AltroDecoderError::ErrorType_t::CHANNEL_ERROR}};
+  for (int errortype = 0; errortype < AltroDecoderError::getNumberOfErrorTypes(); errortype++) {
+    BOOST_CHECK_EQUAL(AltroDecoderError::errorTypeToInt(errortypes[errortype]), errortype);
+    BOOST_CHECK_EQUAL(AltroDecoderError::intToErrorType(errortype), errortypes[errortype]);
+    BOOST_CHECK_EQUAL(std::string(AltroDecoderError::getErrorTypeName(errortype)), errornames[errortype]);
+    BOOST_CHECK_EQUAL(std::string(AltroDecoderError::getErrorTypeName(errortypes[errortype])), errornames[errortype]);
+    BOOST_CHECK_EQUAL(std::string(AltroDecoderError::getErrorTypeTitle(errortype)), errortitles[errortype]);
+    BOOST_CHECK_EQUAL(std::string(AltroDecoderError::getErrorTypeTitle(errortypes[errortype])), errortitles[errortype]);
+    BOOST_CHECK_EQUAL(std::string(AltroDecoderError::getErrorTypeDescription(errortype)), errordescriptions[errortype]);
+    BOOST_CHECK_EQUAL(std::string(AltroDecoderError::getErrorTypeDescription(errortypes[errortype])), errordescriptions[errortype]);
+    auto expecttype = errortypes[errortype];
+    BOOST_CHECK_EXCEPTION(testThrow(errortypes[errortype]), AltroDecoderError, [expecttype](const AltroDecoderError& ex) { return ex.getErrorType() == expecttype; });
+  }
+}
+
+} // namespace emcal
+} // namespace o2
\ No newline at end of file
diff --git a/Detectors/EMCAL/reconstruction/test/testCaloRawFitterError.cxx b/Detectors/EMCAL/reconstruction/test/testCaloRawFitterError.cxx
new file mode 100644
index 0000000000000..1094f419e92ff
--- /dev/null
+++ b/Detectors/EMCAL/reconstruction/test/testCaloRawFitterError.cxx
@@ -0,0 +1,60 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#define BOOST_TEST_MODULE Test EMCAL Reconstruction
+#define BOOST_TEST_MAIN
+#define BOOST_TEST_DYN_LINK
+#include 
+#include 
+#include 
+
+namespace o2
+{
+namespace emcal
+{
+
+BOOST_AUTO_TEST_CASE(CaloRawFitterError_test)
+{
+  BOOST_CHECK_EQUAL(CaloRawFitter::getNumberOfErrorTypes(), 5);
+  std::array errornames = {{"SampleUninitalized",
+                                            "NoConvergence",
+                                            "Chi2Error",
+                                            "BunchRejected",
+                                            "LowSignal"}},
+                             errortitles = {{"sample uninitalized",
+                                             "No convergence",
+                                             "Chi2 error",
+                                             "Bunch rejected",
+                                             "Low signal"}},
+                             errordescriptions = {{"Sample for fit not initialzied or bunch length is 0",
+                                                   "Fit of the raw bunch was not successful",
+                                                   "Chi2 of the fit could not be determined",
+                                                   "Calo bunch could not be selected",
+                                                   "No ADC value above threshold found"}};
+  std::array errortypes = {{CaloRawFitter::RawFitterError_t::SAMPLE_UNINITIALIZED,
+                                                                CaloRawFitter::RawFitterError_t::FIT_ERROR,
+                                                                CaloRawFitter::RawFitterError_t::CHI2_ERROR,
+                                                                CaloRawFitter::RawFitterError_t::BUNCH_NOT_OK,
+                                                                CaloRawFitter::RawFitterError_t::LOW_SIGNAL}};
+  for (int errortype = 0; errortype < CaloRawFitter::getNumberOfErrorTypes(); errortype++) {
+    BOOST_CHECK_EQUAL(CaloRawFitter::getErrorNumber(errortypes[errortype]), errortype);
+    BOOST_CHECK_EQUAL(CaloRawFitter::intToErrorType(errortype), errortypes[errortype]);
+    BOOST_CHECK_EQUAL(std::string(CaloRawFitter::getErrorTypeName(errortype)), errornames[errortype]);
+    BOOST_CHECK_EQUAL(std::string(CaloRawFitter::getErrorTypeName(errortypes[errortype])), errornames[errortype]);
+    BOOST_CHECK_EQUAL(std::string(CaloRawFitter::getErrorTypeTitle(errortype)), errortitles[errortype]);
+    BOOST_CHECK_EQUAL(std::string(CaloRawFitter::getErrorTypeTitle(errortypes[errortype])), errortitles[errortype]);
+    BOOST_CHECK_EQUAL(std::string(CaloRawFitter::getErrorTypeDescription(errortype)), errordescriptions[errortype]);
+    BOOST_CHECK_EQUAL(std::string(CaloRawFitter::getErrorTypeDescription(errortypes[errortype])), errordescriptions[errortype]);
+  }
+}
+
+} // namespace emcal
+} // namespace o2
\ No newline at end of file
diff --git a/Detectors/EMCAL/reconstruction/test/testFastORTimeSeries.cxx b/Detectors/EMCAL/reconstruction/test/testFastORTimeSeries.cxx
new file mode 100644
index 0000000000000..4144273e1eb31
--- /dev/null
+++ b/Detectors/EMCAL/reconstruction/test/testFastORTimeSeries.cxx
@@ -0,0 +1,175 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#define BOOST_TEST_MODULE Test EMCAL Reconstruction
+#define BOOST_TEST_MAIN
+#define BOOST_TEST_DYN_LINK
+#include 
+#include 
+#include 
+#include 
+#include 
+#include "EMCALReconstruction/TRUDecodingErrors.h"
+
+namespace o2
+{
+
+namespace emcal
+{
+
+void printBunch(const gsl::span adcs)
+{
+  bool first = true;
+  for (auto& adc : adcs) {
+    if (!first) {
+      std::cout << ", ";
+    } else {
+      first = false;
+    }
+    std::cout << adc;
+  }
+  std::cout << " (size " << adcs.size() << ")" << std::endl;
+}
+
+std::vector getReversed(const std::vector& original)
+{
+  std::vector reversed(13);
+  for (std::size_t sample = 0; sample < 13; sample++) {
+    reversed[12 - sample] = original[sample];
+  }
+  return reversed;
+}
+
+std::tuple, std::vector> generatePulseTimeReversed()
+{
+  std::vector pulse(13);
+  std::fill(pulse.begin(), pulse.end(), 0);
+  // calculate forward pulse
+  auto peak_signal = static_cast(gRandom->Uniform(0, 1024));
+  pulse[4] = peak_signal;
+  auto last = peak_signal;
+  for (std::size_t sample = 5; sample < 13; sample++) {
+    if (last == 0) {
+      break;
+    }
+    auto current = static_cast(gRandom->Uniform(0, last));
+    pulse[sample] = current;
+    last = current;
+  }
+  last = peak_signal;
+  for (std::size_t sample = 3; sample > 0; sample--) {
+    if (last == 0) {
+      break;
+    }
+    auto current = static_cast(gRandom->Uniform(0, last));
+    pulse[sample] = current;
+    last = current;
+  }
+  // find start time
+  uint8_t starttime = 12;
+  for (std::size_t currenttime = 12; currenttime > 0; currenttime--) {
+    starttime = currenttime;
+    if (pulse[currenttime]) {
+      break;
+    }
+  }
+  // time-reverse pulse
+  auto reversed = getReversed(pulse);
+  // zero-suppress time series
+  std::vector zerosuppressed;
+  bool bunchstart = false;
+  for (std::size_t sample = 0; sample < 13; sample++) {
+    if (reversed[sample] == 0) {
+      if (!bunchstart) {
+        continue;
+      }
+      break;
+    }
+    bunchstart = true;
+    zerosuppressed.push_back(reversed[sample]);
+  }
+  return std::make_tuple(starttime, zerosuppressed, pulse);
+}
+
+uint16_t calculateTimesum(const std::vector samplesOrdered, uint8_t l0time)
+{
+  uint16_t timesum = 0;
+  uint8_t starttime = l0time - 4;
+  for (uint8_t sample = starttime; sample < starttime + 4; sample++) {
+    timesum += samplesOrdered[sample];
+  }
+  return timesum;
+}
+
+std::vector generateSmallBunch(uint8_t bunchlength)
+{
+  std::vector bunch(bunchlength);
+  auto peak_signal = static_cast(gRandom->Uniform(0, 1024));
+  bunch[bunchlength - 2] = peak_signal;
+  bunch[bunchlength - 1] = static_cast(gRandom->Uniform(0, peak_signal));
+  auto last = peak_signal;
+  for (int sample = bunchlength - 3; sample >= 0; sample--) {
+    auto current = static_cast(gRandom->Uniform(0, last));
+    bunch[sample] = current;
+    last = current;
+  }
+  return bunch;
+}
+
+void add_bunch_to_buffer(std::vector& buffer, const std::vector& bunch, uint8_t starttime)
+{
+  for (int sample = 0; sample < bunch.size(); sample++) {
+    buffer[starttime - sample] = bunch[bunch.size() - 1 - sample];
+  }
+}
+
+BOOST_AUTO_TEST_CASE(FastORTimeSeries_test)
+{
+  // test fill and integral
+  for (int itest = 0; itest < 500; itest++) {
+    auto [starttime, zerosuppressed, reference] = generatePulseTimeReversed();
+    FastORTimeSeries testcase(13, zerosuppressed, starttime);
+    auto adcs = testcase.getADCs();
+    BOOST_CHECK_EQUAL_COLLECTIONS(adcs.begin(), adcs.end(), reference.begin(), reference.end());
+    BOOST_CHECK_EQUAL(testcase.calculateL1TimeSum(8), calculateTimesum(reference, 8));
+  }
+
+  // test case where a normal FEC channel is identified as TRU channel. FEC channel can have lenght of 15 and would therefore cause an overflow in the FEC channel (max lenght 14)
+  auto starttime = 14;
+  auto bunch = generateSmallBunch(14);
+  BOOST_CHECK_EXCEPTION(FastORTimeSeries(14, bunch, starttime), FastOrStartTimeInvalidException, [starttime](const FastOrStartTimeInvalidException& e) { return e.getStartTime() == starttime; });
+
+  return;
+
+  // test adding 2 bunches
+  for (int itest = 0; itest < 500; itest++) {
+    auto length_bunch1 = static_cast(gRandom->Uniform(3, 5)),
+         length_bunch2 = static_cast(gRandom->Uniform(3, 5));
+    auto sumbunchlength = length_bunch1 + length_bunch2;
+    auto offset_bunch1 = static_cast(gRandom->Uniform(0, 13 - sumbunchlength)),
+         offset_bunch2 = static_cast(gRandom->Uniform(0, 13 - sumbunchlength - offset_bunch1));
+    auto bunch1 = generateSmallBunch(length_bunch1),
+         bunch2 = generateSmallBunch(length_bunch2);
+    auto starttime_bunch1 = offset_bunch1 + length_bunch1,
+         starttime_bunch2 = starttime_bunch1 + offset_bunch2 + length_bunch2;
+    std::vector buffer_reversed{13};
+    add_bunch_to_buffer(buffer_reversed, bunch2, starttime_bunch2);
+    add_bunch_to_buffer(buffer_reversed, bunch1, starttime_bunch1);
+    FastORTimeSeries testcase(13, bunch2, starttime_bunch2);
+    testcase.setTimeSamples(bunch1, starttime_bunch1);
+    auto adcs_timeordered = getReversed(buffer_reversed);
+    auto adcs_timeseries_reversed = testcase.getADCs();
+    BOOST_CHECK_EQUAL_COLLECTIONS(adcs_timeseries_reversed.begin(), adcs_timeseries_reversed.end(), adcs_timeordered.begin(), adcs_timeordered.end());
+  }
+}
+
+} // namespace emcal
+
+} // namespace o2
diff --git a/Detectors/EMCAL/reconstruction/test/testMinorAltroDecodingError.cxx b/Detectors/EMCAL/reconstruction/test/testMinorAltroDecodingError.cxx
new file mode 100644
index 0000000000000..9fc088ebc6a3e
--- /dev/null
+++ b/Detectors/EMCAL/reconstruction/test/testMinorAltroDecodingError.cxx
@@ -0,0 +1,72 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#define BOOST_TEST_MODULE Test EMCAL Reconstruction
+#define BOOST_TEST_MAIN
+#define BOOST_TEST_DYN_LINK
+#include 
+#include 
+#include 
+
+namespace o2
+{
+namespace emcal
+{
+
+BOOST_AUTO_TEST_CASE(MinorAltroDecodingError_test)
+{
+  BOOST_CHECK_EQUAL(MinorAltroDecodingError::getNumberOfErrorTypes(), 8);
+  std::array errornames = {{"ChannelEndPayloadUnexpected",
+                                            "ChannelPayloadExceed",
+                                            "ChannelOrderError",
+                                            "ChannelHeader",
+                                            "BunchHeaderNull",
+                                            "BunchLengthExceed",
+                                            "BunchLengthAllowExceed",
+                                            "BunchStarttimeExceed"}},
+                             errortitles = {{"Channel end unexpected",
+                                             "Channel exceed",
+                                             "FEC order",
+                                             "Channel header invalid",
+                                             "Bunch header null",
+                                             "Bunch length exceed",
+                                             "Bunch length impossible",
+                                             "Bunch starttime exceed"}},
+                             errordescriptions = {{"Unexpected end of payload in altro channel payload!",
+                                                   "Trying to access out-of-bound payload!",
+                                                   "Invalid FEC order",
+                                                   "Invalid channel header",
+                                                   "Bunch header 0 or not configured!",
+                                                   "Bunch length exceeding channel payload size!",
+                                                   "Bunch length exceeding max. possible bunch size!",
+                                                   "Bunch start time outside range!"}};
+  std::array errortypes = {{MinorAltroDecodingError::ErrorType_t::CHANNEL_END_PAYLOAD_UNEXPECT,
+                                                                     MinorAltroDecodingError::ErrorType_t::CHANNEL_PAYLOAD_EXCEED,
+                                                                     MinorAltroDecodingError::ErrorType_t::CHANNEL_ORDER,
+                                                                     MinorAltroDecodingError::ErrorType_t::CHANNEL_HEADER,
+                                                                     MinorAltroDecodingError::ErrorType_t::BUNCH_HEADER_NULL,
+                                                                     MinorAltroDecodingError::ErrorType_t::BUNCH_LENGTH_EXCEED,
+                                                                     MinorAltroDecodingError::ErrorType_t::BUNCH_LENGTH_ALLOW_EXCEED,
+                                                                     MinorAltroDecodingError::ErrorType_t::BUNCH_STARTTIME}};
+  for (int errortype = 0; errortype < MinorAltroDecodingError::getNumberOfErrorTypes(); errortype++) {
+    BOOST_CHECK_EQUAL(MinorAltroDecodingError::errorTypeToInt(errortypes[errortype]), errortype);
+    BOOST_CHECK_EQUAL(MinorAltroDecodingError::intToErrorType(errortype), errortypes[errortype]);
+    BOOST_CHECK_EQUAL(std::string(MinorAltroDecodingError::getErrorTypeName(errortype)), errornames[errortype]);
+    BOOST_CHECK_EQUAL(std::string(MinorAltroDecodingError::getErrorTypeName(errortypes[errortype])), errornames[errortype]);
+    BOOST_CHECK_EQUAL(std::string(MinorAltroDecodingError::getErrorTypeTitle(errortype)), errortitles[errortype]);
+    BOOST_CHECK_EQUAL(std::string(MinorAltroDecodingError::getErrorTypeTitle(errortypes[errortype])), errortitles[errortype]);
+    BOOST_CHECK_EQUAL(std::string(MinorAltroDecodingError::getErrorTypeDescription(errortype)), errordescriptions[errortype]);
+    BOOST_CHECK_EQUAL(std::string(MinorAltroDecodingError::getErrorTypeDescription(errortypes[errortype])), errordescriptions[errortype]);
+  }
+}
+
+} // namespace emcal
+} // namespace o2
\ No newline at end of file
diff --git a/Detectors/EMCAL/reconstruction/test/testRawDecodingError.cxx b/Detectors/EMCAL/reconstruction/test/testRawDecodingError.cxx
new file mode 100644
index 0000000000000..f701531dc9545
--- /dev/null
+++ b/Detectors/EMCAL/reconstruction/test/testRawDecodingError.cxx
@@ -0,0 +1,87 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#define BOOST_TEST_MODULE Test EMCAL Reconstruction
+#define BOOST_TEST_MAIN
+#define BOOST_TEST_DYN_LINK
+#include 
+#include 
+#include 
+
+namespace o2
+{
+namespace emcal
+{
+
+void testThrow(RawDecodingError::ErrorType_t errtype, unsigned int feeID)
+{
+  throw RawDecodingError(errtype, feeID);
+}
+
+BOOST_AUTO_TEST_CASE(RawDecodingError_test)
+{
+  BOOST_CHECK_EQUAL(RawDecodingError::getNumberOfErrorTypes(), 8);
+  std::array errornames = {{"PageNotFound",
+                                            "HeaderDecoding",
+                                            "PayloadDecoding",
+                                            "HeaderCorruption",
+                                            "PageStartInvalid",
+                                            "PayloadCorruption",
+                                            "TrailerDecoding",
+                                            "TrailerIncomplete"}},
+                             errortitles = {{"Page not found",
+                                             "Header decoding",
+                                             "Payload decoding",
+                                             "Header corruption",
+                                             "Page start invalid",
+                                             "Payload corruption",
+                                             "Trailer decoding",
+                                             "Trailer incomplete"}},
+                             errordescriptions = {{"Page with requested index not found",
+                                                   "RDH of page cannot be decoded",
+                                                   "Payload of page cannot be decoded",
+                                                   "Access to header not belonging to requested superpage",
+                                                   "Page decoding starting outside payload size",
+                                                   "Access to payload not belonging to requested superpage",
+                                                   "Inconsistent trailer in memory",
+                                                   "Incomplete trailer"}};
+  std::array errortypes = {{RawDecodingError::ErrorType_t::PAGE_NOTFOUND,
+                                                              RawDecodingError::ErrorType_t::HEADER_DECODING,
+                                                              RawDecodingError::ErrorType_t::PAYLOAD_DECODING,
+                                                              RawDecodingError::ErrorType_t::HEADER_INVALID,
+                                                              RawDecodingError::ErrorType_t::PAGE_START_INVALID,
+                                                              RawDecodingError::ErrorType_t::PAYLOAD_INVALID,
+                                                              RawDecodingError::ErrorType_t::TRAILER_DECODING,
+                                                              RawDecodingError::ErrorType_t::TRAILER_INCOMPLETE}};
+  for (int errortype = 0; errortype < RawDecodingError::getNumberOfErrorTypes(); errortype++) {
+    BOOST_CHECK_EQUAL(RawDecodingError::ErrorTypeToInt(errortypes[errortype]), errortype);
+    BOOST_CHECK_EQUAL(RawDecodingError::intToErrorType(errortype), errortypes[errortype]);
+    BOOST_CHECK_EQUAL(std::string(RawDecodingError::getErrorCodeNames(errortype)), errornames[errortype]);
+    BOOST_CHECK_EQUAL(std::string(RawDecodingError::getErrorCodeNames(errortypes[errortype])), errornames[errortype]);
+    BOOST_CHECK_EQUAL(std::string(RawDecodingError::getErrorCodeTitles(errortype)), errortitles[errortype]);
+    BOOST_CHECK_EQUAL(std::string(RawDecodingError::getErrorCodeTitles(errortypes[errortype])), errortitles[errortype]);
+    BOOST_CHECK_EQUAL(std::string(RawDecodingError::getErrorCodeDescription(errortype)), errordescriptions[errortype]);
+    BOOST_CHECK_EQUAL(std::string(RawDecodingError::getErrorCodeDescription(errortypes[errortype])), errordescriptions[errortype]);
+    for (unsigned int errtype = 0; errtype < 7; errtype++) {
+      auto errtypeval = RawDecodingError::intToErrorType(errtype);
+      for (unsigned int feeID = 0; feeID < 40; feeID++) {
+        auto checker = [errtypeval, feeID](const RawDecodingError& test) {
+          return test.getErrorType() == errtypeval && test.getFECID() == feeID;
+        };
+        BOOST_CHECK_EXCEPTION(testThrow(errtypeval, feeID),
+                              RawDecodingError, checker);
+      }
+    }
+  }
+}
+
+} // namespace emcal
+} // namespace o2
\ No newline at end of file
diff --git a/Detectors/EMCAL/reconstruction/test/testRecoContainer.cxx b/Detectors/EMCAL/reconstruction/test/testRecoContainer.cxx
new file mode 100644
index 0000000000000..182c7ceacb654
--- /dev/null
+++ b/Detectors/EMCAL/reconstruction/test/testRecoContainer.cxx
@@ -0,0 +1,173 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#define BOOST_TEST_MODULE Test EMCAL Reconstruction
+#define BOOST_TEST_MAIN
+#define BOOST_TEST_DYN_LINK
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+namespace o2
+{
+namespace emcal
+{
+
+BOOST_AUTO_TEST_CASE(RecoContainer_test)
+{
+  RecoContainer testcontainer;
+
+  std::map truthNumberCells, truthNumberLEDMONs;
+
+  // test 1: Check appending cells to existing container
+  o2::InteractionRecord testIR(1023, 384128);
+  auto& currentEvent = testcontainer.getEventContainer(testIR);
+
+  std::vector towers1 = {12, 382, 922, 1911};
+  std::vector energies1 = {0.2, 10., 1.1, 0.4};
+  std::vector times1 = {1, 29, 0, 2};
+
+  for (int icell = 0; icell < 4; icell++) {
+    currentEvent.setCell(towers1[icell], energies1[icell], times1[icell], ChannelType_t::HIGH_GAIN, 9238, 1, true);
+  }
+
+  BOOST_CHECK_EQUAL(currentEvent.getCells().size(), 4);
+
+  std::vector towers2 = {57, 292, 4592, 11922};
+  std::vector energies2 = {0.2, 10., 1.1, 0.4};
+  std::vector times2 = {10, 3, 1, 5};
+  auto& newcurrent = testcontainer.getEventContainer(testIR);
+  for (int icell = 0; icell < 4; icell++) {
+    newcurrent.setCell(towers2[icell], energies2[icell], times2[icell], ChannelType_t::HIGH_GAIN, 9238, 2, true);
+  }
+
+  BOOST_CHECK_EQUAL(newcurrent.getCells().size(), 8);
+  BOOST_CHECK_EQUAL(testcontainer.getNumberOfEvents(), 1);
+  truthNumberCells[testIR] = 8;
+  truthNumberLEDMONs[testIR] = 0;
+
+  // test 2: Adding new event to container
+  o2::InteractionRecord secondIR(2021, 384130);
+  auto& secondevent = testcontainer.getEventContainer(secondIR);
+  BOOST_CHECK_EQUAL(testcontainer.getNumberOfEvents(), 2);
+  BOOST_CHECK_EQUAL(secondevent.getCells().size(), 0);
+
+  // test 3: Merge HG and LG cells
+  secondevent.setCell(1023, 1.41023, 0.2, ChannelType_t::HIGH_GAIN, 2902, 1, true);
+  secondevent.setCell(1023, 1.4013, 0.91, ChannelType_t::LOW_GAIN, 2902, 1, true);
+  secondevent.setCell(2821, 16.2, 6, ChannelType_t::HIGH_GAIN, 1293, 3, true);
+  secondevent.setCell(2821, 129., 10, ChannelType_t::LOW_GAIN, 1293, 3, true);
+  BOOST_CHECK_EQUAL(secondevent.getCells().size(), 2);
+  int nCellHG = 0, nCellLG = 0;
+  for (const auto& cell : secondevent.getCells()) {
+    switch (cell.mCellData.getType()) {
+      case ChannelType_t::HIGH_GAIN:
+        nCellHG++;
+        break;
+      case ChannelType_t::LOW_GAIN:
+        nCellLG++;
+        break;
+
+      default:
+        break;
+    }
+  }
+  BOOST_CHECK_EQUAL(nCellHG, 1);
+  BOOST_CHECK_EQUAL(nCellLG, 1);
+
+  // test 4: test LGnoHG and HGSaturated
+  secondevent.setCell(12034, 0.3, 10, ChannelType_t::LOW_GAIN, 3942, 10, true);
+  secondevent.setCell(12392, 18., 94, ChannelType_t::HIGH_GAIN, 1209, 20, true);
+  int nLGnoHG = 0, nHGOutOfRange = 0;
+  for (const auto& cell : secondevent.getCells()) {
+    if (cell.mIsLGnoHG) {
+      nLGnoHG++;
+    }
+    if (cell.mHGOutOfRange) {
+      nHGOutOfRange++;
+    }
+  }
+  BOOST_CHECK_EQUAL(nLGnoHG, 1);
+  BOOST_CHECK_EQUAL(nHGOutOfRange, 1);
+
+  // test 5: test LEDMon Cells
+  std::vector ledmonTowers = {293, 3842, 1820};
+  std::vector ledmonEnergies = {5.4, 5.2, 6.2};
+  std::vector ledmonTimes = {230, 303, 280};
+  for (int iledmon = 0; iledmon < 3; iledmon++) {
+    secondevent.setLEDMONCell(ledmonTowers[iledmon], ledmonEnergies[iledmon], ledmonTimes[iledmon], ChannelType_t::HIGH_GAIN, 3302, 20, true);
+  }
+  BOOST_CHECK_EQUAL(secondevent.getCells().size(), 4);
+  BOOST_CHECK_EQUAL(secondevent.getLEDMons().size(), 3);
+  truthNumberCells[secondIR] = 4;
+  truthNumberLEDMONs[secondIR] = 3;
+
+  // test 6: test sorting of collisions
+  std::vector collisions{testIR, secondIR};
+  std::sort(collisions.begin(), collisions.end(), std::less<>());
+  auto sortedCollisions = testcontainer.getOrderedInteractions();
+  BOOST_CHECK_EQUAL_COLLECTIONS(collisions.begin(), collisions.end(), sortedCollisions.begin(), sortedCollisions.end());
+
+  // test 7: read the container
+  RecoContainerReader iterator(testcontainer);
+  std::vector foundInteractions;
+  std::map foundNCells, foundLEDMONs;
+  while (iterator.hasNext()) {
+    auto& event = iterator.nextEvent();
+    foundInteractions.push_back(event.getInteractionRecord());
+    foundNCells[event.getInteractionRecord()] = event.getNumberOfCells();
+    foundLEDMONs[event.getInteractionRecord()] = event.getNumberOfLEDMONs();
+    if (event.getNumberOfCells()) {
+      // test sorting cells
+      event.sortCells(false);
+      int lastcell = -1;
+      for (const auto& cell : event.getCells()) {
+        if (lastcell > -1) {
+          BOOST_CHECK_LT(lastcell, cell.mCellData.getTower());
+        }
+        lastcell = cell.mCellData.getTower();
+      }
+    }
+    if (event.getNumberOfLEDMONs()) {
+      // test sorting LEDMONS
+      event.sortCells(true);
+      int lastLEDMON = -1;
+      for (const auto& ledmon : event.getLEDMons()) {
+        if (lastLEDMON > -1) {
+          BOOST_CHECK_LT(lastLEDMON, ledmon.mCellData.getTower());
+        }
+        lastLEDMON = ledmon.mCellData.getTower();
+      }
+    }
+  }
+  BOOST_CHECK_EQUAL_COLLECTIONS(collisions.begin(), collisions.end(), foundInteractions.begin(), foundInteractions.end());
+  for (auto truthCell = truthNumberCells.begin(), foundCell = foundNCells.begin(); truthCell != truthNumberCells.end(); truthCell++, foundCell++) {
+    BOOST_CHECK_EQUAL(truthCell->first.bc, foundCell->first.bc);
+    BOOST_CHECK_EQUAL(truthCell->first.orbit, foundCell->first.orbit);
+    BOOST_CHECK_EQUAL(truthCell->second, foundCell->second);
+  }
+  for (auto truthLEDMON = truthNumberLEDMONs.begin(), foundLEDMON = foundLEDMONs.begin(); truthLEDMON != truthNumberLEDMONs.end(); truthLEDMON++, foundLEDMON++) {
+    BOOST_CHECK_EQUAL(truthLEDMON->first.bc, foundLEDMON->first.bc);
+    BOOST_CHECK_EQUAL(truthLEDMON->first.orbit, foundLEDMON->first.orbit);
+    BOOST_CHECK_EQUAL(truthLEDMON->second, foundLEDMON->second);
+  }
+
+  // test 8: reset Container
+  testcontainer.reset();
+  BOOST_CHECK_EQUAL(testcontainer.getNumberOfEvents(), 0);
+}
+
+} // namespace emcal
+} // namespace o2
\ No newline at end of file
diff --git a/Detectors/EMCAL/reconstruction/test/testTRUDataHandler.cxx b/Detectors/EMCAL/reconstruction/test/testTRUDataHandler.cxx
new file mode 100644
index 0000000000000..5bab7bf04cc1f
--- /dev/null
+++ b/Detectors/EMCAL/reconstruction/test/testTRUDataHandler.cxx
@@ -0,0 +1,92 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#define BOOST_TEST_MODULE Test EMCAL Reconstruction
+#define BOOST_TEST_MAIN
+#define BOOST_TEST_DYN_LINK
+#include 
+#include 
+#include 
+#include 
+#include 
+
+namespace o2
+{
+namespace emcal
+{
+
+BOOST_AUTO_TEST_CASE(TRUDataHandler_test)
+{
+  o2::emcal::TRUDataHandler testhandler;
+
+  // no patch set
+  BOOST_CHECK_EQUAL(testhandler.hasAnyPatch(), false);
+  for (int ipatch = 0; ipatch < o2::emcal::TriggerMappingV2::PATCHESINTRU; ipatch++) {
+    BOOST_CHECK_EQUAL(testhandler.hasPatch(ipatch), false);
+  }
+
+  // set all patches with L0 time 8
+  for (int ipatch = 0; ipatch < o2::emcal::TriggerMappingV2::PATCHESINTRU; ipatch++) {
+    testhandler.setPatch(ipatch, 8);
+  }
+  BOOST_CHECK_EQUAL(testhandler.hasAnyPatch(), true);
+  for (int ipatch = 0; ipatch < o2::emcal::TriggerMappingV2::PATCHESINTRU; ipatch++) {
+    BOOST_CHECK_EQUAL(testhandler.hasPatch(ipatch), true);
+    BOOST_CHECK_EQUAL(testhandler.getPatchTime(ipatch), 8);
+  }
+
+  // test reset
+  testhandler.reset();
+  BOOST_CHECK_EQUAL(testhandler.hasAnyPatch(), false);
+  for (int ipatch = 0; ipatch < o2::emcal::TriggerMappingV2::PATCHESINTRU; ipatch++) {
+    BOOST_CHECK_EQUAL(testhandler.hasPatch(ipatch), false);
+  }
+
+  // test error handling
+  for (int8_t index = o2::emcal::TriggerMappingV2::PATCHESINTRU; index < INT8_MAX; index++) {
+    BOOST_CHECK_EXCEPTION(testhandler.hasPatch(index), o2::emcal::TRUDataHandler::PatchIndexException, [index](const o2::emcal::TRUDataHandler::PatchIndexException& e) { return e.getIndex() == index; });
+    BOOST_CHECK_EXCEPTION(testhandler.setPatch(index, 8), o2::emcal::TRUDataHandler::PatchIndexException, [index](const o2::emcal::TRUDataHandler::PatchIndexException& e) { return e.getIndex() == index; });
+  }
+
+  for (int iiter = 0; iiter < 100; iiter++) {
+    // For 100 iterations simulate patch index and time
+    std::map patchtimes;
+    int npatches_expect = static_cast(gRandom->Uniform(0, o2::emcal::TriggerMappingV2::PATCHESINTRU));
+    while (patchtimes.size() < npatches_expect) {
+      auto patchindex = static_cast(gRandom->Uniform(0, o2::emcal::TriggerMappingV2::PATCHESINTRU));
+      if (patchtimes.find(patchindex) == patchtimes.end()) {
+        auto patchtime = static_cast(gRandom->Gaus(8, 1));
+        if (patchtime >= 12) {
+          patchtime = 11;
+        }
+        patchtimes[patchindex] = patchtime;
+      }
+    }
+    o2::emcal::TRUDataHandler iterhandler;
+    iterhandler.setFired(npatches_expect > 0);
+    for (auto [patchindex, patchtime] : patchtimes) {
+      iterhandler.setPatch(patchindex, patchtime);
+    }
+
+    BOOST_CHECK_EQUAL(iterhandler.isFired(), npatches_expect > 0);
+    BOOST_CHECK_EQUAL(iterhandler.hasAnyPatch(), npatches_expect > 0);
+    for (auto ipatch = 0; ipatch < o2::emcal::TriggerMappingV2::PATCHESINTRU; ipatch++) {
+      auto hasPatch = patchtimes.find(ipatch) != patchtimes.end();
+      BOOST_CHECK_EQUAL(iterhandler.hasPatch(ipatch), hasPatch);
+      if (hasPatch) {
+        BOOST_CHECK_EQUAL(iterhandler.getPatchTime(ipatch), patchtimes[ipatch]);
+      }
+    }
+  }
+}
+
+} // namespace emcal
+
+} // namespace o2
\ No newline at end of file
diff --git a/Detectors/EMCAL/simulation/CMakeLists.txt b/Detectors/EMCAL/simulation/CMakeLists.txt
index ea593f8180aba..c08bc8c53c69b 100644
--- a/Detectors/EMCAL/simulation/CMakeLists.txt
+++ b/Detectors/EMCAL/simulation/CMakeLists.txt
@@ -10,16 +10,21 @@
 # or submit itself to any jurisdiction.
 
 o2_add_library(EMCALSimulation
-               SOURCES src/Detector.cxx src/Digitizer.cxx src/SDigitizer.cxx
-                       src/DigitsWriteoutBuffer.cxx src/DigitsVectorStream.cxx src/SpaceFrame.cxx src/SimParam.cxx
+               SOURCES src/Detector.cxx src/Digitizer.cxx src/DigitizerTRU.cxx src/SDigitizer.cxx
+                       src/DigitsWriteoutBuffer.cxx src/DigitsWriteoutBufferTRU.cxx src/LZEROElectronics.cxx src/TRUElectronics.cxx src/DigitsVectorStream.cxx src/SpaceFrame.cxx src/SimParam.cxx
                        src/LabeledDigit.cxx src/RawWriter.cxx
-               PUBLIC_LINK_LIBRARIES O2::EMCALBase O2::DetectorsBase O2::SimConfig O2::SimulationDataFormat O2::Headers O2::DetectorsRaw O2::EMCALReconstruction O2::DataFormatsCTP)
+               PUBLIC_LINK_LIBRARIES ROOT::TreePlayer O2::EMCALBase O2::DetectorsBase O2::SimConfig O2::SimulationDataFormat O2::Headers O2::DetectorsRaw O2::EMCALReconstruction O2::EMCALCalib O2::DataFormatsCTP)
 
 o2_target_root_dictionary(EMCALSimulation
                           HEADERS include/EMCALSimulation/Detector.h
                                   include/EMCALSimulation/Digitizer.h
+                                  include/EMCALSimulation/DigitizerTRU.h
                                   include/EMCALSimulation/SDigitizer.h
                                   include/EMCALSimulation/DigitsWriteoutBuffer.h
+                                  include/EMCALSimulation/DigitsWriteoutBufferTRU.h
+                                  include/EMCALSimulation/LZEROElectronics.h
+                                  include/EMCALSimulation/TRUElectronics.h
+                                  include/EMCALSimulation/DigitTimebin.h
                                   include/EMCALSimulation/DigitsVectorStream.h
                                   include/EMCALSimulation/RawWriter.h
                                   include/EMCALSimulation/SpaceFrame.h
diff --git a/Detectors/EMCAL/simulation/include/EMCALSimulation/DigitTimebin.h b/Detectors/EMCAL/simulation/include/EMCALSimulation/DigitTimebin.h
new file mode 100644
index 0000000000000..87903688d8788
--- /dev/null
+++ b/Detectors/EMCAL/simulation/include/EMCALSimulation/DigitTimebin.h
@@ -0,0 +1,59 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file DigitTimebin.h
+/// \brief EMCAL DigitTimebin for the DigitsWriteoutBuffer and DigitsWriteoutBufferTRU
+#ifndef ALICEO2_EMCAL_DIGITTIMEBIN_H
+#define ALICEO2_EMCAL_DIGITTIMEBIN_H
+
+#include "DataFormatsEMCAL/Digit.h"
+#include "EMCALSimulation/LabeledDigit.h"
+#include "CommonDataFormat/InteractionRecord.h"
+
+namespace o2
+{
+
+namespace emcal
+{
+
+template 
+struct DigitTimebinBase;
+
+/// \struct DigitTimebinBase
+/// \brief DigitTimebinBase templated, used for the DigitsWriteoutBuffer and DigitsWriteoutBufferTRU
+/// \ingroup EMCALsimulation
+/// \author Markus Fasel, ORNL
+/// \author Hadi Hassan, ORNL
+/// \author Simone Ragoni, Creighton U.
+/// \date 16/12/2022
+///
+/// \param mRecordMode record mode
+/// \param mEndWindow end window
+/// \param mTriggerColl trigger collision
+/// \param mInterRecord InteractionRecord
+/// \param mDigitMap map of the digits, templated
+template 
+struct DigitTimebinBase {
+  bool mRecordMode = false;
+  bool mEndWindow = false;
+  bool mTriggerColl = false;
+  std::optional mInterRecord;
+  std::shared_ptr>> mDigitMap = std::make_shared>>();
+  ClassDefNV(DigitTimebinBase, 1);
+};
+
+/// \brief DigitTimebin is DigitTimebinBase
+using DigitTimebin = DigitTimebinBase;
+using DigitTimebinTRU = DigitTimebinBase;
+
+} // namespace emcal
+} // namespace o2
+#endif /* ALICEO2_EMCAL_DIGITTIMEBIN_H */
diff --git a/Detectors/EMCAL/simulation/include/EMCALSimulation/Digitizer.h b/Detectors/EMCAL/simulation/include/EMCALSimulation/Digitizer.h
index 88971fba97d72..66f85184c98e6 100644
--- a/Detectors/EMCAL/simulation/include/EMCALSimulation/Digitizer.h
+++ b/Detectors/EMCAL/simulation/include/EMCALSimulation/Digitizer.h
@@ -12,6 +12,7 @@
 #ifndef ALICEO2_EMCAL_FEEDIGITIZER_H
 #define ALICEO2_EMCAL_FEEDIGITIZER_H
 
+#include 
 #include 
 #include 
 #include 
@@ -21,6 +22,7 @@
 #include "TObject.h" // for TObject
 #include "TRandom3.h"
 
+#include "DataFormatsEMCAL/Constants.h"
 #include "DataFormatsEMCAL/Digit.h"
 #include "EMCALBase/Hit.h"
 #include "EMCALSimulation/SimParam.h"
@@ -65,13 +67,13 @@ class Digitizer : public TObject
   /// Steer conversion of hits to digits
   void process(const std::vector& labeledDigit);
 
-  void setEventTime(o2::InteractionTimeRecord record);
+  void setEventTime(o2::InteractionTimeRecord record, bool trigger);
   double getTriggerTime() const { return mDigits.getTriggerTime(); }
   double getEventTime() const { return mDigits.getEventTime(); }
   bool isLive(double t) const { return mDigits.isLive(t); }
   bool isLive() const { return mDigits.isLive(); }
+  bool isCurrentEventTriggered() const { return mDigits.isCurrentEventTriggered(); }
 
-  void setWindowStartTime(int time) { mTimeWindowStart = time; }
   void setDebugStreaming(bool doStreaming) { mEnableDebugStreaming = doStreaming; }
 
   // function returns true if the collision occurs 600ns before the readout window is open
@@ -81,6 +83,7 @@ class Digitizer : public TObject
 
   bool doSmearEnergy() const { return mSmearEnergy; }
   double smearEnergy(double energy);
+  double smearTime(double time, double energy);
   bool doSimulateTimeResponse() const { return mSimulateTimeResponse; }
 
   void sampleSDigit(const Digit& sdigit);
@@ -92,24 +95,32 @@ class Digitizer : public TObject
   const std::vector& getTriggerRecords() const { return mDigits.getTriggerRecords(); }
   const o2::dataformats::MCTruthContainer& getMCLabels() const { return mDigits.getMCLabels(); }
 
+  static constexpr int getTOFSamplingBins() { return EMC_TOF_BINS; }
+
  private:
-  short mEventTimeOffset = 0;          ///< event time difference from trigger time (in number of bins)
-  short mPhase = 0;                    ///< event phase
-  UInt_t mROFrameMin = 0;              ///< lowest RO frame of current digits
-  UInt_t mROFrameMax = 0;              ///< highest RO frame of current digits
-  bool mSmearEnergy = true;            ///< do time and energy smearing
-  bool mSimulateTimeResponse = true;   ///< simulate time response
-  const SimParam* mSimParam = nullptr; ///< SimParam object
-
-  std::vector mTempDigitVector; ///< temporary digit storage
-  // std::unordered_map> mDigits; ///< used to sort digits and labels by tower
+  using TimeSampleContainer = std::array;
+  static constexpr int EMC_PHASES = 4;                                                  ///< Number of phases
+  static constexpr int EMC_TOF_BINS = 1500;                                             ///< Number of bins in TOF sampling of the time response
+  static constexpr double EMC_TOF_MIN = 0;                                              ///< Min TOF
+  static constexpr double EMC_TOF_MAX = 1500.;                                          ///< Max TOF
+  static constexpr double EMC_TOF_BINWITH = (EMC_TOF_MAX - EMC_TOF_MIN) / EMC_TOF_BINS; ///< Number time samples simulated
+  short mEventTimeOffset = 0;                                                           ///< event time difference from trigger time (in number of bins)
+  short mPhase = 0;                                                                     ///< event phase
+  UInt_t mROFrameMin = 0;                                                               ///< lowest RO frame of current digits
+  UInt_t mROFrameMax = 0;                                                               ///< highest RO frame of current digits
+  bool mSmearEnergy = true;                                                             ///< do time and energy smearing
+  bool mSimulateTimeResponse = true;                                                    ///< simulate time response
+  const SimParam* mSimParam = nullptr;                                                  ///< SimParam object
+  bool mIsBeforeFirstRO = false;                                                        ///< check if the signal comes before the ROF
+  o2::InteractionRecord mIRFirstSampledTF;                                              ///< IR of the 1st sampled IR, noise-only ROFs will be inserted till this IR only
+  double mTimeBCns;                                                                     ///< time difference between bc and start of ROF in ns
+
+  std::vector mTempDigitVector;     ///< temporary digit storage
   o2::emcal::DigitsWriteoutBuffer mDigits; ///< used to sort digits and labels by tower
 
-  TRandom3* mRandomGenerator = nullptr;                  ///< random number generator
-  std::vector> mAmplitudeInTimeBins; ///< amplitude of signal for each time bin
-
-  int mTimeWindowStart = 7; ///< The start of the time window
-  int mDelay = 7;           ///< number of (full) time bins corresponding to the signal time delay
+  TRandom3* mRandomGenerator = nullptr; ///< random number generator
+  std::array, EMC_PHASES>
+    mAmplitudeInTimeBins; ///< template of the sampled time response function: amplitude of signal for each time bin (per phase)
 
   std::unique_ptr mDebugStream = nullptr;
   bool mEnableDebugStreaming = false;
diff --git a/Detectors/EMCAL/simulation/include/EMCALSimulation/DigitizerTRU.h b/Detectors/EMCAL/simulation/include/EMCALSimulation/DigitizerTRU.h
new file mode 100644
index 0000000000000..1a29176b8e9cc
--- /dev/null
+++ b/Detectors/EMCAL/simulation/include/EMCALSimulation/DigitizerTRU.h
@@ -0,0 +1,161 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef ALICEO2_EMCAL_TRIGGERDIGITIZER_H
+#define ALICEO2_EMCAL_TRIGGERDIGITIZER_H
+
+#include 
+#include 
+#include 
+#include 
+
+#include "Rtypes.h"  // for DigitizerTRU::Class, Double_t, ClassDef, etc
+#include "TObject.h" // for TObject
+#include "TRandom3.h"
+
+#include "DataFormatsEMCAL/Digit.h"
+#include "EMCALBase/Hit.h"
+#include "EMCALBase/TriggerMappingV2.h"
+#include "EMCALSimulation/SimParam.h"
+#include "EMCALSimulation/LabeledDigit.h"
+#include "EMCALSimulation/DigitsWriteoutBufferTRU.h"
+#include "EMCALSimulation/LZEROElectronics.h"
+#include "SimulationDataFormat/MCTruthContainer.h"
+#include "DataFormatsEMCAL/TriggerRecord.h"
+#include "CommonUtils/TreeStreamRedirector.h"
+#include "EMCALCalib/FeeDCS.h"
+
+namespace o2
+{
+namespace utils
+{
+class TreeStreamRedirector;
+}
+namespace emcal
+{
+class FeeDCS;
+
+/// \class DigitizerTRU
+/// \brief EMCAL DigitizerTRU, digitizes with the help of a temporary description based upon a pol9*Heavyside
+/// \ingroup EMCALsimulation
+/// \author Anders Knospe, University of Houston
+/// \author Hadi Hassan, ORNL
+/// \author Simone Ragoni, Creighton
+class DigitizerTRU
+{
+ public:
+  DigitizerTRU() = default;
+  ~DigitizerTRU() = default;
+  DigitizerTRU(const DigitizerTRU&) = delete;
+  DigitizerTRU& operator=(const DigitizerTRU&) = delete;
+
+  void init();
+  void clear();
+
+  /// \brief Sets patches for the current geometry
+  void setPatches();
+
+  /// clear DigitsVectorStream
+  void flush() { mDigits.flush(); }
+
+  /// This is for the readout window that was interrupted by the end of the run
+  void finish();
+
+  /// Steer conversion of hits to digits
+  void process(const gsl::span summableDigits);
+
+  /// Postprocessing of the digits, gathers by Fastors, not by Tower/Cell
+  /// \param sdigits results of the SDigitizer
+  std::vector> makeAnaloguesFastorSums(const gsl::span sdigits);
+
+  void setEventTime(o2::InteractionTimeRecord record);
+
+  /// Sets geometry for trigger mapping
+  void setGeometry(o2::emcal::Geometry* gm) { mGeometry = gm; }
+
+  /// Sets FEE DCS for the masking of the fastOrs
+  void setFEE(o2::emcal::FeeDCS* fees) { mFeeDCS = fees; }
+
+  /// Sets the masked fastOrs from the CCDB in the LZERO
+  void setMaskedFastOrsInLZERO();
+  void printMaskedFastOrsInLZERO();
+
+  void setWindowStartTime(int time) { mTimeWindowStart = time; }
+  void setDebugStreaming(bool doStreaming) { mEnableDebugStreaming = doStreaming; }
+
+  void fillOutputContainer(std::vector& digits, o2::dataformats::MCTruthContainer& labels);
+
+  bool doSmearEnergy() const { return mSmearEnergy; }
+  double smearEnergy(double energy);
+  double smearTime(double time, double energy);
+  bool doSimulateTimeResponse() const { return mSimulateTimeResponse; }
+
+  void sampleSDigit(const Digit& sdigit);
+
+  /// Close the TreeStreamer to make the file readable
+  void endDebugStream()
+  {
+    mDebugStream->Close();
+    // mDebugStreamPatch->Close();
+  }
+
+  /// Getter for debug mode
+  bool isDebugMode() { return mEnableDebugStreaming; }
+
+  /// Getter for triggers
+  const std::vector& getTriggerInputs() { return LZERO.getTriggerInputs(); }
+
+  /// Getter for patches
+  std::vector getPatchesVector() { return patchesFromAllTRUs; }
+
+  /// raw pointers used here to allow interface with TF1
+  static double rawResponseFunction(double* x, double* par);
+
+  /// Utility functions obtained from QC for EMC
+  int GetTRUIndexFromSTUIndex(Int_t id, Int_t detector);
+  int GetChannelForMaskRun2(int mask, int bitnumber, bool onethirdsm);
+  std::vector GetAbsFastORIndexFromMask();
+
+ private:
+  short mEventTimeOffset = 0;        ///< event time difference from trigger time (in number of bins)
+  bool mSmearEnergy = true;          ///< do time and energy smearing
+  bool mSimulateTimeResponse = true; ///< simulate time response
+  // const SimParam* mSimParam = nullptr; ///< SimParam object
+
+  std::vector mTempDigitVector; ///< temporary digit storage
+  // std::unordered_map> mDigits; ///< used to sort digits and labels by tower
+  o2::emcal::DigitsWriteoutBufferTRU mDigits;     ///< used to sort digits by tower
+  o2::emcal::LZEROElectronics LZERO;              ///< to start the trigger
+  std::vector patchesFromAllTRUs; ///< patches from all TRUs
+
+  // TRandom3* mRandomGenerator = nullptr; ///< random number generator
+  std::array
+    mAmplitudeInTimeBins; ///< template of the sampled time response function: amplitude of signal for each time bin (per phase)
+
+  // TriggerMappingV2* mTriggerMap = nullptr; ///< Trigger map for tower to fastor ID
+  Geometry* mGeometry = nullptr; ///< EMCAL geometry
+
+  int mTimeWindowStart = 7;      ///< The start of the time window
+  int mDelay = 7;                ///< number of (full) time bins corresponding to the signal time delay
+  bool mWasTriggerFound = false; ///< To save the data
+  int mPreviousTriggerSize = 0;  ///< To save the data
+
+  std::unique_ptr mDebugStream = nullptr;
+  // std::unique_ptr mDebugStreamPatch = nullptr;
+  bool mEnableDebugStreaming = false;
+  o2::emcal::FeeDCS* mFeeDCS; ///< EMCAL FEE DCS
+
+  ClassDefNV(DigitizerTRU, 1);
+};
+} // namespace emcal
+} // namespace o2
+
+#endif /* ALICEO2_EMCAL_TRIGGERDIGITIZER_H */
diff --git a/Detectors/EMCAL/simulation/include/EMCALSimulation/DigitsVectorStream.h b/Detectors/EMCAL/simulation/include/EMCALSimulation/DigitsVectorStream.h
index 15848ce4839eb..861b4d44d54b7 100644
--- a/Detectors/EMCAL/simulation/include/EMCALSimulation/DigitsVectorStream.h
+++ b/Detectors/EMCAL/simulation/include/EMCALSimulation/DigitsVectorStream.h
@@ -25,6 +25,7 @@
 #include "DataFormatsEMCAL/TriggerRecord.h"
 #include "SimulationDataFormat/MCTruthContainer.h"
 #include "EMCALSimulation/SimParam.h"
+#include "EMCALSimulation/DigitTimebin.h"
 
 namespace o2
 {
@@ -38,14 +39,6 @@ namespace emcal
 /// \author Markus Fasel, ORNL
 /// \date 16/02/2022
 
-struct DigitTimebin {
-  bool mRecordMode = false;
-  bool mEndWindow = false;
-  bool mTriggerColl = false;
-  std::optional mInterRecord;
-  std::shared_ptr>> mDigitMap = std::make_shared>>();
-};
-
 class DigitsVectorStream
 {
 
diff --git a/Detectors/EMCAL/simulation/include/EMCALSimulation/DigitsWriteoutBuffer.h b/Detectors/EMCAL/simulation/include/EMCALSimulation/DigitsWriteoutBuffer.h
index 01920c1912949..5713f2ef18ad9 100644
--- a/Detectors/EMCAL/simulation/include/EMCALSimulation/DigitsWriteoutBuffer.h
+++ b/Detectors/EMCAL/simulation/include/EMCALSimulation/DigitsWriteoutBuffer.h
@@ -60,8 +60,21 @@ class DigitsWriteoutBuffer
 
   double getTriggerTime() const { return mTriggerTime; }
   double getEventTime() const { return mLastEventTime; }
-  bool isLive(double t) const { return ((t - mTriggerTime) < mLiveTime || (t - mTriggerTime) >= (mLiveTime + mBusyTime - mPreTriggerTime)); }
-  bool isLive() const { return ((mLastEventTime - mTriggerTime) < mLiveTime || (mLastEventTime - mTriggerTime) >= (mLiveTime + mBusyTime - mPreTriggerTime)); }
+  bool isLive(double t) const
+  {
+    return ((t - mTriggerTime) < mLiveTime || (t - mTriggerTime) >= (mLiveTime + mBusyTime - mPreTriggerTime));
+  }
+  bool isLive() const
+  {
+    return ((mLastEventTime - mTriggerTime) < (mLiveTime - mPreTriggerTime) || (mLastEventTime - mTriggerTime) >= (mLiveTime + mBusyTime - mPreTriggerTime));
+  }
+
+  /// Check if current collision was triggered
+  /// \return true if event was triggered
+  bool isCurrentEventTriggered() const
+  {
+    return mLastEventTime == mTriggerTime;
+  }
 
   // function returns true if the collision occurs 600ns before the readout window is open
   // Look here for more details https://alice.its.cern.ch/jira/browse/EMCAL-681
@@ -81,7 +94,7 @@ class DigitsWriteoutBuffer
   unsigned int getBufferSize() const { return mBufferSize; }
 
   /// forward the marker for every 100 ns
-  void forwardMarker(o2::InteractionTimeRecord record);
+  void forwardMarker(o2::InteractionTimeRecord record, bool trigger);
 
   /// Setters for the live time, busy time, pre-trigger time
   void setLiveTime(unsigned int liveTime) { mLiveTime = liveTime; }
@@ -102,6 +115,7 @@ class DigitsWriteoutBuffer
   unsigned long mTriggerTime = 0;                         ///< Time of the collision that fired the trigger (ns)
   unsigned long mLastEventTime = 0;                       ///< The event time of last collisions in the readout window
   unsigned int mPhase = 0;                                ///< The event L1 phase
+  unsigned int mSwapPhase = 0;                            ///< BC phase swap
   bool mFirstEvent = true;                                ///< Flag to the first event in the run
   std::deque mTimedDigitsFuture; ///< Container for time sampled digits per tower ID for future digits
   std::deque mTimedDigitsPast;   ///< Container for time sampled digits per tower ID for past digits
diff --git a/Detectors/EMCAL/simulation/include/EMCALSimulation/DigitsWriteoutBufferTRU.h b/Detectors/EMCAL/simulation/include/EMCALSimulation/DigitsWriteoutBufferTRU.h
new file mode 100644
index 0000000000000..d6452a91b920e
--- /dev/null
+++ b/Detectors/EMCAL/simulation/include/EMCALSimulation/DigitsWriteoutBufferTRU.h
@@ -0,0 +1,126 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+//_____________________________________________
+// GENERAL IDEA
+//
+// - Focus on https://github.com/AliceO2Group/AliceO2/blob/dev/Detectors/TPC/simulation/include/TPCSimulation/DigitContainer.h
+// - Comform it to the classes used by EMCAL in https://github.com/AliceO2Group/AliceO2/blob/dev/Detectors/EMCAL/simulation/include/EMCALSimulation/DigitsWriteoutBuffer.h
+//   and in https://github.com/AliceO2Group/AliceO2/blob/dev/Detectors/EMCAL/simulation/include/EMCALSimulation/DigitsVectorStream.h
+// - The two concepts are similar
+// - EMCAL's was however adapted to triggered mode
+// - The principle is to reapply the old TPC format to the continuous readout directly
+//
+// 1) the idea of dumping the time bins in 15 bins is dumped
+// 2) the data are pushed to the output stream whenever possible
+// 3) the EndOfRun is implemented to break the push to stream
+//    it is a flag set to 1 which is read by the filloutputcontainer
+// 4) no more check of start or end of trigger
+// 5) what about the precollision flags?
+// 6) is there any longer a need to set the time of the sampled digits time?
+
+#ifndef ALICEO2_EMCAL_DIGITSWRITEOUTBUFFERTRU_H_
+#define ALICEO2_EMCAL_DIGITSWRITEOUTBUFFERTRU_H_
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include "DataFormatsEMCAL/Digit.h"
+#include "CommonDataFormat/InteractionRecord.h"
+#include "EMCALSimulation/LZEROElectronics.h"
+#include "EMCALSimulation/DigitTimebin.h"
+
+// using namespace o2::emcal;
+
+namespace o2
+{
+namespace emcal
+{
+
+/// \class DigitsWriteoutBufferTRU
+/// \brief Container class for time sampled digits to be sent to TRUs in true continuous readout
+/// \ingroup EMCALsimulation
+/// \author Hadi Hassan, ORNL
+/// \author Markus Fasel, ORNL
+/// \author Simone Ragoni, ORNL
+/// \date 27/09/2022
+
+class DigitsWriteoutBufferTRU
+{
+ public:
+  /// Default constructor
+  DigitsWriteoutBufferTRU(unsigned int nTimeBins = 15);
+
+  /// Destructor
+  ~DigitsWriteoutBufferTRU() = default;
+
+  /// clear the container
+  void clear();
+
+  /// clear DigitsVectorStream
+  void flush()
+  {
+    // mDigitStream.clear();
+  }
+
+  void init();
+
+  /// Reserve space for the future container
+  /// \param eventTimeBin resize adding at the end
+  void reserve(int eventTimeBin);
+
+  /// This is for the readout window that was interrupted by the end of the run
+  void finish();
+
+  /// Add digit to the container
+  /// \param towerID Cell ID
+  /// \param dig Labaled digit to add
+  void addDigits(unsigned int towerID, std::vector& digList);
+
+  /// Fill output streamer
+  /// \param isEndOfTimeFrame End of Time Frame
+  /// \param nextInteractionRecord Next interaction record, to compute the amount of TimeBins to be saved
+  void fillOutputContainer(bool isEndOfTimeFrame, InteractionRecord& nextInteractionRecord, std::vector& patchesFromAllTRUs, LZEROElectronics& LZERO);
+
+  /// Setters for the live time, busy time, pre-trigger time
+  void setLiveTime(unsigned int liveTime) { mLiveTime = liveTime; }
+  void setBusyTime(unsigned int busyTime) { mBusyTime = busyTime; }
+
+  const std::deque& getTimeBins() const { return mTimeBins; }
+
+ private:
+  unsigned int mBufferSize = 15;  ///< The size of the buffer
+  unsigned int mLiveTime = 1500;  ///< EMCal live time (ns)
+  unsigned int mBusyTime = 35000; ///< EMCal busy time (ns)
+  // unsigned int mPreTriggerTime = 600;               ///< EMCal pre-trigger time (ns)
+  unsigned long mTriggerTime = 0;                   ///< Time of the collision that fired the trigger (ns)
+  unsigned long mLastEventTime = 0;                 ///< The event time of last collisions in the readout window
+  unsigned int mPhase = 0;                          ///< The event L1 phase
+  bool mFirstEvent = true;                          ///< Flag to the first event in the run
+  std::deque mTimeBins; ///< Container for time sampled digits per tower ID for continuous digits
+  unsigned int mFirstTimeBin = 0;
+  bool mEndOfRun = 0;
+  bool mNoPileupMode = false;                      ///< pileup mode from SimParam
+  o2::InteractionRecord mCurrentInteractionRecord; ///< Interaction Record of the current event, to be used to fill the output container
+
+  ClassDefNV(DigitsWriteoutBufferTRU, 5);
+  // ClassDefNV are for objects which do not inherit from tobject
+  // you do not need classIMP instead
+};
+
+} // namespace emcal
+
+} // namespace o2
+
+#endif /* ALICEO2_EMCAL_DIGITSWRITEOUTBUFFERTRU_H_ */
\ No newline at end of file
diff --git a/Detectors/EMCAL/simulation/include/EMCALSimulation/LZEROElectronics.h b/Detectors/EMCAL/simulation/include/EMCALSimulation/LZEROElectronics.h
new file mode 100644
index 0000000000000..4927515b7691c
--- /dev/null
+++ b/Detectors/EMCAL/simulation/include/EMCALSimulation/LZEROElectronics.h
@@ -0,0 +1,152 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef ALICEO2_EMCAL_LZEROELECTRONICS_H_
+#define ALICEO2_EMCAL_LZEROELECTRONICS_H_
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include "TRandom3.h"
+#include "DataFormatsEMCAL/Digit.h"
+#include "EMCALSimulation/DigitTimebin.h"
+#include "DataFormatsEMCAL/TriggerRecord.h"
+#include "SimulationDataFormat/MCTruthContainer.h"
+#include "EMCALSimulation/SimParam.h"
+#include "EMCALSimulation/TRUElectronics.h"
+#include "EMCALBase/TriggerMappingV2.h"
+#include  // for LOG
+
+namespace o2
+{
+namespace emcal
+{
+
+/// @brief Trigger Inputs object, summary of the entire information needed for the L1 algorithm
+/// \param mInterRecord Last known interaction record
+/// \param mLastTimesumAllFastOrs Vector of tuples with TRU ID, FastOrID with STU indexing, and their last Timesums
+struct EMCALTriggerInputs {
+  o2::InteractionRecord mInterRecord;                               ///< Last known interaction record
+  int mTriggeredTRU;                                                ///< Trigger TRU
+  std::vector mTriggeredPatches;                               ///< Trigger patches, in local STU indexing
+  std::vector> mLastTimesumAllFastOrs; ///< TRU ID, FastOrID with STU indexing, and its last Timesum
+};
+
+/// @brief Trigger Inputs object, summary of the entire information needed for the L1 algorithm
+/// \param mInterRecord Last known interaction record
+/// \param mLastTimesumAllPatches Vector of tuples with TRU ID, PatchID, and their last Timesums
+struct EMCALTriggerInputsPatch {
+  o2::InteractionRecord mInterRecord;                                     ///< Last known interaction record
+  std::vector> mLastTimesumAllPatches; ///< TRU ID, PatchID, and its last Timesum
+};
+
+/// \class LZEROElectronics
+/// \brief Container class for Digits, MC lebels, and trigger records
+/// \ingroup EMCALsimulation
+/// \author Markus Fasel, ORNL
+/// \author Simone Ragoni, Creighton
+/// \date 22/11/2022
+
+class LZEROElectronics
+{
+
+ public:
+  /// Default constructor
+  LZEROElectronics() = default;
+
+  /// Destructor
+  ~LZEROElectronics() = default;
+
+  /// clear the L0 electronics
+  void clear();
+
+  /// Initialize the L0 electronics
+  void init();
+
+  /// Sets geometry for trigger mapping
+  void setGeometry(o2::emcal::Geometry* gm) { mGeometry = gm; }
+
+  /// Sets the masked fastOrs from the CCDB in the LZERO
+  void setMaskedFastOrs(std::vector const& maskedfastors) { mMaskedFastOrs = maskedfastors; }
+  void printMaskedFastOrs();
+
+  /// Set Threshold for LZERO algorithm
+  /// \param threshold LZERO algorithm threshold
+  void setThreshold(double threshold) { mThreshold = threshold; }
+
+  /// Implements the peak finder algorithm on the patch
+  /// \param p TRUElectronics object
+  /// \param patchID Patch ID to implement the peak finding algorithm
+  bool peakFinderOnPatch(TRUElectronics& p, unsigned int patchID);
+
+  /// Calls the peak finder algorithm on all patches
+  /// \param p TRUElectronics object
+  bool peakFinderOnAllPatches(TRUElectronics& p);
+
+  /// Update patches
+  /// \param p TRUElectronics object
+  void updatePatchesADC(TRUElectronics& p);
+
+  /// Add noise to this digit
+  void addNoiseDigits(Digit& d1);
+
+  /// Implements the fill of the patches. Runs the peak finding, and ships to L1 in case it finds something
+  /// \param digitlist digits to be assigned to patches
+  /// \param record interaction record time to be propagated
+  /// \param patchesFromAllTRUs vector contained the patches of all TRUs
+  void fill(const std::deque& digitlist, const o2::InteractionRecord record, std::vector& patchesFromAllTRUs);
+
+  /// Getter for the pattern of peaks found by the LZERO algorithm
+  /// \param p TRUElectronics object
+  const std::vector& getFiredPatches(TRUElectronics const& p) const
+  {
+    return p.mFiredPatches;
+  }
+
+  // Getter for the threshold used for the integral of the ADC values in the LZERO algorithm
+  const double getLZEROThreshold() const { return mThreshold; }
+
+  /// Getter for the trigger inputs found by the LZERO algorithm
+  const std::vector& getTriggerInputs() const
+  {
+    LOG(debug) << "DIG TRU getTriggerInputs in LZEROElectronics: size of  mTriggers = " << mTriggers.size();
+    return mTriggers;
+  }
+
+  /// Getter for the trigger inputs per patches found by the LZERO algorithm
+  const std::vector& getTriggerInputsPatches() const
+  {
+    return mTriggersPatch;
+  }
+
+ private:
+  double mThreshold = 0;
+  // TRandom3* mRandomGenerator = nullptr;                ///< random number generator
+  // const SimParam* mSimParam = nullptr;                 ///< SimParam object
+  std::vector mTriggers;           ///< Triggers to be sent out
+  std::vector mTriggersPatch; ///< Triggers to be sent out
+  std::vector mMaskedFastOrs;                     ///< Masked fastOrs from CCDB
+  bool mSimulateNoiseDigits = true;                    ///< simulate noise digits
+  // TriggerMappingV2* mTriggerMap = nullptr;             ///< Trigger map to properly assign an absolute FastOr to TRU FastOr
+  Geometry* mGeometry = nullptr; ///< EMCAL geometry
+
+  ClassDefNV(LZEROElectronics, 2);
+};
+
+} // namespace emcal
+
+} // namespace o2
+
+#endif /* ALICEO2_EMCAL_LZEROELECTRONICS_H_ */
diff --git a/Detectors/EMCAL/simulation/include/EMCALSimulation/RawWriter.h b/Detectors/EMCAL/simulation/include/EMCALSimulation/RawWriter.h
index 91c534c3ba82b..4388035215d4f 100644
--- a/Detectors/EMCAL/simulation/include/EMCALSimulation/RawWriter.h
+++ b/Detectors/EMCAL/simulation/include/EMCALSimulation/RawWriter.h
@@ -191,6 +191,18 @@ class RawWriter
   /// The input data is converted to 10 but ALTRO words and put on the stream.
   std::vector encodeBunchData(const std::vector& data);
 
+  /// \brief Extracting branch index from the hardware address
+  /// \param hwaddress Hardware address of the channel
+  int getBranchIndexFromHwAddress(int hwaddress) { return ((hwaddress >> 11) & 0x1); }
+
+  /// \brief Extracting FEC index in branch from the hardware address
+  /// \param hwaddress Hardware address of the channel
+  int getFecIndexFromHwAddress(int hwaddress) { return ((hwaddress >> 7) & 0xF); }
+
+  /// \brief Extracting Channel index in FEC from the hardware address
+  /// \param hwaddress Hardware address of the channel
+  int getChannelIndexFromHwAddress(int hwaddress) { return (hwaddress & 0xF); }
+
  private:
   int mNADCSamples = 15;                                      ///< Number of time samples
   int mPedestal = 1;                                          ///< Pedestal
diff --git a/Detectors/EMCAL/simulation/include/EMCALSimulation/SimParam.h b/Detectors/EMCAL/simulation/include/EMCALSimulation/SimParam.h
index 0146953e9a94b..e1cd6728f537d 100644
--- a/Detectors/EMCAL/simulation/include/EMCALSimulation/SimParam.h
+++ b/Detectors/EMCAL/simulation/include/EMCALSimulation/SimParam.h
@@ -33,9 +33,6 @@ class SimParam : public o2::conf::ConfigurableParamHelper
   Int_t getDigitThreshold() const { return mDigitThreshold; }
   Float_t getPinNoise() const { return mPinNoise; }
   Float_t getPinNoiseLG() const { return mPinNoiseLG; }
-  Float_t getTimeNoise() const { return mTimeNoise; }
-  Float_t getTimeDelay() const { return mTimeDelay; }
-  Bool_t isTimeDelayFromOCDB() const { return mTimeDelayFromOCDB; }
   Float_t getTimeResolutionPar0() const { return mTimeResolutionPar0; }
   Float_t getTimeResolutionPar1() const { return mTimeResolutionPar1; }
   Double_t getTimeResolution(Double_t energy) const;
@@ -45,6 +42,16 @@ class SimParam : public o2::conf::ConfigurableParamHelper
   Float_t getTimeResponseTau() const { return mTimeResponseTau; }
   Float_t getTimeResponsePower() const { return mTimeResponsePower; }
   Float_t getTimeResponseThreshold() const { return mTimeResponseThreshold; }
+  Int_t getBCPhaseSwap() const { return mSwapPhase; }
+
+  // Parameters used in TRU Digitizer
+  Float_t getPinNoiseTRU() const { return mPinNoiseTRU; }
+  Float_t getTimeResponseTauTRU() const { return mTimeResponseTauTRU; }
+  Float_t getTimeResponsePowerTRU() const { return mTimeResponsePowerTRU; }
+  Float_t getTimeResponseNormalisationTRU() const { return mTimeResponseNormalisationTRU; }
+
+  // Parameters used in LZERO
+  Float_t getThresholdLZERO() const { return mThresholdLZERO; }
 
   // Parameters used in SDigitizer
   Float_t getA() const { return mA; }
@@ -67,22 +74,28 @@ class SimParam : public o2::conf::ConfigurableParamHelper
 
   void PrintStream(std::ostream& stream) const;
 
- private:
   // Digitizer
   Int_t mDigitThreshold{3};              ///< Threshold for storing digits in EMC
   Int_t mMeanPhotonElectron{4400};       ///< number of photon electrons per GeV deposited energy
   Float_t mGainFluctuations{15.};        ///< correct fMeanPhotonElectron by the gain fluctuations
   Float_t mPinNoise{0.012};              ///< Electronics noise in EMC, APD
   Float_t mPinNoiseLG{0.1};              ///< Electronics noise in EMC, APD, Low Gain
-  Float_t mTimeNoise{1.28e-5};           ///< Electronics noise in EMC, time
-  Float_t mTimeDelay{600e-9};            ///< Simple time delay to mimick roughly delay in data
-  Bool_t mTimeDelayFromOCDB{false};      ///< Get time delay from OCDB
   Float_t mTimeResolutionPar0{0.26666};  ///< Time resolution of FEE electronics
   Float_t mTimeResolutionPar1{1.4586};   ///< Time resolution of FEE electronics
   Int_t mNADCEC{0x10000};                ///< number of channels in EC section ADC
   Float_t mTimeResponseTau{2.35};        ///< Raw time response function tau parameter
   Float_t mTimeResponsePower{2};         ///< Raw time response function power parameter
   Float_t mTimeResponseThreshold{0.001}; ///< Raw time response function energy threshold
+  Int_t mSwapPhase{0};                   ///< BC phase swap similar to data
+
+  // TRU Digitizer
+  Float_t mPinNoiseTRU{0.04};                                         ///< Electronics noise in EMC, TRU, normalised by fastOr
+  Float_t mTimeResponseTauTRU{61.45 / 25.};                           ///< Raw time response function tau parameter TRU from Martin Poghosyan
+  Float_t mTimeResponsePowerTRU{2.};                                  ///< Raw time response function power parameter TRU from Martin Poghosyan
+  Float_t mTimeResponseNormalisationTRU{(5. / 3.93) * (45. / 39.25)}; ///< Raw time response function normalisation parameter TRU from correlation studies
+
+  // LZERO peak finding
+  Float_t mThresholdLZERO{132.}; ///< ADC threshold for peak finding in the LZEROElectronics
 
   // SDigitizer
   Float_t mA{0.};                 ///< Pedestal parameter
@@ -112,6 +125,16 @@ class SimParam : public o2::conf::ConfigurableParamHelper
 std::ostream& operator<<(std::ostream& stream, const SimParam& s);
 
 } // namespace emcal
+
+namespace framework
+{
+template 
+struct is_messageable;
+template <>
+struct is_messageable : std::true_type {
+};
+} // namespace framework
+
 } // namespace o2
 
 #endif
diff --git a/Detectors/EMCAL/simulation/include/EMCALSimulation/TRUElectronics.h b/Detectors/EMCAL/simulation/include/EMCALSimulation/TRUElectronics.h
new file mode 100644
index 0000000000000..8d4a14cfee9bb
--- /dev/null
+++ b/Detectors/EMCAL/simulation/include/EMCALSimulation/TRUElectronics.h
@@ -0,0 +1,129 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file TRUElectronics.h
+/// \brief EMCAL TRUElectronics for the LZEROElectronics
+#ifndef ALICEO2_EMCAL_PATCH_H
+#define ALICEO2_EMCAL_PATCH_H
+
+#include 
+#include 
+#include 
+#include "Rtypes.h"
+// #include "DataFormatsEMCAL/Cluster.h"
+// #include "DataFormatsEMCAL/Digit.h"
+// #include "DataFormatsEMCAL/Cell.h"
+// #include "EMCALBase/Geometry.h"
+
+// #include  // for LOG
+
+namespace o2
+{
+
+namespace emcal
+{
+
+/// \param mADCvalues vector of the ADC values
+/// \param previousTimebinADCvalue previous ADC value, popped out (i-4) if i refers to the current Timebin
+struct FastOrStruct {
+  std::vector mADCvalues;
+  double mPreviousTimebinADCvalue = 0.;
+
+  /// Compute current timesum
+  double timesum()
+  {
+    double timesumvalue = 0;
+    for (auto ADCvalue : mADCvalues) {
+      timesumvalue += ADCvalue;
+    }
+    return timesumvalue;
+  }
+
+  void init()
+  {
+  }
+
+  // Update internal ADC values (4 timebins)
+  void updateADC(double ADCvalue)
+  {
+
+    if (mADCvalues.size() == 4) {
+      mPreviousTimebinADCvalue = mADCvalues.front();
+      mADCvalues.erase(mADCvalues.begin());
+    }
+    mADCvalues.push_back(ADCvalue);
+  }
+};
+
+/// \struct TRUElectronics
+/// \brief TRUElectronics creator, based on the TRUElectronics
+/// \ingroup EMCALsimulation
+/// \author Markus Fasel, ORNL
+/// \author Simone Ragoni, Creighton U.
+/// \date 03/12/2022
+///
+
+struct TRUElectronics {
+
+  /// \brief Main constructor
+  /// \param patchSize patch size: 2x2, or 4x4
+  /// \param whichSide 0 = A side, 1 = C side
+  /// \param whichSuperModuleSize 0 = Full 1 = 1/3
+  TRUElectronics(int patchSize, int whichSide, int whichSuperModuleSize);
+
+  /// \brief Default constructor
+  TRUElectronics();
+
+  /// \brief Destructor
+  ~TRUElectronics() = default;
+
+  /// \brief Clear internal members
+  void clear();
+
+  /// \brief Initialise internal members
+  void init();
+
+  /// \brief Assign seed module to a Full SM
+  /// \param patchID Patch ID that need to be assigned a seed
+  void assignSeedModuleToPatchWithSTUIndexingFullModule(int& patchID);
+
+  /// \brief Assign seed module to a 1/3 SM
+  /// \param patchID Patch ID that need to be assigned a seed
+  void assignSeedModuleToPatchWithSTUIndexingOneThirdModule(int& patchID);
+
+  /// \brief Assign seed module to all patches
+  void assignSeedModuleToAllPatches();
+
+  /// \brief Assign modules to all patches
+  void assignModulesToAllPatches();
+
+  /// \brief Updates the patches
+  void updateADC();
+
+  int mPatchSize;            //!> mPatchIDSeedFastOrIDs;                  //!>> mIndexMapPatch;            //!>> mFiredFastOrIndexMapPatch; //! mFiredPatches;                                           //!>> mADCvalues;             //!>> mTimesum;               //!> mPreviousTimebinADCvalue;            //! mFastOrs;                                       //!
 #include 
diff --git a/Detectors/EMCAL/simulation/src/Digitizer.cxx b/Detectors/EMCAL/simulation/src/Digitizer.cxx
index 0edc2c3533958..8360cb4fd67e8 100644
--- a/Detectors/EMCAL/simulation/src/Digitizer.cxx
+++ b/Detectors/EMCAL/simulation/src/Digitizer.cxx
@@ -22,9 +22,12 @@
 #include 
 #include 
 #include 
-#include "FairLogger.h" // for LOG
+#include  // for LOG
 #include "CommonDataFormat/InteractionRecord.h"
 #include "CommonUtils/TreeStreamRedirector.h"
+#include "SimConfig/DigiParams.h"
+#include "DetectorsRaw/HBFUtilsInitializer.h"
+#include "DetectorsRaw/HBFUtils.h"
 
 ClassImp(o2::emcal::Digitizer);
 
@@ -37,39 +40,50 @@ using namespace o2::emcal;
 void Digitizer::init()
 {
   mSimParam = &(o2::emcal::SimParam::Instance());
-  mRandomGenerator = new TRandom3(std::chrono::high_resolution_clock::now().time_since_epoch().count());
+  auto randomSeed = std::chrono::high_resolution_clock::now().time_since_epoch().count();
+  if (o2::conf::DigiParams::Instance().seed != 0) {
+    randomSeed = o2::conf::DigiParams::Instance().seed;
+  }
+  mRandomGenerator = new TRandom3(randomSeed);
 
   float tau = mSimParam->getTimeResponseTau();
   float N = mSimParam->getTimeResponsePower();
-  float delay = std::fmod(mSimParam->getSignalDelay() / constants::EMCAL_TIMESAMPLE, 1);
-  mDelay = ((int)(std::floor(mSimParam->getSignalDelay() / constants::EMCAL_TIMESAMPLE)));
-  mTimeWindowStart = ((unsigned int)(std::floor(mSimParam->getTimeBinOffset() / constants::EMCAL_TIMESAMPLE)));
 
   mSmearEnergy = mSimParam->doSmearEnergy();
   mSimulateTimeResponse = mSimParam->doSimulateTimeResponse();
 
   mDigits.init();
-  if ((mDelay - mTimeWindowStart) != 0) {
+  mDigits.reserve();
+  /*
+  if ((mDelay - mTimeWindowStart) != 0)
+  {
     mDigits.setBufferSize(mDigits.getBufferSize() + (mDelay - mTimeWindowStart));
     mDigits.reserve();
   }
+  */
 
-  mAmplitudeInTimeBins.clear();
-
-  // for each phase create a template distribution
-  TF1 RawResponse("RawResponse", rawResponseFunction, 0, 256, 5);
-  RawResponse.SetParameters(1., 0., tau, N, 0.);
-
-  for (int i = 0; i < 4; i++) {
-
-    std::vector sf;
-    RawResponse.SetParameter(1, 0.25 * i);
-    for (int j = 0; j < constants::EMCAL_MAXTIMEBINS; j++) {
-      sf.push_back(RawResponse.Eval(j - mTimeWindowStart));
+  if (mSimulateTimeResponse) {
+    // for each phase create a template distribution
+    TF1 RawResponse("RawResponse", rawResponseFunction, 0, 256, 5);
+    RawResponse.SetParameters(1., 0., tau, N, 0.);
+
+    for (int phase = 0; phase < 4; phase++) {
+      // parameter 1: Handling phase + delay
+      // phase: 25 ns * phase index (-4)
+      // delay: Average signal delay
+      for (int itofbin = 0; itofbin < EMC_TOF_BINS; itofbin++) {
+        double tofbincenter = itofbin * EMC_TOF_BINWITH + 0.5 * EMC_TOF_BINWITH;
+        RawResponse.SetParameter(1, 0.25 * phase + (tofbincenter + mSimParam->getSignalDelay()) / constants::EMCAL_TIMESAMPLE);
+        for (int sample = 0; sample < constants::EMCAL_MAXTIMEBINS; sample++) {
+          mAmplitudeInTimeBins[phase][itofbin][sample] = RawResponse.Eval(sample);
+        }
+      }
     }
-    mAmplitudeInTimeBins.push_back(sf);
+  } else {
   }
 
+  mIRFirstSampledTF = o2::raw::HBFUtils::Instance().getFirstSampledTFIR();
+
   if (mEnableDebugStreaming) {
     mDebugStream = std::make_unique("emcaldigitsDebug.root", "RECREATE");
   }
@@ -154,18 +168,39 @@ void Digitizer::sampleSDigit(const Digit& sDigit)
     return;
   }
 
+  // check if this hit because it comes from an event before readout starts and it does not effect this RO
+  LOG(debug) << "mIsBeforeFirstRO " << mIsBeforeFirstRO << "     sDigit.getTimeStamp() " << sDigit.getTimeStamp() << "    mSimParam->getSignalDelay() " << mSimParam->getSignalDelay() << "   mPhase " << mPhase << "   total: " << sDigit.getTimeStamp() + mSimParam->getSignalDelay() + mPhase * 25 << "   EMC_TOF_MAX " << EMC_TOF_MAX << "  mTimeBCns " << mTimeBCns;
+  if (mIsBeforeFirstRO && sDigit.getTimeStamp() + mTimeBCns < 0) {
+    LOG(debug) << "disregard this hit because it comes from an event before readout starts and it does not effect this RO";
+    return;
+  }
+
   Double_t energies[15];
   if (mSimulateTimeResponse) {
-    for (int j = 0; j < mAmplitudeInTimeBins.at(mPhase).size(); j++) {
+    if (sDigit.getTimeStamp() + mSimParam->getSignalDelay() + mPhase * 25 > EMC_TOF_MAX) {
+      // Digit time larger than sampling window, will not be sampled
+      // For time response simulation take also signal delay and phase into account
+      return;
+    }
+    int tofbin = static_cast(sDigit.getTimeStamp() / EMC_TOF_BINWITH);
+    if (tofbin >= EMC_TOF_BINS) {
+      tofbin = EMC_TOF_BINS - 1;
+    }
+    for (int sample = 0; sample < mAmplitudeInTimeBins[mPhase][tofbin].size(); sample++) {
 
-      double val = energy * (mAmplitudeInTimeBins.at(mPhase).at(j));
-      energies[j] = val;
-      double digitTime = (mEventTimeOffset + mDelay - mTimeWindowStart) * constants::EMCAL_TIMESAMPLE;
+      double val = energy * (mAmplitudeInTimeBins[mPhase][tofbin][sample]);
+      energies[sample] = val;
+      double digitTime = mEventTimeOffset * constants::EMCAL_TIMESAMPLE;
       Digit digit(tower, val, digitTime);
       mTempDigitVector.push_back(digit);
     }
   } else {
-    Digit digit(tower, energy, (mDelay - mTimeWindowStart) * constants::EMCAL_TIMESAMPLE);
+    if (sDigit.getTimeStamp() > EMC_TOF_MAX) {
+      // Digit time larger than sampling window, will not be sampled
+      // In non-sampled mode only apply the max. time window
+      return;
+    }
+    Digit digit(tower, energy, smearTime(sDigit.getTimeStamp(), energy));
     mTempDigitVector.push_back(digit);
   }
 
@@ -203,11 +238,16 @@ double Digitizer::smearEnergy(double energy)
   return energy;
 }
 
+double Digitizer::smearTime(double time, double energy)
+{
+  return mRandomGenerator->Gaus(time + mSimParam->getSignalDelay(), mSimParam->getTimeResolution(energy));
+}
+
 //_______________________________________________________________________
-void Digitizer::setEventTime(o2::InteractionTimeRecord record)
+void Digitizer::setEventTime(o2::InteractionTimeRecord record, bool trigger)
 {
 
-  mDigits.forwardMarker(record);
+  mDigits.forwardMarker(record, trigger);
 
   mPhase = mSimParam->doSimulateL1Phase() ? mDigits.getPhase() : 0;
 
@@ -217,4 +257,16 @@ void Digitizer::setEventTime(o2::InteractionTimeRecord record)
     mPhase = 0;
     mEventTimeOffset++;
   }
+
+  // get time difference between current bc and start of RO in ns
+  auto nbc = record.differenceInBC(mIRFirstSampledTF);
+  mTimeBCns = record.getTimeOffsetWrtBC();
+  mTimeBCns += nbc * o2::constants::lhc::LHCBunchSpacingNS;
+
+  if (nbc < 0) {
+    // this event is before the first RO
+    mIsBeforeFirstRO = true;
+  } else {
+    mIsBeforeFirstRO = false;
+  }
 }
\ No newline at end of file
diff --git a/Detectors/EMCAL/simulation/src/DigitizerTRU.cxx b/Detectors/EMCAL/simulation/src/DigitizerTRU.cxx
new file mode 100644
index 0000000000000..7f0da790eff8c
--- /dev/null
+++ b/Detectors/EMCAL/simulation/src/DigitizerTRU.cxx
@@ -0,0 +1,514 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "EMCALSimulation/DigitizerTRU.h"
+#include "EMCALSimulation/SimParam.h"
+#include "EMCALSimulation/DigitsWriteoutBuffer.h"
+#include "EMCALCalib/FeeDCS.h"
+#include "DataFormatsEMCAL/Digit.h"
+#include "EMCALBase/Hit.h"
+#include "MathUtils/Cartesian.h"
+#include "SimulationDataFormat/MCCompLabel.h"
+#include "EMCALSimulation/DigitsWriteoutBuffer.h"
+#include 
+#include 
+#include 
+#include 
+#include 
+#include  // for LOG
+#include "CommonDataFormat/InteractionRecord.h"
+#include "CommonUtils/TreeStreamRedirector.h"
+
+#include "Framework/ConfigParamRegistry.h"
+#include "Framework/ControlService.h"
+#include "Framework/DataProcessorSpec.h"
+#include "Framework/DataRefUtils.h"
+#include "Framework/Lifetime.h"
+
+ClassImp(o2::emcal::DigitizerTRU);
+
+using o2::emcal::Digit;
+using o2::emcal::Hit;
+
+using namespace o2::emcal;
+
+//_______________________________________________________________________
+void DigitizerTRU::init()
+{
+  setPatches();
+  auto mSimParam = &(o2::emcal::SimParam::Instance());
+  // mRandomGenerator = new TRandom3(std::chrono::high_resolution_clock::now().time_since_epoch().count());
+
+  float tau = mSimParam->getTimeResponseTauTRU();
+  float N = mSimParam->getTimeResponsePowerTRU();
+
+  mSmearEnergy = mSimParam->doSmearEnergy();
+  mSimulateTimeResponse = mSimParam->doSimulateTimeResponse();
+
+  mDigits.init();
+  mDigits.reserve(15);
+
+  LZERO.setGeometry(mGeometry);
+  LZERO.init();
+
+  // Parameters from data (@Martin Poghosyan)
+  // tau = 61.45 / 25.; // 61.45 ns, according to the fact that the
+  // N = 2.;
+
+  if (mSimulateTimeResponse) {
+    // for each phase create a template distribution
+    TF1 RawResponse("RawResponse", rawResponseFunction, 0, 256, 5);
+    RawResponse.SetParameters(1., 0., tau, N, 0.);
+    RawResponse.SetParameter(1, 425. / o2::emcal::constants::EMCAL_TIMESAMPLE);
+
+    // parameter 1: Handling phase + delay
+    // phase: 25 ns * phase index (-4)
+    // delay: Average signal delay
+    RawResponse.SetParameter(1, mSimParam->getSignalDelay() / constants::EMCAL_TIMESAMPLE);
+    for (int sample = 0; sample < constants::EMCAL_MAXTIMEBINS; sample++) {
+      mAmplitudeInTimeBins[sample] = RawResponse.Eval(sample);
+      LOG(info) << "DIG TRU init in DigitizerTRU: amplitudes[" << sample << "] = " << mAmplitudeInTimeBins[sample];
+    }
+
+    // Rescale the TRU time response function
+    // so that the sum of the samples at [5], [6], [7] and [8]
+    // is equal to 1
+    auto maxElement = std::max_element(mAmplitudeInTimeBins.begin(), mAmplitudeInTimeBins.end());
+    double rescalingFactor = *(maxElement - 1) + *maxElement + *(maxElement + 1) + *(maxElement + 2);
+    double normalisationTRU = mSimParam->getTimeResponseNormalisationTRU();
+    rescalingFactor /= normalisationTRU;
+    // rescalingFactor /= (5. / 3.93);   // the slope seen in the single fastOr correlation
+    // rescalingFactor /= (45. / 39.25); // the slope seen in the single fastOr correlation
+    for (int sample = 0; sample < constants::EMCAL_MAXTIMEBINS; sample++) {
+      mAmplitudeInTimeBins[sample] /= rescalingFactor;
+      LOG(info) << "DIG TRU init in DigitizerTRU after RESCALING: amplitudes[" << sample << "] = " << mAmplitudeInTimeBins[sample];
+    }
+  } else {
+  }
+
+  if (mEnableDebugStreaming) {
+    mDebugStream = std::make_unique("emcaldigitsDebugTRU.root", "RECREATE");
+    // mDebugStreamPatch = std::make_unique("emcaldigitsDebugPatchTRU.root", "RECREATE");
+  }
+}
+//_______________________________________________________________________
+double DigitizerTRU::rawResponseFunction(double* x, double* par)
+{
+  double signal = 0.;
+  double tau = par[2];
+  double n = par[3];
+  double ped = par[4];
+  double xx = (x[0] - par[1] + tau) / tau;
+
+  // par[0] amp, par[1] peak time
+
+  if (xx <= 0) {
+    signal = ped;
+  } else {
+    signal = ped + par[0] * std::pow(xx, n) * std::exp(n * (1 - xx));
+  }
+
+  return signal;
+}
+//_______________________________________________________________________
+void DigitizerTRU::clear()
+{
+  mDigits.clear();
+}
+//_______________________________________________________________________
+void DigitizerTRU::process(const gsl::span summableDigits)
+{
+
+  auto processedSDigits = makeAnaloguesFastorSums(summableDigits);
+
+  for (auto vectorelement : processedSDigits) {
+
+    int& fastorID = std::get<0>(vectorelement);
+    auto& digit = std::get<1>(vectorelement);
+
+    int tower = digit.getTower();
+
+    sampleSDigit(digit);
+
+    if (mTempDigitVector.size() == 0) {
+      continue;
+    }
+
+    mDigits.addDigits(fastorID, mTempDigitVector);
+  }
+}
+//_______________________________________________________________________
+std::vector> DigitizerTRU::makeAnaloguesFastorSums(const gsl::span sdigits)
+{
+  TriggerMappingV2 mTriggerMap(mGeometry);
+  std::unordered_map sdigitsFastOR;
+  std::vector fastorIndicesFound;
+  for (const auto& dig : sdigits) {
+    o2::emcal::TriggerMappingV2::IndexCell towerid = dig.getTower();
+    int fastorIndex = mTriggerMap.getAbsFastORIndexFromCellIndex(towerid);
+    auto whichTRU = std::get<0>(mTriggerMap.getTRUFromAbsFastORIndex(fastorIndex));
+    auto whichFastOrTRU = std::get<1>(mTriggerMap.getTRUFromAbsFastORIndex(fastorIndex));
+    auto found = sdigitsFastOR.find(fastorIndex);
+    if (found != sdigitsFastOR.end()) {
+      // sum energy
+      Digit digitToSum((found->second).getTower(), dig.getAmplitude(), (found->second).getTimeStamp());
+      (found->second) += digitToSum;
+    } else {
+      // create new digit
+      fastorIndicesFound.emplace_back(fastorIndex);
+      sdigitsFastOR.emplace(fastorIndex, dig);
+    }
+  }
+  // sort digits for output
+  std::sort(fastorIndicesFound.begin(), fastorIndicesFound.end(), std::less<>());
+
+  for (auto& elem : sdigitsFastOR) {
+    auto dig = elem.second;
+    int fastorIndex = elem.first;
+    auto whichTRU = std::get<0>(mTriggerMap.getTRUFromAbsFastORIndex(fastorIndex));
+    auto whichFastOrTRU = std::get<1>(mTriggerMap.getTRUFromAbsFastORIndex(fastorIndex));
+  }
+
+  // Setting them to be TRU digits
+  for (auto& elem : sdigitsFastOR) {
+    (elem.second).setTRU();
+  }
+
+  for (auto& elem : sdigitsFastOR) {
+    auto dig = elem.second;
+    int fastorIndex = elem.first;
+    auto whichTRU = std::get<0>(mTriggerMap.getTRUFromAbsFastORIndex(fastorIndex));
+    auto whichFastOrTRU = std::get<1>(mTriggerMap.getTRUFromAbsFastORIndex(fastorIndex));
+  }
+
+  std::vector> outputFastorSDigits;
+  std::for_each(fastorIndicesFound.begin(), fastorIndicesFound.end(), [&outputFastorSDigits, &sdigitsFastOR](int fastorIndex) { outputFastorSDigits.emplace_back(fastorIndex, sdigitsFastOR[fastorIndex]); });
+  return outputFastorSDigits;
+}
+
+//_______________________________________________________________________
+void DigitizerTRU::sampleSDigit(const Digit& sDigit)
+{
+  mTempDigitVector.clear();
+  Int_t tower = sDigit.getTower();
+  Double_t energy = sDigit.getAmplitude();
+
+  if (mSmearEnergy) {
+    energy = smearEnergy(energy);
+  }
+
+  if (energy < __DBL_EPSILON__) {
+    return;
+  }
+
+  Double_t energies[15];
+  if (mSimulateTimeResponse) {
+    for (int sample = 0; sample < mAmplitudeInTimeBins.size(); sample++) {
+
+      double val = energy * (mAmplitudeInTimeBins[sample]);
+      energies[sample] = val;
+      double digitTime = mEventTimeOffset * constants::EMCAL_TIMESAMPLE;
+      Digit digit(tower, val, digitTime);
+      digit.setTRU();
+      mTempDigitVector.push_back(digit);
+    }
+
+  } else {
+    Digit digit(tower, energy, smearTime(sDigit.getTimeStamp(), energy));
+    digit.setTRU();
+    mTempDigitVector.push_back(digit);
+  }
+
+  if (mEnableDebugStreaming) {
+    double timeStamp = sDigit.getTimeStamp();
+    (*mDebugStream).GetFile()->cd();
+    (*mDebugStream) << "DigitsTimeSamples"
+                    << "Tower=" << tower
+                    << "Time=" << timeStamp
+                    << "DigitEnergy=" << energy
+                    << "Sample0=" << energies[0]
+                    << "Sample1=" << energies[1]
+                    << "Sample2=" << energies[2]
+                    << "Sample3=" << energies[3]
+                    << "Sample4=" << energies[4]
+                    << "Sample5=" << energies[5]
+                    << "Sample6=" << energies[6]
+                    << "Sample7=" << energies[7]
+                    << "Sample8=" << energies[8]
+                    << "Sample9=" << energies[9]
+                    << "Sample10=" << energies[10]
+                    << "Sample11=" << energies[11]
+                    << "Sample12=" << energies[12]
+                    << "Sample13=" << energies[13]
+                    << "Sample14=" << energies[14]
+                    << "\n";
+  }
+}
+
+//_______________________________________________________________________
+double DigitizerTRU::smearEnergy(double energy)
+{
+  auto mSimParam = &(o2::emcal::SimParam::Instance());
+  Double_t fluct = (energy * mSimParam->getMeanPhotonElectron()) / mSimParam->getGainFluctuations();
+  TRandom3 mRandomGenerator(std::chrono::high_resolution_clock::now().time_since_epoch().count());
+  energy *= mRandomGenerator.Poisson(fluct) / fluct;
+  return energy;
+}
+//_______________________________________________________________________
+double DigitizerTRU::smearTime(double time, double energy)
+{
+  auto mSimParam = &(o2::emcal::SimParam::Instance());
+  TRandom3 mRandomGenerator(std::chrono::high_resolution_clock::now().time_since_epoch().count());
+  return mRandomGenerator.Gaus(time + mSimParam->getSignalDelay(), mSimParam->getTimeResolution(energy));
+}
+
+//_______________________________________________________________________
+void DigitizerTRU::setEventTime(o2::InteractionTimeRecord record)
+{
+
+  // For the digitisation logic, at this time you would do:
+  // mDigits.forwardMarker(record);
+  // In the case of the trigger simulation, what is needed is to
+  // fill the corresponding LZEROElectronics object and
+  // launch the peak finder.
+  // If a trigger is found the logic is set to be live.
+  mDigits.fillOutputContainer(false, record, patchesFromAllTRUs, LZERO);
+  mEventTimeOffset = 0;
+
+  if (mEnableDebugStreaming) {
+    auto TriggerInputsAll = LZERO.getTriggerInputs();
+    auto TriggerInputsPatchesAll = LZERO.getTriggerInputsPatches();
+
+    std::vector TriggerInputs;
+    if (TriggerInputsAll.size() != mPreviousTriggerSize) {
+      mWasTriggerFound = true;
+      mPreviousTriggerSize = TriggerInputsAll.size();
+    } else {
+      mWasTriggerFound = false;
+    }
+    if (TriggerInputsAll.size() > 0 && mWasTriggerFound == true) {
+      TriggerInputs.push_back(TriggerInputsAll.back());
+    }
+    std::vector TriggerInputsPatches;
+    if (TriggerInputsPatchesAll.size() > 0 && mWasTriggerFound == true) {
+      TriggerInputsPatches.push_back(TriggerInputsPatchesAll.back());
+    }
+    int nIter = TriggerInputs.size();
+
+    if (nIter != 0) {
+      for (auto& trigger : TriggerInputs) {
+        auto InteractionRecordData = trigger.mInterRecord;
+        auto bc = InteractionRecordData.bc;
+        auto orbit = InteractionRecordData.orbit;
+        for (auto& fastor : trigger.mLastTimesumAllFastOrs) {
+          auto WhichTRU = std::get<0>(fastor);
+          auto WhichFastOr = std::get<1>(fastor);
+          auto FastOrAmp = std::get<2>(fastor);
+          (*mDebugStream).GetFile()->cd();
+          (*mDebugStream) << "L0Timesums"
+                          << "bc=" << bc
+                          << "orbit=" << orbit
+                          << "WhichTRU=" << WhichTRU
+                          << "WhichFastOr=" << WhichFastOr
+                          << "FastOrAmp=" << FastOrAmp
+                          << "\n";
+        }
+      }
+      // for (auto& trigger : TriggerInputsPatches) {
+      //   auto lastTimeSum = trigger.mLastTimesumAllPatches.end() - 1;
+      //   for (auto& patches : trigger.mLastTimesumAllPatches) {
+      //     auto WhichTRU = std::get<0>(patches);
+      //     auto WhichPatch = std::get<1>(patches);
+      //     auto PatchTimesum = std::get<2>(patches);
+      //     auto isFired = std::get<3>(patches);
+      //     (*mDebugStreamPatch).GetFile()->cd();
+      //     (*mDebugStreamPatch) << "L0TimesumsPatch"
+      //                          << "WhichTRU=" << WhichTRU
+      //                          << "WhichPatch=" << WhichPatch
+      //                          << "PatchTimesum=" << PatchTimesum
+      //                          << "isFired=" << isFired
+      //                          << "\n";
+      //   }
+      // }
+    }
+  }
+}
+//_______________________________________________________________________
+void DigitizerTRU::setPatches()
+{
+  // Using Run 2 geometry, found in:
+  // https://www.dropbox.com/s/pussj0olcctroim/PHOS+DCAL_STU.pdf?dl=0
+  // TRUs[0+6n, 1+6n, 2+6n] are on the A-side
+  // TRUs[3+6n, 4+6n, 5+6n] are on the C-side
+
+  patchesFromAllTRUs.clear();
+  // patchesFromAllTRUs.resize();
+  TRUElectronics FullAside(2, 0, 0);
+  TRUElectronics FullCside(2, 1, 0);
+  TRUElectronics ThirdAside(2, 0, 1);
+  TRUElectronics ThirdCside(2, 1, 1);
+  FullAside.init();
+  FullCside.init();
+  ThirdAside.init();
+  ThirdCside.init();
+
+  for (int j = 0; j < 3; j++) {
+    patchesFromAllTRUs.push_back(FullAside); // TRU ID 0,1,2    EMCAL A-side, Full
+  }
+  for (int j = 0; j < 3; j++) {
+    patchesFromAllTRUs.push_back(FullCside); // TRU ID 3,4,5    EMCAL C-side, Full
+  }
+  for (int j = 0; j < 3; j++) {
+    patchesFromAllTRUs.push_back(FullAside); // TRU ID 6,7,8    EMCAL A-side, Full
+  }
+  for (int j = 0; j < 3; j++) {
+    patchesFromAllTRUs.push_back(FullCside); // TRU ID 9,10,11  EMCAL C-side, Full
+  }
+  for (int j = 0; j < 3; j++) {
+    patchesFromAllTRUs.push_back(FullAside); // TRU ID 12,13,14 EMCAL A-side, Full
+  }
+  for (int j = 0; j < 3; j++) {
+    patchesFromAllTRUs.push_back(FullCside); // TRU ID 15,16,17 EMCAL C-side, Full
+  }
+  for (int j = 0; j < 3; j++) {
+    patchesFromAllTRUs.push_back(FullAside); // TRU ID 18,19,20 EMCAL A-side, Full
+  }
+  for (int j = 0; j < 3; j++) {
+    patchesFromAllTRUs.push_back(FullCside); // TRU ID 21,22,23 EMCAL C-side, Full
+  }
+  for (int j = 0; j < 3; j++) {
+    patchesFromAllTRUs.push_back(FullAside); // TRU ID 24,25,26 EMCAL A-side, Full
+  }
+  for (int j = 0; j < 3; j++) {
+    patchesFromAllTRUs.push_back(FullCside); // TRU ID 27,28,29 EMCAL C-side, Full
+  }
+  patchesFromAllTRUs.push_back(ThirdAside); // TRU ID 30 EMCAL A-side, Third
+  patchesFromAllTRUs.push_back(ThirdCside); // TRU ID 31 EMCAL C-side, Third
+  for (int j = 0; j < 3; j++) {
+    patchesFromAllTRUs.push_back(FullAside); // TRU ID 32,33,34 DCAL A-side, Full
+  }
+  for (int j = 0; j < 3; j++) {
+    patchesFromAllTRUs.push_back(FullCside); // TRU ID 35,36,37 DCAL C-side, Full
+  }
+  for (int j = 0; j < 3; j++) {
+    patchesFromAllTRUs.push_back(FullAside); // TRU ID 38,39,40 DCAL A-side, Full
+  }
+  for (int j = 0; j < 3; j++) {
+    patchesFromAllTRUs.push_back(FullCside); // TRU ID 41,42,43 DCAL C-side, Full
+  }
+  for (int j = 0; j < 3; j++) {
+    patchesFromAllTRUs.push_back(FullAside); // TRU ID 44,45,46 DCAL A-side, Full
+  }
+  for (int j = 0; j < 3; j++) {
+    patchesFromAllTRUs.push_back(FullCside); // TRU ID 47,48,49 DCAL C-side, Full
+  }
+  patchesFromAllTRUs.push_back(ThirdAside); // TRU ID 50 DCAL A-side, Third
+  patchesFromAllTRUs.push_back(ThirdCside); // TRU ID 51 DCAL C-side, Third
+
+  while (patchesFromAllTRUs[30].mPatchIDSeedFastOrIDs.size() > 69) {
+    patchesFromAllTRUs[30].mPatchIDSeedFastOrIDs.pop_back();
+    patchesFromAllTRUs[30].mIndexMapPatch.pop_back();
+  }
+  while (patchesFromAllTRUs[31].mPatchIDSeedFastOrIDs.size() > 69) {
+    patchesFromAllTRUs[31].mPatchIDSeedFastOrIDs.pop_back();
+    patchesFromAllTRUs[31].mIndexMapPatch.pop_back();
+  }
+  while (patchesFromAllTRUs[50].mPatchIDSeedFastOrIDs.size() > 69) {
+    patchesFromAllTRUs[50].mPatchIDSeedFastOrIDs.pop_back();
+    patchesFromAllTRUs[50].mIndexMapPatch.pop_back();
+  }
+  while (patchesFromAllTRUs[51].mPatchIDSeedFastOrIDs.size() > 69) {
+    patchesFromAllTRUs[51].mPatchIDSeedFastOrIDs.pop_back();
+    patchesFromAllTRUs[51].mIndexMapPatch.pop_back();
+  }
+}
+//______________________________________________________________________
+void DigitizerTRU::finish()
+{
+  mDigits.finish();
+  if (isDebugMode() == true) {
+    endDebugStream();
+  }
+}
+//______________________________________________________________________
+int DigitizerTRU::GetTRUIndexFromSTUIndex(Int_t id, Int_t detector)
+{
+  Int_t kEMCAL = 0;
+  Int_t kDCAL = 1;
+
+  if ((id > 31 && detector == kEMCAL) || (id > 13 && detector == kDCAL) || id < 0) {
+    return -1; // Error Condition
+  }
+
+  if (detector == kEMCAL) {
+    return id;
+  } else if (detector == kDCAL) {
+    return 32 + ((int)(id / 4) * 6) + ((id % 4 < 2) ? (id % 4) : (id % 4 + 2));
+  }
+  return -1;
+}
+//______________________________________________________________________
+int DigitizerTRU::GetChannelForMaskRun2(int mask, int bitnumber, bool onethirdsm)
+{
+  if (onethirdsm) {
+    return mask * 16 + bitnumber;
+  }
+  const int kChannelMap[6][16] = {{8, 9, 10, 11, 20, 21, 22, 23, 32, 33, 34, 35, 44, 45, 46, 47},    // Channels in mask0
+                                  {56, 57, 58, 59, 68, 69, 70, 71, 80, 81, 82, 83, 92, 93, 94, 95},  // Channels in mask1
+                                  {4, 5, 6, 7, 16, 17, 18, 19, 28, 29, 30, 31, 40, 41, 42, 43},      // Channels in mask2
+                                  {52, 53, 54, 55, 64, 65, 66, 67, 76, 77, 78, 79, 88, 89, 90, 91},  // Channels in mask3
+                                  {0, 1, 2, 3, 12, 13, 14, 15, 24, 25, 26, 27, 36, 37, 38, 39},      // Channels in mask4
+                                  {48, 49, 50, 51, 60, 61, 62, 63, 72, 73, 74, 75, 84, 85, 86, 87}}; // Channels in mask5
+  return kChannelMap[mask][bitnumber];
+}
+//______________________________________________________________________
+std::vector DigitizerTRU::GetAbsFastORIndexFromMask()
+{
+  TriggerMappingV2 mTriggerMap(mGeometry);
+  std::vector maskedfastors;
+  int itru = 0;
+  for (Int_t i = 0; i < 46; i++) {
+    int localtru = itru % 32, detector = itru >= 32 ? 1 : 0,
+        globaltru = GetTRUIndexFromSTUIndex(localtru, detector);
+    bool onethirdsm = ((globaltru >= 30 && globaltru < 32) || (globaltru >= 50 && globaltru < 52));
+    for (int ipos = 0; ipos < 6; ipos++) {
+      auto regmask = mFeeDCS->getTRUDCS(i).getMaskReg(ipos);
+      std::bitset<16> bitsregmask(regmask);
+      for (int ibit = 0; ibit < 16; ibit++) {
+        if (bitsregmask.test(ibit)) {
+          auto channel = GetChannelForMaskRun2(ipos, ibit, onethirdsm);
+          int absfastor = mTriggerMap.getAbsFastORIndexFromIndexInTRU(globaltru, channel);
+          maskedfastors.push_back(absfastor);
+        }
+      }
+    }
+    itru++;
+  }
+  return maskedfastors;
+}
+//______________________________________________________________________
+void DigitizerTRU::setMaskedFastOrsInLZERO()
+{
+  auto maskedFastOrs = GetAbsFastORIndexFromMask();
+  LOG(info) << "======================================";
+  LOG(info) << "== PRINT MASK COMPUTED IN DIGITIZER ==";
+  int counter = 0;
+  for (auto fastOr : maskedFastOrs) {
+    LOG(info) << "fastOr masked (number, ID) = (" << counter << ", " << fastOr;
+    counter += 1;
+  }
+  LZERO.setMaskedFastOrs(maskedFastOrs);
+}
+//______________________________________________________________________
+void DigitizerTRU::printMaskedFastOrsInLZERO()
+{
+  LZERO.printMaskedFastOrs();
+}
diff --git a/Detectors/EMCAL/simulation/src/DigitsVectorStream.cxx b/Detectors/EMCAL/simulation/src/DigitsVectorStream.cxx
index 2b6e6c1dd4a00..f0ed5d6d9d68a 100644
--- a/Detectors/EMCAL/simulation/src/DigitsVectorStream.cxx
+++ b/Detectors/EMCAL/simulation/src/DigitsVectorStream.cxx
@@ -15,18 +15,22 @@
 #include 
 #include 
 #include 
-#include "FairLogger.h"
+#include 
 #include "EMCALSimulation/LabeledDigit.h"
 #include "EMCALSimulation/DigitsVectorStream.h"
 #include "CommonConstants/Triggers.h"
+#include "SimConfig/DigiParams.h"
 
 using namespace o2::emcal;
 
 void DigitsVectorStream::init()
 {
   mSimParam = &(o2::emcal::SimParam::Instance());
-
-  mRandomGenerator = new TRandom3(std::chrono::high_resolution_clock::now().time_since_epoch().count());
+  auto randomSeed = std::chrono::high_resolution_clock::now().time_since_epoch().count();
+  if (o2::conf::DigiParams::Instance().seed != 0) {
+    randomSeed = o2::conf::DigiParams::Instance().seed;
+  }
+  mRandomGenerator = new TRandom3(randomSeed);
 
   mRemoveDigitsBelowThreshold = mSimParam->doRemoveDigitsBelowThreshold();
   mSimulateNoiseDigits = mSimParam->doSimulateNoiseDigits();
@@ -47,7 +51,6 @@ void DigitsVectorStream::addNoiseDigits(LabeledDigit& d1)
 
   d1 += d;
 }
-
 //_______________________________________________________________________
 void DigitsVectorStream::fill(std::deque& digitlist, o2::InteractionRecord record)
 {
@@ -121,7 +124,7 @@ void DigitsVectorStream::fill(std::deque& digitlist, o2
   mTriggerRecords.emplace_back(record, o2::trigger::PhT, mStartIndex, numberOfNewDigits);
   mStartIndex = mDigits.size();
 
-  LOG(info) << "Have " << mStartIndex << " digits ";
+  LOG(info) << "Trigger Orbit " << record.orbit << ", BC " << record.bc << ": have " << numberOfNewDigits << " digits (" << mStartIndex << " total)";
 }
 
 //_______________________________________________________________________
diff --git a/Detectors/EMCAL/simulation/src/DigitsWriteoutBuffer.cxx b/Detectors/EMCAL/simulation/src/DigitsWriteoutBuffer.cxx
index d4eee391a5d6b..c5c17f4c41d57 100644
--- a/Detectors/EMCAL/simulation/src/DigitsWriteoutBuffer.cxx
+++ b/Detectors/EMCAL/simulation/src/DigitsWriteoutBuffer.cxx
@@ -19,6 +19,7 @@
 #include "EMCALSimulation/DigitsWriteoutBuffer.h"
 #include "CommonDataFormat/InteractionRecord.h"
 #include "TMath.h"
+#include 
 
 using namespace o2::emcal;
 
@@ -36,7 +37,7 @@ void DigitsWriteoutBuffer::init()
   mLiveTime = simParam->getLiveTime();
   mBusyTime = simParam->getBusyTime();
   mPreTriggerTime = simParam->getPreTriggerTime();
-
+  mSwapPhase = simParam->getBCPhaseSwap();
   mDigitStream.init();
 }
 
@@ -80,7 +81,7 @@ void DigitsWriteoutBuffer::addDigits(unsigned int towerID, std::vector= (mLiveTime + mBusyTime) || mFirstEvent) {
+  if (trigger && ((eventTime - mTriggerTime) >= (mLiveTime + mBusyTime) || mFirstEvent)) {
+    LOG(debug) << "Trigger received and accepted";
     mTriggerTime = eventTime;
     mTimedDigitsFuture.front().mTriggerColl = true;
     mTimedDigitsFuture.front().mInterRecord = record;
@@ -142,8 +144,8 @@ void DigitsWriteoutBuffer::forwardMarker(o2::InteractionTimeRecord record)
   }
 
   // If we have a pre-trigger collision, all the time bins in the future buffer will be set to record mode
-  if ((eventTime - mTriggerTime) >= (mLiveTime + mBusyTime - mPreTriggerTime)) {
-    std::cout << "Pre-trigger collision\n";
+  if ((eventTime - mTriggerTime) >= (mLiveTime + mBusyTime - mPreTriggerTime) || mFirstEvent) {
+    LOG(debug) << "Pre-trigger collision";
     long timeStamp = (eventTime / 100) * 100; /// This is to make the event time multiple of 100s
     for (auto& iNode : mTimedDigitsFuture) {
       long diff = (timeStamp - eventTime);
@@ -156,7 +158,9 @@ void DigitsWriteoutBuffer::forwardMarker(o2::InteractionTimeRecord record)
   }
 
   mLastEventTime = eventTime;
-  mPhase = ((int)(std::fmod(mLastEventTime, 100) / 25));
+  // mPhase = ((int)(std::fmod(mLastEventTime, 100) / 25));
+  mPhase = (record.bc + mSwapPhase) % 4;
+
   if (mFirstEvent) {
     mFirstEvent = false;
   }
diff --git a/Detectors/EMCAL/simulation/src/DigitsWriteoutBufferTRU.cxx b/Detectors/EMCAL/simulation/src/DigitsWriteoutBufferTRU.cxx
new file mode 100644
index 0000000000000..8b642b2c767eb
--- /dev/null
+++ b/Detectors/EMCAL/simulation/src/DigitsWriteoutBufferTRU.cxx
@@ -0,0 +1,159 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include "EMCALSimulation/DigitsWriteoutBufferTRU.h"
+#include "EMCALSimulation/LZEROElectronics.h"
+#include "CommonDataFormat/InteractionRecord.h"
+#include  // for LOG
+#include "EMCALBase/TriggerMappingV2.h"
+
+using namespace o2::emcal;
+
+//_____________________________________________________________________
+//
+void DigitsWriteoutBufferTRU::fillOutputContainer(bool isEndOfTimeFrame, InteractionRecord& nextInteractionRecord, std::vector& patchesFromAllTRUs, LZEROElectronics& LZERO)
+{
+  int eventTimeBin = 13;
+  bool needsEmptyTimeBins = false;
+  int nProcessedTimeBins = 0;
+
+  std::deque
+    mDequeTime;
+
+  // If end of run or end of timeframe read out what we have
+  bool isEnd = mEndOfRun || isEndOfTimeFrame;
+  // Checking the Interaction Record for the time of the next event in BC units
+  // Difference becomes the new marker if the collision happens before 13 samples
+  auto difference = nextInteractionRecord.toLong() - mCurrentInteractionRecord.toLong();
+  if (mNoPileupMode || difference >= 13 || isEnd) {
+    // Next collision happening way after the current one, just
+    // send out and clear entire buffer
+    // Simplification and optimization at software level - hardware would continuosly sample, but here we don't need to allocate memory dynamically which we are never going to use
+    // Also in a dedicated no-pileup mode we always write out after each collision.
+    for (auto& time : mTimeBins) {
+      mDequeTime.push_back(time);
+    }
+    LZERO.fill(mDequeTime, mCurrentInteractionRecord, patchesFromAllTRUs);
+    auto TriggerInputs = LZERO.getTriggerInputs();
+    auto TriggerInputsPatches = LZERO.getTriggerInputsPatches();
+    int nIter = TriggerInputs.size();
+    LOG(debug) << "DIG TRU fillOutputContainer in DigitsWriteoutBufferTRU: size of TriggerInputs = " << nIter;
+    // for (auto& patches : patchesFromAllTRUs) {
+    //   LZERO.updatePatchesADC(patches);
+    //   LZERO.peakFinderOnAllPatches(patches);
+    // }
+    LOG(debug) << "DIG TRU fillOutputContainer in DigitsWriteoutBufferTRU: LOW IR";
+
+    mCurrentInteractionRecord = nextInteractionRecord;
+    clear();
+
+  } else {
+    // Next collsions happing in the tail of the current one,
+    // Copy out up to difference and pop and push this amount
+    // of samples
+
+    // Now filling the vector stream
+    for (auto& time : mTimeBins) {
+      if (!(nProcessedTimeBins + mFirstTimeBin < difference)) {
+        break;
+      }
+
+      mDequeTime.push_back(time);
+
+      ++nProcessedTimeBins;
+    }
+
+    LZERO.fill(mDequeTime, mCurrentInteractionRecord, patchesFromAllTRUs);
+    auto TriggerInputs = LZERO.getTriggerInputs();
+    auto TriggerInputsPatches = LZERO.getTriggerInputsPatches();
+    int nIter = TriggerInputs.size();
+    LOG(debug) << "DIG TRU fillOutputContainer in DigitsWriteoutBufferTRU: size of TriggerInputs = " << nIter;
+    // for (auto& patches : patchesFromAllTRUs) {
+    //   LZERO.updatePatchesADC(patches);
+    //   LZERO.peakFinderOnAllPatches(patches);
+    // }
+    mCurrentInteractionRecord = nextInteractionRecord;
+
+    if (nProcessedTimeBins > 0) {
+      mFirstTimeBin += nProcessedTimeBins;
+      while (nProcessedTimeBins--) {
+        mTimeBins.pop_front(); // deque
+      }
+    }
+    reserve(15);
+
+    LOG(debug) << "DIG TRU fillOutputContainer in DigitsWriteoutBufferTRU: HIGH IR";
+  }
+}
+//________________________________________________________
+// Constructor: reserves space to keep always a minimum buffer size
+DigitsWriteoutBufferTRU::DigitsWriteoutBufferTRU(unsigned int nTimeBins) : mBufferSize(nTimeBins)
+{
+  for (int itime = 0; itime < nTimeBins; itime++) {
+    mTimeBins.push_back(o2::emcal::DigitTimebinTRU());
+  }
+}
+//________________________________________________________
+// Reserve space to keep always a minimum buffer size (put it in the cpp)
+void DigitsWriteoutBufferTRU::reserve(int eventTimeBin)
+{
+  const auto space = mBufferSize + eventTimeBin - mFirstTimeBin;
+  if (mTimeBins.size() < space) {
+    mTimeBins.resize(space);
+  }
+}
+//________________________________________________________
+void DigitsWriteoutBufferTRU::init()
+{
+  const SimParam* simParam = &(o2::emcal::SimParam::Instance());
+
+  mLiveTime = simParam->getLiveTime();
+  mBusyTime = simParam->getBusyTime();
+  mNoPileupMode = simParam->isDisablePileup();
+  mEndOfRun = 0;
+
+  // mDigitStream.init();
+}
+//________________________________________________________
+void DigitsWriteoutBufferTRU::clear()
+{
+  mTimeBins.clear();
+  reserve(15);
+  // mEndOfRun = 0;
+}
+//________________________________________________________
+void DigitsWriteoutBufferTRU::finish()
+{
+  mEndOfRun = 1;
+}
+//________________________________________________________
+// Add digits to the buffer
+void DigitsWriteoutBufferTRU::addDigits(unsigned int towerID, std::vector& digList)
+{
+  // mTimeBin has to have the absolute time information
+  for (int ientry = 0; ientry < digList.size(); ientry++) {
+
+    auto& buffEntry = mTimeBins[ientry];
+    auto& dig = digList.at(ientry);
+
+    auto towerEntry = buffEntry.mDigitMap->find(towerID);
+    if (towerEntry == buffEntry.mDigitMap->end()) {
+      towerEntry = buffEntry.mDigitMap->insert(std::pair>(towerID, std::list())).first;
+    }
+    towerEntry->second.push_back(dig);
+  }
+}
diff --git a/Detectors/EMCAL/simulation/src/EMCALSimulationLinkDef.h b/Detectors/EMCAL/simulation/src/EMCALSimulationLinkDef.h
index 05b27b61542d5..d070aa2470177 100644
--- a/Detectors/EMCAL/simulation/src/EMCALSimulationLinkDef.h
+++ b/Detectors/EMCAL/simulation/src/EMCALSimulationLinkDef.h
@@ -20,11 +20,26 @@
 #pragma link C++ class o2::emcal::Digitizer + ;
 #pragma link C++ class o2::emcal::SDigitizer + ;
 #pragma link C++ class o2::emcal::DigitsWriteoutBuffer + ;
+#pragma link C++ class o2::emcal::DigitsWriteoutBufferTRU + ;
 #pragma link C++ class o2::emcal::DigitsVectorStream + ;
+#pragma link C++ class o2::emcal::LZEROElectronics + ;
+#pragma link C++ class o2::emcal::TRUElectronics + ;
+#pragma link C++ class o2::emcal::FastOrStruct + ;
+#pragma link C++ class o2::emcal::EMCALTriggerInputs + ;
+#pragma link C++ class o2::emcal::EMCALTriggerInputsPatch + ;
+#pragma link C++ class o2::emcal::DigitizerTRU + ;
+// #pragma link C++ class o2::emcal::DigitizerTRU + ;
+#pragma link C++ class o2::emcal::DigitTimebinBase < o2::emcal::Digit> + ;
+#pragma link C++ class o2::emcal::DigitTimebinBase < o2::emcal::LabeledDigit> + ;
 #pragma link C++ class o2::emcal::SimParam + ;
+#pragma link C++ class o2::conf::ConfigurableParamHelper < o2::emcal::SimParam> + ;
 #pragma link C++ class o2::emcal::LabeledDigit + ;
 #pragma link C++ class o2::emcal::RawWriter + ;
 
 #pragma link C++ class std::list < o2::emcal::LabeledDigit> + ;
+#pragma link C++ class std::vector < o2::emcal::LabeledDigit> + ;
+#pragma link C++ class std::vector < o2::emcal::FastOrStruct> + ;
+#pragma link C++ class std::vector < o2::emcal::EMCALTriggerInputs> + ;
+#pragma link C++ class std::vector < o2::emcal::EMCALTriggerInputsPatch> + ;
 
 #endif
diff --git a/Detectors/EMCAL/simulation/src/LZEROElectronics.cxx b/Detectors/EMCAL/simulation/src/LZEROElectronics.cxx
new file mode 100644
index 0000000000000..672c9c5b1ca89
--- /dev/null
+++ b/Detectors/EMCAL/simulation/src/LZEROElectronics.cxx
@@ -0,0 +1,252 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include "EMCALSimulation/LabeledDigit.h"
+#include "EMCALSimulation/LZEROElectronics.h"
+#include "CommonDataFormat/InteractionRecord.h"
+#include "EMCALSimulation/DigitTimebin.h"
+#include "EMCALBase/TriggerMappingV2.h"
+#include "TMath.h"
+#include  // for LOG
+
+using namespace o2::emcal;
+
+//_____________________________________________________________________
+// Peak finding algorithm
+//
+// It checks if there is a rising trend on four consecutive Timebins
+// If yes, then it compares the integral to a programmable threshold
+bool LZEROElectronics::peakFinderOnPatch(TRUElectronics& p, unsigned int patchID)
+{
+  auto& CurrentPatchTimeSum = p.mTimesum[patchID];
+  auto& TimeSums = std::get<1>(CurrentPatchTimeSum);
+  bool trendOfDigitsInTower = false;
+  if (TimeSums.size() < 4) {
+    return false;
+    // } else if (TimeSums[0] < TimeSums[1] && TimeSums[1] < TimeSums[2] && TimeSums[2] >= TimeSums[3]) {
+  } else if (TimeSums[3] < TimeSums[2] && TimeSums[2] < TimeSums[1] && TimeSums[1] >= TimeSums[0]) {
+    trendOfDigitsInTower = true;
+  } else if (TimeSums[0] > 0. || TimeSums[1] > 0. || TimeSums[2] > 0. || TimeSums[3] > 0.) {
+  }
+  double integralOfADCvalues = 0;
+  for (auto it = TimeSums.begin(); it != TimeSums.end(); it++) {
+    integralOfADCvalues += *it;
+  }
+  bool peakOverThreshold = false;
+  if (integralOfADCvalues > mThreshold && trendOfDigitsInTower) {
+    peakOverThreshold = true;
+  }
+  return peakOverThreshold;
+}
+//_____________________________________________________________________
+// Peak finding algorithm on all patches
+// It fills the mPeakFound vector with potential 1s
+bool LZEROElectronics::peakFinderOnAllPatches(TRUElectronics& p)
+{
+  bool isFoundGlobal = false;
+  p.mFiredPatches.clear();
+  for (auto& patches : p.mIndexMapPatch) {
+    auto PatchID = std::get<0>(patches);
+    auto isFound = peakFinderOnPatch(p, PatchID);
+    if (isFound) {
+      p.mFiredPatches.push_back(PatchID);
+      isFoundGlobal = true;
+    }
+  }
+  return isFoundGlobal;
+}
+//________________________________________________________
+void LZEROElectronics::init()
+{
+  auto mSimParam = &(o2::emcal::SimParam::Instance());
+  mSimulateNoiseDigits = mSimParam->doSimulateNoiseDigits();
+  setThreshold(mSimParam->getThresholdLZERO());
+  // setThreshold(132.);
+}
+//________________________________________________________
+void LZEROElectronics::clear()
+{
+}
+//________________________________________________________
+void LZEROElectronics::updatePatchesADC(TRUElectronics& p)
+{
+  p.updateADC();
+}
+//_______________________________________________________________________
+void LZEROElectronics::addNoiseDigits(Digit& d1)
+{
+  auto mSimParam = &(o2::emcal::SimParam::Instance());
+  double amplitude = d1.getAmplitude();
+  double sigma = mSimParam->getPinNoiseTRU();
+
+  TRandom3 mRandomGenerator(std::chrono::high_resolution_clock::now().time_since_epoch().count());
+  uint16_t noise = std::floor(std::abs(mRandomGenerator.Gaus(0, sigma) / constants::EMCAL_TRU_ADCENERGY)); // ADC
+
+  Digit d(d1.getTower(), 0., noise, d1.getTimeStamp());
+
+  d1 += d;
+}
+//_______________________________________________________________________
+void LZEROElectronics::fill(const std::deque& digitlist, const o2::InteractionRecord record, std::vector& patchesFromAllTRUs)
+{
+  int counterDigitTimeBin = 0;
+  int sizemDigitMap = -999;
+  TriggerMappingV2 mTriggerMap(mGeometry);
+
+  for (auto& digitsTimeBin : digitlist) {
+    // Inside the DigitTimebinTRU
+    // Fill the LZEROElectronics with the new ADC value
+    // At the end of the loop run the peak finder
+    // Ship to LONEElectronics in case a peak is found
+    // Entire logic limited to timebin by timebin -> effectively implementing time scan
+    counterDigitTimeBin++;
+
+    for (auto& [fastor, digitsList] : *digitsTimeBin.mDigitMap) {
+      // Digit loop
+      // The peak finding algorithm is run after getting out of the loop!
+      if (digitsList.size() == 0) {
+        continue;
+      }
+      digitsList.sort();
+
+      int digIndex = 0;
+      Digit summedDigit;
+      bool first = true;
+      for (auto& ld : digitsList) {
+        if (first) {
+          summedDigit = ld;
+          first = false;
+        } else {
+          // summedDigit += ld;
+
+          // safety device in case same fastOr
+          // but different towers, i.e. remember
+          // that the += operator fails WITHOUT
+          // feedback if that were to happen
+          Digit digitToSum(summedDigit.getTower(), ld.getAmplitude(), summedDigit.getTimeStamp());
+          summedDigit += digitToSum;
+        }
+      }
+
+      sizemDigitMap = (*digitsTimeBin.mDigitMap).size();
+      if (mSimulateNoiseDigits) {
+        addNoiseDigits(summedDigit);
+      }
+
+      auto [whichTRU, whichFastOrTRU] = mTriggerMap.getTRUFromAbsFastORIndex(fastor);
+
+      auto whichFastOr = std::get<1>(mTriggerMap.convertFastORIndexTRUtoSTU(whichTRU, whichFastOrTRU));
+      auto& patchTRU = patchesFromAllTRUs[whichTRU];
+      auto& fastOrPatchTRU = patchTRU.mFastOrs[whichFastOr];
+      // fastOrPatchTRU.updateADC(summedDigit.getAmplitudeADC());
+      if (std::find(mMaskedFastOrs.begin(), mMaskedFastOrs.end(), fastor) != mMaskedFastOrs.end()) {
+        fastOrPatchTRU.updateADC(summedDigit.getAmplitudeADC());
+      }
+
+      digIndex++;
+    }
+
+    // Evaluate -> peak finder (ALL TRUElectronics in ALL TRUs)
+    // in case peak found:
+    // - Create trigger input (IR of that timebin - delay [typically 8 or 9 samples - rollback (0)])
+    // - Create L1 timesums (trivial - last time integral) -> Collect from all fastOrs in ALL TRUs
+
+    // Trigger Inputs needs to have the correction of the delay
+    // Propagating for now all the interaction record
+    // The typical delay is 8 to 9 BCs and the rollback as well
+    // The rollback is taken from EMCALReconstruction/RecoParam.h
+    // The delay is due to the interactions between EMCAL and CTP
+    // It accounts for the difference in times between L0a, L0b, and then there will be a L1 and L1b delay
+    // There is 1BC uncertainty on the trigger readout due to steps in the interaction between CTP and detector simulations
+    bool foundPeak = false;
+    int counterWhichTRU = 0;
+    int triggeredTRU = -1;
+    std::vector triggeredPatches;
+    for (auto& patches : patchesFromAllTRUs) {
+      updatePatchesADC(patches);
+      bool foundPeakCurrentTRU = peakFinderOnAllPatches(patches);
+      auto firedPatches = getFiredPatches(patches);
+      if (foundPeakCurrentTRU == true && foundPeak == false) {
+        triggeredTRU = counterWhichTRU;
+        triggeredPatches = firedPatches;
+      }
+      if (foundPeakCurrentTRU) {
+        foundPeak = true;
+      }
+      counterWhichTRU += 1;
+    }
+
+    if (foundPeak == true) {
+      LOG(debug) << "DIG TRU fill in LZEROElectronics: foundPeak = " << foundPeak;
+    }
+    EMCALTriggerInputs TriggerInputsForL1;
+    if (foundPeak) {
+      TriggerInputsForL1.mInterRecord = record;
+      TriggerInputsForL1.mTriggeredTRU = triggeredTRU;
+      TriggerInputsForL1.mTriggeredPatches = triggeredPatches;
+      int whichTRU = 0;
+      for (auto& patches : patchesFromAllTRUs) {
+        int whichFastOr = 0;
+        for (auto& fastor : patches.mFastOrs) {
+          TriggerInputsForL1.mLastTimesumAllFastOrs.push_back(std::make_tuple(whichTRU, std::get<1>(mTriggerMap.convertFastORIndexSTUtoTRU(mTriggerMap.convertTRUIndexTRUtoSTU(whichTRU), whichFastOr, o2::emcal::TriggerMappingV2::DetType_t::DET_EMCAL)), fastor.timesum()));
+          whichFastOr++;
+        }
+        whichTRU++;
+      }
+    }
+
+    // EMCALTriggerInputsPatch TriggerInputsPatch;
+    // if (foundPeak) {
+    //   TriggerInputsPatch.mInterRecord = record;
+    //   int whichTRU = 0;
+    //   for (auto& patches : patchesFromAllTRUs) {
+    //     if(whichTRU < 46){
+    //     int whichPatch = 0;
+    //     bool firedpatch = false;
+    //     if(std::find(patches.mFiredPatches.begin(), patches.mFiredPatches.end(), whichPatch) != patches.mFiredPatches.end()){
+    //       firedpatch = true;
+    //     }
+    //     for (auto& patchTimeSums : patches.mTimesum) {
+    //       auto& CurrentPatchTimesum = std::get<1>(patchTimeSums);
+    //       // if( whichTRU == 30 || whichTRU == 31 || whichTRU == 44 || whichTRU == 45 ){
+    //       //   if( whichPatch > 68) continue;
+    //       // }
+    //       TriggerInputsPatch.mLastTimesumAllPatches.push_back(std::make_tuple(whichTRU, whichPatch, CurrentPatchTimesum[3], firedpatch ));
+    //       whichPatch++;
+    //     }
+    //     }
+    //     whichTRU++;
+    //   }
+    // }
+
+    if (foundPeak) {
+      mTriggers.push_back(TriggerInputsForL1);
+      // mTriggersPatch.push_back(TriggerInputsPatch);
+    }
+  }
+}
+//________________________________________________________
+void LZEROElectronics::printMaskedFastOrs()
+{
+  LOG(info) << "===============================";
+  LOG(info) << "== PRINT MASK SAVED IN LZERO ==";
+  int counter = 0;
+  for (auto fastOr : mMaskedFastOrs) {
+    LOG(info) << "fastOr masked (number, ID) = (" << counter << ", " << fastOr;
+    counter += 1;
+  }
+}
diff --git a/Detectors/EMCAL/simulation/src/RawCreator.cxx b/Detectors/EMCAL/simulation/src/RawCreator.cxx
index 916184d9ba8de..386602e32b91c 100644
--- a/Detectors/EMCAL/simulation/src/RawCreator.cxx
+++ b/Detectors/EMCAL/simulation/src/RawCreator.cxx
@@ -13,7 +13,7 @@
 #include 
 #include 
 #include "Framework/Logger.h"
-#include "FairLogger.h"
+#include 
 
 #include 
 
@@ -73,7 +73,7 @@ int main(int argc, const char** argv)
 
   auto debuglevel = vm["debug"].as();
   if (debuglevel > 0) {
-    FairLogger::GetLogger()->SetLogScreenLevel("DEBUG");
+    fair::Logger::SetConsoleSeverity("DEBUG");
   }
 
   std::string confDig = vm["hbfutils-config"].as();
@@ -122,7 +122,7 @@ int main(int argc, const char** argv)
   rawwriter.init();
 
   // Loop over all entries in the tree, where each tree entry corresponds to a time frame
-  for (auto en : *treereader) {
+  while (treereader->Next()) {
     rawwriter.digitsToRaw(*digitbranch, *triggerbranch);
   }
   rawwriter.getWriter().writeConfFile("EMC", "RAWDATA", o2::utils::Str::concat_string(outputdir, "/EMCraw.cfg"));
diff --git a/Detectors/EMCAL/simulation/src/RawWriter.cxx b/Detectors/EMCAL/simulation/src/RawWriter.cxx
index 04e3789fd4f2e..14c32c0e299ae 100644
--- a/Detectors/EMCAL/simulation/src/RawWriter.cxx
+++ b/Detectors/EMCAL/simulation/src/RawWriter.cxx
@@ -9,7 +9,9 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 
-#include "FairLogger.h"
+#include 
+
+#include 
 
 #include 
 #include 
@@ -127,12 +129,39 @@ bool RawWriter::processTrigger(const o2::emcal::TriggerRecord& trg)
       continue;
     }
 
+    // sort found towers according to FEC inside
+    // within the FEC channels are also sorted according
+    // their local channel ID
+    std::map> fecSortedTowersWithSignal;
+    auto& mappingDDL = mMappingHandler->getMappingForDDL(srucont.mSRUid);
     for (const auto& [tower, channel] : srucont.mChannels) {
 
-      bool saturatedBunchHG = false;
-      createPayload(channel, ChannelType_t::HIGH_GAIN, srucont.mSRUid, payload, saturatedBunchHG);
-      if (saturatedBunchHG) {
-        createPayload(channel, ChannelType_t::LOW_GAIN, srucont.mSRUid, payload, saturatedBunchHG);
+      auto hwaddress = mappingDDL.getHardwareAddress(channel.mRow, channel.mCol, ChannelType_t::HIGH_GAIN);
+      auto fecInDLL = getBranchIndexFromHwAddress(hwaddress) * 10 + getFecIndexFromHwAddress(hwaddress);
+      auto channelID = getChannelIndexFromHwAddress(hwaddress);
+      auto fecFound = fecSortedTowersWithSignal.find(fecInDLL);
+      if (fecFound != fecSortedTowersWithSignal.end()) {
+        fecFound->second[channelID] = tower;
+      } else {
+        std::map channelsInFec;
+        channelsInFec[channelID] = tower;
+        fecSortedTowersWithSignal[fecInDLL] = channelsInFec;
+      }
+    }
+
+    // encode payload for sorted channels
+    for (const auto& [fec, channelsInFec] : fecSortedTowersWithSignal) {
+      for (auto [channelID, tower] : channelsInFec) {
+        auto towerChannel = srucont.mChannels.find(tower);
+        if (towerChannel != srucont.mChannels.end()) {
+          bool saturatedBunchHG = false;
+          createPayload(towerChannel->second, ChannelType_t::HIGH_GAIN, srucont.mSRUid, payload, saturatedBunchHG);
+          if (saturatedBunchHG) {
+            createPayload(towerChannel->second, ChannelType_t::LOW_GAIN, srucont.mSRUid, payload, saturatedBunchHG);
+          }
+        } else {
+          LOG(error) << "No data found for FEC " << fec << ", channel " << channelID << "(tower " << tower << ")";
+        }
       }
     }
 
diff --git a/Detectors/EMCAL/simulation/src/SDigitizer.cxx b/Detectors/EMCAL/simulation/src/SDigitizer.cxx
index e62aa75267844..cea6792438785 100644
--- a/Detectors/EMCAL/simulation/src/SDigitizer.cxx
+++ b/Detectors/EMCAL/simulation/src/SDigitizer.cxx
@@ -21,7 +21,7 @@
 #include 
 #include 
 #include 
-#include "FairLogger.h" // for LOG
+#include  // for LOG
 
 ClassImp(o2::emcal::SDigitizer);
 
diff --git a/Detectors/EMCAL/simulation/src/SimParam.cxx b/Detectors/EMCAL/simulation/src/SimParam.cxx
index 5dfa348476c8f..818ecda3ecb58 100644
--- a/Detectors/EMCAL/simulation/src/SimParam.cxx
+++ b/Detectors/EMCAL/simulation/src/SimParam.cxx
@@ -17,7 +17,7 @@ O2ParamImpl(o2::emcal::SimParam);
 
 using namespace o2::emcal;
 
-std::ostream& operator<<(std::ostream& stream, const o2::emcal::SimParam& s)
+std::ostream& o2::emcal::operator<<(std::ostream& stream, const o2::emcal::SimParam& s)
 {
   s.PrintStream(stream);
   return stream;
@@ -30,15 +30,15 @@ void SimParam::PrintStream(std::ostream& stream) const
   stream << "\nEMCal::SimParam.mGainFluctuations = " << mGainFluctuations;
   stream << "\nEMCal::SimParam.mPinNoise = " << mPinNoise;
   stream << "\nEMCal::SimParam.mPinNoiseLG = " << mPinNoiseLG;
-  stream << "\nEMCal::SimParam.mTimeNoise = " << mTimeNoise;
-  stream << "\nEMCal::SimParam.mTimeDelay = " << mTimeDelay;
-  stream << "\nEMCal::SimParam.mTimeDelayFromOCDB = " << ((mTimeDelayFromOCDB) ? "true" : "false");
+  stream << "\nEMCal::SimParam.mPinNoiseTRU = " << mPinNoiseTRU;
   stream << "\nEMCal::SimParam.mTimeResolutionPar0 = " << mTimeResolutionPar0;
   stream << "\nEMCal::SimParam.mTimeResolutionPar1 = " << mTimeResolutionPar1;
   stream << "\nEMCal::SimParam.mTimeResponseTau = " << mTimeResponseTau;
   stream << "\nEMCal::SimParam.mTimeResponsePower = " << mTimeResponsePower;
   stream << "\nEMCal::SimParam.mTimeResponseThreshold = " << mTimeResponseThreshold;
+  stream << "\nEMCal::SimParam.mThresholdLZERO = " << mThresholdLZERO;
   stream << "\nEMCal::SimParam.mNADCEC = " << mNADCEC;
+  stream << "\nEMCal::SimParam.mSwapPhase = " << mSwapPhase;
   stream << "\nEMCal::SimParam.mA = " << mA;
   stream << "\nEMCal::SimParam.mB = " << mB;
   stream << "\nEMCal::SimParam.mECPrimThreshold = " << mECPrimThreshold;
diff --git a/Detectors/EMCAL/simulation/src/SpaceFrame.cxx b/Detectors/EMCAL/simulation/src/SpaceFrame.cxx
index f2f1b683a84bc..7a45eff0adf50 100644
--- a/Detectors/EMCAL/simulation/src/SpaceFrame.cxx
+++ b/Detectors/EMCAL/simulation/src/SpaceFrame.cxx
@@ -9,7 +9,7 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 
-#include 
+#include 
 #include 
 #include 
 #include 
diff --git a/Detectors/EMCAL/simulation/src/TRUElectronics.cxx b/Detectors/EMCAL/simulation/src/TRUElectronics.cxx
new file mode 100644
index 0000000000000..160fff8cfb062
--- /dev/null
+++ b/Detectors/EMCAL/simulation/src/TRUElectronics.cxx
@@ -0,0 +1,230 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file TRUElectronics.cxx
+/// \brief Implementation of the EMCAL TRUElectronics for the LZEROElectronics
+#include 
+#include 
+#include  // for LOG
+#include "EMCALSimulation/TRUElectronics.h"
+#include "EMCALBase/TriggerMappingV2.h"
+
+using namespace o2::emcal;
+
+//____________________________________________________________________________
+TRUElectronics::TRUElectronics() : mPatchSize(2), mWhichSide(0), mWhichSuperModuleSize(0)
+{
+  // DEFAULT CONSTRUCTOR
+  mFastOrs.resize(96);
+}
+//____________________________________________________________________________
+TRUElectronics::TRUElectronics(int patchSize, int whichSide, int whichSuperModuleSize) : mPatchSize(patchSize), mWhichSide(whichSide), mWhichSuperModuleSize(whichSuperModuleSize)
+{
+  // CONSTRUCTOR
+  mFastOrs.resize(96);
+}
+//____________________________________________________________________________
+void TRUElectronics::init()
+{
+  mIndexMapPatch.resize(77);
+  for (auto IndexMapPatch : mIndexMapPatch) {
+    (std::get<1>(IndexMapPatch)).resize(mPatchSize * mPatchSize);
+  }
+  mFiredFastOrIndexMapPatch.resize(77);
+  for (auto FiredFastOrIndexMapPatch : mFiredFastOrIndexMapPatch) {
+    (std::get<1>(FiredFastOrIndexMapPatch)).resize(mPatchSize * mPatchSize);
+  }
+  mADCvalues.resize(77);
+  mTimesum.resize(77);
+  mPatchIDSeedFastOrIDs.resize(77);
+  mPreviousTimebinADCvalue.resize(77);
+  mFastOrs.resize(96);
+  for (auto FastOr : mFastOrs) {
+    FastOr.init();
+  }
+  assignSeedModuleToAllPatches();
+  assignModulesToAllPatches();
+}
+//____________________________________________________________________________
+void TRUElectronics::clear()
+{
+  mIndexMapPatch.clear();
+  mFiredFastOrIndexMapPatch.clear();
+  mADCvalues.clear();
+  mTimesum.clear();
+  mPatchIDSeedFastOrIDs.clear();
+  mPreviousTimebinADCvalue.clear();
+}
+//____________________________________________________________________________
+void TRUElectronics::assignSeedModuleToPatchWithSTUIndexingFullModule(int& patchID)
+{
+  if (mWhichSide == 0) {
+    // A side
+    int rowSeed = patchID / 7;
+    int columnSeed = patchID % 7;
+    int SeedID = rowSeed + patchID;
+    std::get<1>(mPatchIDSeedFastOrIDs[patchID]) = SeedID;
+  } else if (mWhichSide == 1) {
+    // C side
+    int rowSeed = patchID / 7;
+    int columnSeed = patchID % 7;
+    int SeedID = 95 - (rowSeed + patchID);
+    std::get<1>(mPatchIDSeedFastOrIDs[patchID]) = SeedID;
+  }
+}
+//____________________________________________________________________________
+void TRUElectronics::assignSeedModuleToPatchWithSTUIndexingOneThirdModule(int& patchID)
+{
+  if (mWhichSide == 0) {
+    // A side
+    int rowSeed = patchID / 23;
+    int columnSeed = patchID % 23;
+    int SeedID = rowSeed + patchID;
+    std::get<1>(mPatchIDSeedFastOrIDs[patchID]) = SeedID;
+  } else if (mWhichSide == 1) {
+    // C side
+    int rowSeed = patchID / 23;
+    int columnSeed = patchID % 23;
+    int SeedID = 95 - (rowSeed + patchID);
+    std::get<1>(mPatchIDSeedFastOrIDs[patchID]) = SeedID;
+  }
+}
+//____________________________________________________________________________
+void TRUElectronics::assignSeedModuleToAllPatches()
+{
+  if (mWhichSuperModuleSize == 0) {
+    // Full Size
+    for (int i = 0; i < 77; i++) {
+      std::get<0>(mPatchIDSeedFastOrIDs[i]) = i;
+      assignSeedModuleToPatchWithSTUIndexingFullModule(i);
+    }
+  } else if (mWhichSuperModuleSize == 1) {
+    // One third Size
+    for (int i = 0; i < 69; i++) {
+      std::get<0>(mPatchIDSeedFastOrIDs[i]) = i;
+      assignSeedModuleToPatchWithSTUIndexingOneThirdModule(i);
+    }
+  }
+}
+//____________________________________________________________________________
+void TRUElectronics::assignModulesToAllPatches()
+{
+  if (mWhichSuperModuleSize == 0) {
+    // Full Size
+    for (int i = 0; i < 77; i++) {
+      auto& mIndexMapPatchID = std::get<0>(mIndexMapPatch[i]);
+      auto& mFiredFastOrIndexMapPatchID = std::get<0>(mFiredFastOrIndexMapPatch[i]);
+      auto& mADCvaluesID = std::get<0>(mADCvalues[i]);
+      auto& mPreviousTimebinADCvalueID = std::get<0>(mPreviousTimebinADCvalue[i]);
+      mIndexMapPatchID = i;
+      mFiredFastOrIndexMapPatchID = i;
+      mADCvaluesID = i;
+      mPreviousTimebinADCvalueID = i;
+      int SeedID = std::get<1>(mPatchIDSeedFastOrIDs[i]);
+      // assigning a square
+      if (mWhichSide == 0) {
+        // A side
+        for (int iRow = 0; iRow < mPatchSize; iRow++) {            // row advancement
+          for (int iColumn = 0; iColumn < mPatchSize; iColumn++) { // column advancement
+            auto& IndexMapPatch = std::get<1>(mIndexMapPatch[i]);
+            IndexMapPatch.push_back(SeedID + iRow + iColumn * 8);
+            // (std::get<1>(mIndexMapPatch[i])).push_back(SeedID + k + l * 8);
+          }
+        }
+      } else if (mWhichSide == 1) {
+        // C side
+        for (int iRow = 0; iRow < mPatchSize; iRow++) {            // row advancement
+          for (int iColumn = 0; iColumn < mPatchSize; iColumn++) { // column advancement
+            auto& IndexMapPatch = std::get<1>(mIndexMapPatch[i]);
+            IndexMapPatch.push_back(SeedID - iRow - iColumn * 8);
+            // (std::get<1>(mIndexMapPatch[i])).push_back(SeedID - k - l * 8);
+          }
+        }
+      }
+    }
+  } else if (mWhichSuperModuleSize == 1) {
+    // One third Size
+    for (int i = 0; i < 69; i++) {
+      auto& mIndexMapPatchID = std::get<0>(mIndexMapPatch[i]);
+      auto& mFiredFastOrIndexMapPatchID = std::get<0>(mFiredFastOrIndexMapPatch[i]);
+      auto& mADCvaluesID = std::get<0>(mADCvalues[i]);
+      auto& mPreviousTimebinADCvalueID = std::get<0>(mPreviousTimebinADCvalue[i]);
+      mIndexMapPatchID = i;
+      mFiredFastOrIndexMapPatchID = i;
+      mADCvaluesID = i;
+      mPreviousTimebinADCvalueID = i;
+      int SeedID = std::get<1>(mPatchIDSeedFastOrIDs[i]);
+      // assigning a square
+      if (mWhichSide == 0) {
+        // A side
+        for (int iRow = 0; iRow < mPatchSize; iRow++) {            // row advancement
+          for (int iColumn = 0; iColumn < mPatchSize; iColumn++) { // column advancement
+            auto& IndexMapPatch = std::get<1>(mIndexMapPatch[i]);
+            IndexMapPatch.push_back(SeedID + iRow + iColumn * 24);
+            // (std::get<1>(mIndexMapPatch[i])).push_back(SeedID + k + l * 24);
+          }
+        }
+      } else if (mWhichSide == 1) {
+        // C side
+        for (int iRow = 0; iRow < mPatchSize; iRow++) {            // row advancement
+          for (int iColumn = 0; iColumn < mPatchSize; iColumn++) { // column advancement
+            auto& IndexMapPatch = std::get<1>(mIndexMapPatch[i]);
+            IndexMapPatch.push_back(SeedID - iRow - iColumn * 8);
+            // (std::get<1>(mIndexMapPatch[i])).push_back(SeedID - k - l * 8);
+          }
+        }
+      }
+    }
+  }
+}
+//________________________________________________________
+void TRUElectronics::updateADC()
+{
+  // Loop over all patches and their ADC values
+  for (auto& patch : mADCvalues) {
+    auto& ADCvalues = std::get<1>(patch);
+    auto& PatchID = std::get<0>(patch);
+    if (ADCvalues.size() == 4) {
+      // If there are already four elements, pop the first
+      std::get<1>(mPreviousTimebinADCvalue[PatchID]) = ADCvalues.front();
+      ADCvalues.erase(ADCvalues.begin());
+    } else {
+      if (ADCvalues.size() > 4) {
+        LOG(debug) << "DIG TRU updateADC in TRUElectronics: ERROR!!!!! ";
+      }
+    }
+    double integralADCnew = 0;
+
+    for (auto FastOrs : std::get<1>(mIndexMapPatch[PatchID])) {
+      // Loop over all the fastOrs addigned to the current patch
+      // and sum the current ADC values together
+      auto elem = mFastOrs[FastOrs].mADCvalues;
+      if (elem.size() == 0) {
+        continue;
+      }
+      auto it = elem.end() - 1;
+      auto pointedvalue = *it;
+      integralADCnew += pointedvalue;
+    }
+    ADCvalues.push_back(integralADCnew);
+
+    // Saving the timesum
+    auto& CurrentPatchTimesum = std::get<1>(mTimesum[PatchID]);
+    if (CurrentPatchTimesum.size() == 4) {
+      CurrentPatchTimesum.erase(CurrentPatchTimesum.begin());
+    }
+    double IntegralADCvalues = 0;
+    for (auto ADC : ADCvalues) {
+      IntegralADCvalues += ADC;
+    }
+    CurrentPatchTimesum.push_back(IntegralADCvalues);
+  }
+}
\ No newline at end of file
diff --git a/Detectors/EMCAL/testsimulation/CMakeLists.txt b/Detectors/EMCAL/testsimulation/CMakeLists.txt
index 2d3ca30089f68..6d6496e456281 100644
--- a/Detectors/EMCAL/testsimulation/CMakeLists.txt
+++ b/Detectors/EMCAL/testsimulation/CMakeLists.txt
@@ -10,31 +10,31 @@
 # or submit itself to any jurisdiction.
 
 if(NOT BUILD_TEST_ROOT_MACROS)
-        return()
+  return()
 endif()
 
-set(genmacros fixedEnergyElectronGun.C fixedEnergyPionGun.C fixedEnergyPhotonGun.C)
-foreach(genmacro ${genmacros}) 
+set(genmacros fixedEnergyElectronGun.C fixedEnergyPionGun.C fixedEnergyPhotonGun.C fixedEnergyMuonGun.C)
+
+foreach(genmacro ${genmacros})
   MESSAGE(STATUS "Adding test for macro ${genmacro}")
   o2_add_test_root_macro(${genmacro}
-                         COMPILE_ONLY
-                         PUBLIC_LINK_LIBRARIES FairRoot::Base
-                         LABELS emcal)
+    COMPILE_ONLY
+    PUBLIC_LINK_LIBRARIES FairRoot::Base
+    LABELS emcal)
   o2_data_file(COPY ${genmacro} DESTINATION Detectors/EMC/testsimulation)
 endforeach()
 
-
 o2_add_test_root_macro(PutEmcalInTop.C
-                       PUBLIC_LINK_LIBRARIES O2::EMCALSimulation
-                                             O2::DetectorsPassive FairRoot::Base
-                       LABELS emcal)
+  PUBLIC_LINK_LIBRARIES O2::EMCALSimulation
+  O2::DetectorsPassive FairRoot::Base
+  LABELS emcal)
 
 o2_add_test_root_macro(drawEMCALgeometry.C
-                       PUBLIC_LINK_LIBRARIES O2::EMCALSimulation
-                                             O2::DetectorsPassive FairRoot::Base
-                       LABELS emcal)
+  PUBLIC_LINK_LIBRARIES O2::EMCALSimulation
+  O2::DetectorsPassive FairRoot::Base
+  LABELS emcal)
 
 o2_add_test_root_macro(run_sim_emcal.C
-                       PUBLIC_LINK_LIBRARIES FairRoot::Base O2::EMCALSimulation
-                                             O2::DetectorsPassive O2::Generators
-                       LABELS emcal)
+  PUBLIC_LINK_LIBRARIES FairRoot::Base O2::EMCALSimulation
+  O2::DetectorsPassive O2::Generators
+  LABELS emcal)
diff --git a/Detectors/EMCAL/testsimulation/PutEmcalInTop.C b/Detectors/EMCAL/testsimulation/PutEmcalInTop.C
index 3778f5f0bf12c..b744bbe249369 100644
--- a/Detectors/EMCAL/testsimulation/PutEmcalInTop.C
+++ b/Detectors/EMCAL/testsimulation/PutEmcalInTop.C
@@ -6,7 +6,7 @@
 #include "DetectorsPassive/Cave.h"
 #include "DetectorsPassive/FrameStructure.h"
 #include "EMCALSimulation/Detector.h"
-#include "FairLogger.h"
+#include 
 #include "FairRunSim.h"
 #include 
 #endif
diff --git a/Detectors/EMCAL/testsimulation/fixedEnergyMuonGun.C b/Detectors/EMCAL/testsimulation/fixedEnergyMuonGun.C
new file mode 100644
index 0000000000000..09c4f95bae95e
--- /dev/null
+++ b/Detectors/EMCAL/testsimulation/fixedEnergyMuonGun.C
@@ -0,0 +1,25 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#if !defined(__CLING__) || defined(__ROOTCLING__)
+#include 
+#include "FairBoxGenerator.h"
+#endif
+
+FairGenerator* fixedEnergyMuonGun(double energy)
+{
+  std::cout << "Single muon generator shooting at EMCAL with Energy " << energy << "GeV/c" << std::endl;
+  auto elecgen = new FairBoxGenerator(13, 1);
+  elecgen->SetEtaRange(-0.67, 0.67);
+  elecgen->SetPhiRange(90, 340);
+  elecgen->SetPRange(energy, energy);
+  return elecgen;
+}
\ No newline at end of file
diff --git a/Detectors/EMCAL/workflow/CMakeLists.txt b/Detectors/EMCAL/workflow/CMakeLists.txt
index 4e1916e5d7414..7222b0aedc320 100644
--- a/Detectors/EMCAL/workflow/CMakeLists.txt
+++ b/Detectors/EMCAL/workflow/CMakeLists.txt
@@ -10,65 +10,93 @@
 # or submit itself to any jurisdiction.
 
 o2_add_library(EMCALWorkflow
-               SOURCES src/EMCALDigitWriterSpec.cxx
-                       src/EMCALDigitizerSpec.cxx
-                       src/RecoWorkflow.cxx
-                       src/PublisherSpec.cxx
-                       src/CellConverterSpec.cxx
-                       src/ClusterizerSpec.cxx
-                       src/DigitsPrinterSpec.cxx
-                       src/AnalysisClusterSpec.cxx
-                       src/RawToCellConverterSpec.cxx
-                       src/EntropyEncoderSpec.cxx
-                       src/EntropyDecoderSpec.cxx
-                       src/StandaloneAODProducerSpec.cxx
-               PUBLIC_LINK_LIBRARIES O2::Framework O2::DataFormatsCTP O2::DataFormatsEMCAL O2::EMCALSimulation O2::Steer
-                                     O2::DPLUtils O2::EMCALBase O2::EMCALReconstruction O2::Algorithm O2::MathUtils)
+        SOURCES src/CalibLoader.cxx
+        src/EMCALDigitWriterSpec.cxx
+        src/EMCALDigitizerSpec.cxx
+        src/RecoWorkflow.cxx
+        src/PublisherSpec.cxx
+        src/CellConverterSpec.cxx
+        src/ClusterizerSpec.cxx
+        src/DigitsPrinterSpec.cxx
+        src/AnalysisClusterSpec.cxx
+        src/RawToCellConverterSpec.cxx
+        src/EntropyEncoderSpec.cxx
+        src/OfflineCalibSpec.cxx
+        src/CellRecalibratorSpec.cxx
+        src/EntropyDecoderSpec.cxx
+        src/StandaloneAODProducerSpec.cxx
+        PUBLIC_LINK_LIBRARIES O2::Framework
+        O2::DataFormatsCTP
+        O2::DataFormatsEMCAL
+        O2::DataFormatsFT0
+        O2::DataFormatsFV0
+        O2::EMCALSimulation
+        O2::Steer
+        O2::DPLUtils
+        O2::EMCALBase
+        O2::EMCALCalib
+        O2::EMCALCalibration
+        O2::EMCALReconstruction
+        O2::Algorithm
+        O2::MathUtils)
 
 o2_add_executable(reco-workflow
-                  COMPONENT_NAME emcal
-                  SOURCES src/emc-reco-workflow.cxx
-                  PUBLIC_LINK_LIBRARIES O2::EMCALWorkflow)
+        COMPONENT_NAME emcal
+        SOURCES src/emc-reco-workflow.cxx
+        PUBLIC_LINK_LIBRARIES O2::EMCALWorkflow)
 
 o2_add_executable(entropy-encoder-workflow
-                  COMPONENT_NAME emcal
-                  SOURCES src/entropy-encoder-workflow.cxx
-                  PUBLIC_LINK_LIBRARIES O2::EMCALWorkflow)
+        COMPONENT_NAME emcal
+        SOURCES src/entropy-encoder-workflow.cxx
+        PUBLIC_LINK_LIBRARIES O2::EMCALWorkflow)
 
 o2_add_executable(emc-dcs-workflow
-                  COMPONENT_NAME calibration
-                  SOURCES src/emc-dcs-data-workflow.cxx
-                  PUBLIC_LINK_LIBRARIES O2::Framework
-                                       O2::EMCALCalibration
-                                       O2::EMCALWorkflow
-                                       O2::DetectorsDCS)
+        COMPONENT_NAME calibration
+        SOURCES src/emc-dcs-data-workflow.cxx
+        PUBLIC_LINK_LIBRARIES O2::Framework
+        O2::EMCALCalibration
+        O2::EMCALWorkflow
+        O2::DetectorsDCS)
 
 o2_add_executable(emc-dcs-sim-workflow
-                  COMPONENT_NAME calibration
-                  SOURCES src/emc-dcs-sim-workflow.cxx
-                  PUBLIC_LINK_LIBRARIES O2::DCStestWorkflow)
+        COMPONENT_NAME calibration
+        SOURCES src/emc-dcs-sim-workflow.cxx
+        PUBLIC_LINK_LIBRARIES O2::DCStestWorkflow)
 
 o2_add_executable(cell-writer-workflow
-                  COMPONENT_NAME emcal
-                  SOURCES src/cell-writer-workflow.cxx
-                  PUBLIC_LINK_LIBRARIES O2::EMCALWorkflow)
+        COMPONENT_NAME emcal
+        SOURCES src/cell-writer-workflow.cxx
+        PUBLIC_LINK_LIBRARIES O2::EMCALWorkflow)
+
+o2_add_executable(emc-offline-calib-workflow
+        COMPONENT_NAME emcal
+        SOURCES src/emc-offline-calib-workflow.cxx
+        PUBLIC_LINK_LIBRARIES O2::Framework
+        O2::EMCALWorkflow
+        O2::EMCALCalibration)
+
+o2_add_executable(cell-recalibrator-workflow
+        COMPONENT_NAME emcal
+        SOURCES src/cell-recalibrator-workflow.cxx
+        PUBLIC_LINK_LIBRARIES O2::Framework
+        O2::EMCALWorkflow)
 
 o2_add_executable(cell-reader-workflow
-                  COMPONENT_NAME emcal
-                  SOURCES src/cell-reader-workflow.cxx
-                  PUBLIC_LINK_LIBRARIES O2::EMCALWorkflow)
+        COMPONENT_NAME emcal
+        SOURCES src/cell-reader-workflow.cxx
+        PUBLIC_LINK_LIBRARIES O2::EMCALWorkflow)
 
 o2_add_executable(digit-reader-workflow
-                  COMPONENT_NAME emcal
-                  SOURCES src/digit-reader-workflow.cxx
-                  PUBLIC_LINK_LIBRARIES O2::EMCALWorkflow)
+        COMPONENT_NAME emcal
+        SOURCES src/digit-reader-workflow.cxx
+        PUBLIC_LINK_LIBRARIES O2::EMCALWorkflow)
 
 o2_add_executable(standalone-aod-producer-workflow
-                  COMPONENT_NAME emcal
-                  SOURCES src/standalone-aod-producer-workflow.cxx
-                  PUBLIC_LINK_LIBRARIES O2::EMCALWorkflow)
+        COMPONENT_NAME emcal
+        SOURCES src/standalone-aod-producer-workflow.cxx
+        PUBLIC_LINK_LIBRARIES O2::EMCALWorkflow)
 
 o2_add_executable(channel-data-producer
-                  COMPONENT_NAME emcal
-                  SOURCES src/emc-channel-data-producer.cxx
-                  PUBLIC_LINK_LIBRARIES O2::EMCALWorkflow)
+        COMPONENT_NAME emcal
+        SOURCES src/emc-channel-data-producer.cxx
+        PUBLIC_LINK_LIBRARIES O2::EMCALWorkflow)
diff --git a/Detectors/EMCAL/workflow/include/EMCALWorkflow/AnalysisClusterSpec.h b/Detectors/EMCAL/workflow/include/EMCALWorkflow/AnalysisClusterSpec.h
index 1c16c4a270454..4bcf6e9f2c577 100644
--- a/Detectors/EMCAL/workflow/include/EMCALWorkflow/AnalysisClusterSpec.h
+++ b/Detectors/EMCAL/workflow/include/EMCALWorkflow/AnalysisClusterSpec.h
@@ -9,6 +9,9 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 
+#ifndef O2_EMCAL_ANALYSISCLUSTER_SPEC
+#define O2_EMCAL_ANALYSISCLUSTER_SPEC
+
 #include 
 
 #include "DataFormatsEMCAL/Cluster.h"
@@ -67,10 +70,10 @@ class AnalysisClusterSpec : public framework::Task
 
  private:
   void updateTimeDependentParams(framework::ProcessingContext& pc);
-  o2::emcal::Clusterizer mClusterizer;                        ///< Clusterizer object
-  o2::emcal::Geometry* mGeometry = nullptr;                              ///< Pointer to geometry object
-  o2::emcal::EventHandler* mEventHandler = nullptr;           ///< Pointer to the event builder
-  o2::emcal::ClusterFactory* mClusterFactory = nullptr;       ///< Pointer to the cluster builder
+  o2::emcal::Clusterizer mClusterizer;                  ///< Clusterizer object
+  o2::emcal::Geometry* mGeometry = nullptr;                        ///< Pointer to geometry object
+  o2::emcal::EventHandler* mEventHandler = nullptr;     ///< Pointer to the event builder
+  o2::emcal::ClusterFactory* mClusterFactory = nullptr; ///< Pointer to the cluster builder
   std::shared_ptr mGGCCDBRequest;
   std::vector* mOutputAnaClusters = nullptr; ///< Container with output clusters (pointer)
 };
@@ -86,3 +89,5 @@ framework::DataProcessorSpec getAnalysisClusterSpec(bool useDigits);
 } // namespace emcal
 
 } // namespace o2
+
+#endif // O2_EMCAL_ANALYSISCLUSTER_SPEC
diff --git a/Detectors/EMCAL/workflow/include/EMCALWorkflow/CalibLoader.h b/Detectors/EMCAL/workflow/include/EMCALWorkflow/CalibLoader.h
new file mode 100644
index 0000000000000..06ffc1f5db3c9
--- /dev/null
+++ b/Detectors/EMCAL/workflow/include/EMCALWorkflow/CalibLoader.h
@@ -0,0 +1,255 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef O2_EMCAL_CALIBLOADER
+#define O2_EMCAL_CALIBLOADER
+
+#include 
+#include 
+#include 
+#include "Framework/ConcreteDataMatcher.h"
+#include "Framework/InputSpec.h"
+
+namespace o2
+{
+
+namespace emcal
+{
+class BadChannelMap;
+class FeeDCS;
+class EMCALChannelScaleFactors;
+class TimeCalibrationParams;
+class GainCalibrationFactors;
+class TempCalibrationParams;
+class SimParam;
+class RecoParam;
+
+/// \class CalibLoader
+/// \brief Handler for EMCAL calibration objects in DPL workflows
+/// \ingroup EMCALWorkflow
+/// \author Markus Fasel , Oak Ridge National Laboratory
+/// \since Sept. 5, 2022
+///
+/// Loading the calibration object i based on the objects stored in the CCDB, which
+/// is fully delegated to the framework CCDB support. Alternativelty, particularly for
+/// testing purpose, local CCDB paths for files for the different calibration objects
+/// can be provided. In this case the CCDB is bypassed. The function `static_load`
+/// must always be called in the init function - it will take care of loading the
+/// calibration objects to be taken from file. Furthermore in finalizeCCDB the workflow
+/// must call finalizeCCDB from the CalibLoader in order to make load the calibration
+/// objects requested from CCDB.
+class CalibLoader
+{
+ public:
+  /// \brief Constructor
+  CalibLoader() = default;
+
+  /// \brief Destructor
+  ~CalibLoader() = default;
+
+  /// \brief Access to current bad channel map
+  /// \return Current bad channel map (nullptr if not loaded)
+  BadChannelMap* getBadChannelMap() const { return mBadChannelMap; }
+
+  /// \brief Access to current BCM Scale factors
+  /// \return Current BCM scale factors (nullptr if not loaded)
+  EMCALChannelScaleFactors* getBCMScaleFactors() const { return mBCMScaleFactors; }
+
+  /// \brief Access to current FEE DCS params
+  /// \return Current FEE DCS params (nullptr if not loaded)
+  FeeDCS* getFEEDCS() const { return mFeeDCS; }
+
+  /// \brief Access to current time calibration params
+  /// \return Current time calibration params
+  TimeCalibrationParams* getTimeCalibration() const { return mTimeCalibParams; }
+
+  /// \brief Access to current gain calibration factors
+  /// \return Current gain calibration factors
+  GainCalibrationFactors* getGainCalibration() const { return mGainCalibParams; }
+
+  /// \brief Access to current temperature calibration params
+  /// \return Current temperature calibration factors
+  TempCalibrationParams* getTemperatureCalibration() const { return mTempCalibParams; }
+
+  /// \brief Check whether the bad channel map is handled
+  /// \return True if the bad channel map is handled, false otherwise
+  bool hasBadChannelMap() const { return mEnableStatus.test(OBJ_BADCHANNELMAP); }
+
+  /// \brief Check whether the BCM scale factors are handled
+  /// \return True if the BCM scale factors are handled, false otherwise
+  bool hasBCMScaleFactors() const { return mEnableStatus.test(OBJ_BCMSCALEFACTORS); }
+
+  /// \brief Check whether the FEE DCS params are handled
+  /// \return True if the FEE DCS params are handled, false otherwise
+  bool hasFEEDCS() const { return mEnableStatus.test(OBJ_FEEDCS); }
+
+  /// \brief Check whether the time calibration params are handled
+  /// \return True if the time calibration params are handled, false otherwise
+  bool hasTimeCalib() const { return mEnableStatus.test(OBJ_TIMECALIB); }
+
+  /// \brief Check whether the gain calibration factors are handled
+  /// \return True if the gain calibration factors are handled, false otherwise
+  bool hasGainCalib() const { return mEnableStatus.test(OBJ_GAINCALIB); }
+
+  /// \brief Check whether the temperature calibration params are handled
+  /// \return True if the temperature calibration params are handled, false otherwise
+  bool hasTemperatureCalib() const { return mEnableStatus.test(OBJ_TEMPCALIB); }
+
+  /// \brief Check whether the reconstruction params are handled
+  /// \return True if the reconstruction params are handled, false otherwise
+  bool hasRecoParams() const { return mEnableStatus.test(OBJ_RECOPARAM); }
+
+  /// \brief Check whether the reconstruction params are handled
+  /// \return True if the reconstruction params are handled, false otherwise
+  bool hasSimParams() const { return mEnableStatus.test(OBJ_SIMPARAM); }
+
+  /// \brief Check whether the bad channel map has been updated
+  /// \return True if the bad channel map has been updated, false otherwise
+  bool hasUpdateBadChannelMap() const { return mUpdateStatus.test(OBJ_BADCHANNELMAP); }
+
+  /// \brief Check whether the BCM scale factors have been updated
+  /// \return True if the bad channel map has been updated, false otherwise
+  bool hasUpdateBCMScaleFactors() const { return mUpdateStatus.test(OBJ_BCMSCALEFACTORS); }
+
+  /// \brief Check whether the FEE DCS params have been updated
+  /// \return True if the FEE DCS params have been updated, false otherwise
+  bool hasUpdateFEEDCS() const { return mUpdateStatus.test(OBJ_FEEDCS); }
+
+  /// \brief Check whether the time calibration params have been updated
+  /// \return True if the time calibration params have been updated, false otherwise
+  bool hasUpdateTimeCalib() const { return mUpdateStatus.test(OBJ_TIMECALIB); }
+
+  /// \brief Check whether the gain calibration params have been updated
+  /// \return True if the gain calibration params have been updated, false otherwise
+  bool hasUpdateGainCalib() const { return mUpdateStatus.test(OBJ_GAINCALIB); }
+
+  /// \brief Check whether the temperature calibration params have been updated
+  /// \return True if the temperature calibration params have been updated, false otherwise
+  bool hasUpdateTemperatureCalib() const { return mUpdateStatus.test(OBJ_TEMPCALIB); }
+
+  /// \brief Check whether the reconstruction params have been updated
+  /// \return True if the reconstruction params have been updated, false otherwise
+  bool hasUpdateRecoParam() const { return mUpdateStatus.test(OBJ_RECOPARAM); }
+
+  /// \brief Check whether the simulation params have been updated
+  /// \return True if the simulation params have been updated, false otherwise
+  bool hasUpdateSimParam() const { return mUpdateStatus.test(OBJ_SIMPARAM); }
+
+  /// \brief Enable loading of the bad channel map
+  /// \param doEnable If true the bad channel map is loaded (per default from CCDB)
+  void enableBadChannelMap(bool doEnable) { mEnableStatus.set(OBJ_BADCHANNELMAP, doEnable); }
+
+  /// \brief Enable loading of the BCM scale factors
+  /// \param doEnable If true the BCM scale factors are loaded (per default from CCDB)
+  void enableBCMScaleFactors(bool doEnable) { mEnableStatus.set(OBJ_BCMSCALEFACTORS, doEnable); }
+
+  /// \brief Enable loading of the FEE DCS params
+  /// \param doEnable If true the FEE DCS params are loaded (per default from CCDB)
+  void enableFEEDCS(bool doEnable) { mEnableStatus.set(OBJ_FEEDCS, doEnable); }
+
+  /// \brief Enable loading of the time calibration params
+  /// \param doEnable If true the time calibration params are loaded (per default from CCDB)
+  void enableTimeCalib(bool doEnable) { mEnableStatus.set(OBJ_TIMECALIB, doEnable); }
+
+  /// \brief Enable loading of the gain calibration factors
+  /// \param doEnable If true the gain calibration factors are loaded (per default from CCDB)
+  void enableGainCalib(bool doEnable) { mEnableStatus.set(OBJ_GAINCALIB, doEnable); }
+
+  /// \brief Enable loading of the temperature calibration params
+  /// \param doEnable If true the temperature calibration params are loaded (per default from CCDB)
+  void enableTemperatureCalib(bool doEnable) { mEnableStatus.set(OBJ_TEMPCALIB, doEnable); }
+
+  /// \brief Enable loading of the reconstruction params
+  /// \param doEnable If true the reconstruction params are loaded (per default from CCDB)
+  void enableRecoParams(bool doEnable) { mEnableStatus.set(OBJ_RECOPARAM, doEnable); }
+
+  /// \brief Enable loading of the simulation params
+  /// \param doEnable If true the simulation params are loaded (per default from CCDB)
+  void enableSimParams(bool doEnable) { mEnableStatus.set(OBJ_SIMPARAM, doEnable); }
+
+  /// \brief Mark bad channel map as updated
+  void setUpdateBadChannelMap() { mUpdateStatus.set(OBJ_BADCHANNELMAP, true); }
+
+  /// \brief Mark BCM scale factors as updated
+  void setUpdateBCMScaleFactors() { mUpdateStatus.set(OBJ_BCMSCALEFACTORS, true); }
+
+  /// \brief Mark FEE DCS params as updated
+  void setUpdateFEEDCS() { mUpdateStatus.set(OBJ_FEEDCS, true); }
+
+  /// \brief Mark time calibration params as updated
+  void setUpdateTimeCalib() { mUpdateStatus.set(OBJ_TIMECALIB, true); }
+
+  /// \brief Mark gain calibration params as updated
+  void setUpdateGainCalib() { mUpdateStatus.set(OBJ_GAINCALIB, true); }
+
+  /// \brief Mark temperature calibration params as updated
+  void setUpdateTemperatureCalib() { mUpdateStatus.set(OBJ_TEMPCALIB, true); }
+
+  /// \brief Mark reconstruction params as updated
+  void setUpdateRecoParams() { mUpdateStatus.set(OBJ_RECOPARAM, true); }
+
+  /// \brief Mark simulation params as updated
+  void setUpdateSimParams() { mUpdateStatus.set(OBJ_SIMPARAM, true); }
+
+  /// \brief Reset the update status (all objects marked as false)
+  void resetUpdateStatus() { mUpdateStatus.reset(); }
+
+  /// \brief Define input specs in workflow for calibration objects to be loaded from the CCDB
+  /// \param ccdbInputs List of inputs where the CCDB input specs will be added to
+  ///
+  /// Defining only objects which are enabled and for which no local path is specified.
+  void defineInputSpecs(std::vector& ccdbInputs);
+
+  /// \brief Check for updates of the calibration objects in the processing context
+  /// \param ctx ProcessingContext with InputSpecs
+  ///
+  /// Triggers finalizeCCDB in case of updates.
+  void checkUpdates(o2::framework::ProcessingContext& ctx);
+
+  /// \brief Callback for objects loaded from CCDB
+  /// \param matcher Type of the CCDB object
+  /// \param obj CCDB object loaded by the framework
+  ///
+  /// Only CCDB objects compatible with one of the types are handled. In case the object type
+  /// is requested and not loaded from local source the object accepted locally and accessible
+  /// via the corresponding getter function.
+  bool finalizeCCDB(framework::ConcreteDataMatcher& matcher, void* obj);
+
+ private:
+  enum CalibObject_t {
+    OBJ_BADCHANNELMAP,   ///< Bad channel map
+    OBJ_TIMECALIB,       ///< Time calibration params
+    OBJ_GAINCALIB,       ///< Energy calibration params
+    OBJ_TEMPCALIB,       ///< Temperature calibration params
+    OBJ_TIMSLEWINGCORR,  ///< Time slewing correction
+    OBJ_BCMSCALEFACTORS, ///< BCM scale factors
+    OBJ_FEEDCS,          ///< FEE DCS params
+    OBJ_SIMPARAM,        ///< Simulation params
+    OBJ_RECOPARAM,       ///< Reco params
+    OBJ_CALIBPARAM       ///< Calibration params
+  };
+  o2::emcal::BadChannelMap* mBadChannelMap = nullptr;              ///< Container of current bad channel map
+  o2::emcal::FeeDCS* mFeeDCS = nullptr;                            ///< Container of current FEE DCS params
+  o2::emcal::TimeCalibrationParams* mTimeCalibParams = nullptr;    ///< Container of current time calibration object
+  o2::emcal::GainCalibrationFactors* mGainCalibParams = nullptr;   ///< Container of current gain calibration object
+  o2::emcal::TempCalibrationParams* mTempCalibParams = nullptr;    ///< Container of current temperature calibration object
+  o2::emcal::EMCALChannelScaleFactors* mBCMScaleFactors = nullptr; ///< Container of current bad channel scale factors
+  o2::emcal::RecoParam* mRecoParam;                                ///< Current reconstruction parameters
+  o2::emcal::SimParam* mSimParam;                                  ///< Current simulation parameters
+  std::bitset<16> mEnableStatus;                                   ///< Object enable status
+  std::bitset<16> mUpdateStatus;                                   ///< Object update status
+};
+
+} // namespace emcal
+
+} // namespace o2
+
+#endif //  O2_EMCAL_CALIBLOADER
diff --git a/Detectors/EMCAL/workflow/include/EMCALWorkflow/CellConverterSpec.h b/Detectors/EMCAL/workflow/include/EMCALWorkflow/CellConverterSpec.h
index 37cecc33f1b44..bd6c2699a7e33 100644
--- a/Detectors/EMCAL/workflow/include/EMCALWorkflow/CellConverterSpec.h
+++ b/Detectors/EMCAL/workflow/include/EMCALWorkflow/CellConverterSpec.h
@@ -9,6 +9,9 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 
+#ifndef O2_EMCAL_CELLCONVERTER_SPEC
+#define O2_EMCAL_CELLCONVERTER_SPEC
+
 #include 
 
 #include "DataFormatsEMCAL/Cell.h"
@@ -19,6 +22,7 @@
 #include "EMCALBase/Geometry.h"
 #include "EMCALReconstruction/CaloRawFitter.h"
 #include "EMCALReconstruction/AltroHelper.h"
+#include "EMCALWorkflow/CalibLoader.h"
 #include "SimulationDataFormat/MCTruthContainer.h"
 
 namespace o2
@@ -46,7 +50,10 @@ class CellConverterSpec : public framework::Task
  public:
   /// \brief Constructor
   /// \param propagateMC If true the MCTruthContainer is propagated to the output
-  CellConverterSpec(bool propagateMC) : framework::Task(), mPropagateMC(propagateMC){};
+  /// \param useccdb If true the TecoParams
+  /// \param inputSubsepc Subsepc of input objects
+  /// \param outputSubspec Subspec of output objects
+  CellConverterSpec(bool propagateMC, int inputSubsepc, int outputSubspec, std::shared_ptr calibhandler) : framework::Task(), mPropagateMC(propagateMC), mSubspecificationIn(inputSubsepc), mSubspecificationOut(outputSubspec), mCalibHandler(calibhandler){};
 
   /// \brief Destructor
   ~CellConverterSpec() override = default;
@@ -73,6 +80,8 @@ class CellConverterSpec : public framework::Task
   /// Output MC-truth: {"EMC", "CELLSMCTR", 0, Lifetime::Timeframe}
   void run(framework::ProcessingContext& ctx) final;
 
+  void finaliseCCDB(o2::framework::ConcreteDataMatcher& matcher, void* obj) final;
+
  protected:
   std::vector digitsToBunches(gsl::span digits, std::vector>& mcLabels);
 
@@ -82,10 +91,16 @@ class CellConverterSpec : public framework::Task
 
   int selectMaximumBunch(const gsl::span& bunchvector);
 
+  /// \brief Update calibration objects
+  void updateCalibrationObjects();
+
  private:
   bool mPropagateMC = false;                                           ///< Switch whether to process MC true labels
-  o2::emcal::Geometry* mGeometry = nullptr;                            ///! mRawFitter;                ///! mCalibHandler;               //! mRawFitter;                //! mOutputCells;                           ///< Container with output cells
   std::vector mOutputTriggers;               ///< Container with output trigger records
   o2::dataformats::MCTruthContainer mOutputLabels; ///< Container with output MC labels
@@ -94,12 +109,17 @@ class CellConverterSpec : public framework::Task
 /// \brief Creating DataProcessorSpec for the EMCAL Cell Converter Spec
 /// \ingroup EMCALworkflow
 /// \param propagateMC If true the MC truth container is propagated to the output
+/// \param useccdb If true the RecoParams are loaded from the CCDB
+/// \param inputSubsepc Subspec of input objects
+/// \param outputSubspec Subspec of output objects
 ///
 /// Refer to CellConverterSpec::run for input and output specs
-framework::DataProcessorSpec getCellConverterSpec(bool propagateMC);
+framework::DataProcessorSpec getCellConverterSpec(bool propagateMC, int inputSubsepc = 0, int outputSubspec = 0);
 
 } // namespace reco_workflow
 
 } // namespace emcal
 
 } // namespace o2
+
+#endif // O2_EMCAL_CELLCONVERTER_SPEC
diff --git a/Detectors/EMCAL/workflow/include/EMCALWorkflow/CellRecalibratorSpec.h b/Detectors/EMCAL/workflow/include/EMCALWorkflow/CellRecalibratorSpec.h
new file mode 100644
index 0000000000000..5c55377290862
--- /dev/null
+++ b/Detectors/EMCAL/workflow/include/EMCALWorkflow/CellRecalibratorSpec.h
@@ -0,0 +1,167 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef O2_EMCAL_CELLRECALIBRATOR_SPEC
+#define O2_EMCAL_CELLRECALIBRATOR_SPEC
+
+#include 
+#include 
+#include 
+#include "DataFormatsEMCAL/MCLabel.h"
+#include "EMCALCalib/CellRecalibrator.h"
+#include "EMCALWorkflow/CalibLoader.h"
+#include "Framework/ConcreteDataMatcher.h"
+#include "Framework/DataProcessorSpec.h"
+#include "Framework/Task.h"
+#include "SimulationDataFormat/MCTruthContainer.h"
+
+namespace o2
+{
+
+namespace emcal
+{
+
+class Cell;
+class TriggerRecord;
+
+/// \class CellRecalibratorSpec
+/// \brief Recalibration workflow at cell level
+/// \ingroup EMCALworkflow
+/// \author Markus Fasel , Oak Ridge National Laboratory
+/// \since Sept 5, 2022
+///
+/// # Recalibration at cell level
+///
+/// The workflow recalibrates a cell vector (with corresponding trigger records) for
+/// a given timeframe. The following calibrations are supported:
+/// - Bad channel calibration
+/// - Time calibration
+/// - Gain (energy) calibration
+/// Calibration objects are handled automatically via the CCDB. Calibrations must be
+/// enabled manually and are only applied in case the calibration objects exists in
+/// the CCDB.
+///
+/// While the energy and time calibration only change values within a cell the bad channel
+/// calibration decides whether a cell is accepted. Therefore the amount of cells can change
+/// for the given trigger. New trigger record objects are created and published to the same
+/// subspec as what is used for the output cell vector.
+class CellRecalibratorSpec : public framework::Task
+{
+ public:
+  /// \enum LEDEventSettings
+  /// \brief Dedicated handling for LED events
+  enum class LEDEventSettings {
+    KEEP,    ///< Keep LED events in timeframe (uncalibrated)
+    DROP,    ///< Drop LED events
+    REDIRECT ///< Redirect LED events to dedicated output
+  };
+  /// \brief Constructor
+  /// \param outputspec Subspecification under which the output is posted
+  /// \param ledsettings Handling of cells from LED events
+  /// \param badChannelCalib If true the bad channel calibration is enabled
+  /// \param timeCalib If true the time calibration is enabled
+  /// \param gainCalib If true the gain calibration is enabled
+  /// \param isMC If true the MCLabelContainer is adapted
+  /// \param calibHandler Handler for calibration object loading
+  CellRecalibratorSpec(uint32_t outputspec, LEDEventSettings ledsettings, bool badChannelCalib, bool timeCalib, bool gainCalib, bool isMC, std::shared_ptr(calibHandler));
+
+  /// \brief Destructor
+  ~CellRecalibratorSpec() final = default;
+
+  /// \brief Initialize recalibrator
+  /// \param ctx Init context
+  void init(framework::InitContext& ctx) final;
+
+  /// \brief Run recalibration of cells for a new timeframe
+  /// \param ctx Processing context
+  ///
+  /// Iterates over all triggers in the timeframe and recalibrates cells. Only
+  /// calibrations which are enabled are applied. Calibrated cells and new trigger
+  /// records are posted at the end of the timeframe to the subspecification requested
+  /// in the constructor using standard types for cells and trigger records
+  void run(framework::ProcessingContext& ctx) final;
+
+  /// \brief Fetching cell objects and assigning them to the internal handlers
+  /// \param matcher Data type matcher of the CCDB object
+  /// \param obj Pointer to CCDB object loaded
+  ///
+  /// Loading CCDB object into internal cache for the 3 supported CCDB entries (bad
+  /// channel map, time calibration params, gain calibration params). Objects are only
+  /// loaded in case the calibration type was enabled.
+  void finaliseCCDB(framework::ConcreteDataMatcher& matcher, void* obj) final;
+
+  /// \brief Switch for bad channel calibration
+  /// \param doRun If true the bad channel calibration is applied
+  void setRunBadChannelCalibration(bool doRun) { mCalibrationSettings.set(BADCHANNEL_CALIB, doRun); };
+
+  /// \brief Switch for time calibration
+  /// \param doRun If true the time calibration is applied
+  void setRunTimeCalibration(bool doRun) { mCalibrationSettings.set(TIME_CALIB, doRun); }
+
+  /// \brief Switch for the gain calibration
+  /// \param doRun If true the gain calibration is applied
+  void setRunGainCalibration(bool doRun) { mCalibrationSettings.set(GAIN_CALIB, doRun); }
+
+  /// \brief Check if the bad channel calibration is enabled
+  /// \return True if the bad channel calibration is enabled, false otherwise
+  bool isRunBadChannlCalibration() const { return mCalibrationSettings.test(BADCHANNEL_CALIB); }
+
+  /// \brief Check if the time calibration is enabled
+  /// \return True if the time calibration is enabled, false otherwise
+  bool isRunTimeCalibration() const { return mCalibrationSettings.test(TIME_CALIB); }
+
+  /// \brief Check if the gain calibration is enabled
+  /// \return True if the gain calibration is enabled, false otherwise
+  bool isRunGainCalibration() const { return mCalibrationSettings.test(GAIN_CALIB); }
+
+ private:
+  /// \brief Update calibration objects (if changed)
+  void updateCalibObjects();
+
+  /// \brief write event cell container to output
+  /// \param selectedCells Cells to be added
+  /// \param outputcontainer Output container
+  /// \param outputtriggers Output trigger records
+  void writeTrigger(const gsl::span selectedCells, const o2::emcal::TriggerRecord& eventtrigger, std::vector& outputcontainer, std::vector& outputtriggers);
+
+  void writeMCLabels(const o2::dataformats::MCTruthContainer& inputlabels, o2::dataformats::MCTruthContainer& outputContainer, const std::vector& keptIndices, int firstindex);
+
+  /// \enum CalibrationType_t
+  /// \brief Calibrations handled by the recalibration workflow
+  enum CalibrationType_t {
+    BADCHANNEL_CALIB = 0, ///< Bad channel calibration
+    TIME_CALIB = 1,       ///< Time calibration
+    GAIN_CALIB = 2        ///< Gain calibration
+  };
+
+  uint32_t mOutputSubspec = 0;                            ///< output subspecification;
+  bool mIsMC = false;                                     ///< MC mode
+  LEDEventSettings mLEDsettings = LEDEventSettings::KEEP; ///< Handling of LED events
+  std::bitset<8> mCalibrationSettings;                    ///< Recalibration settings (which calibration to be applied)
+  std::shared_ptr mCalibrationHandler;       ///< Handler loading calibration objects
+  CellRecalibrator mCellRecalibrator;                     ///< Recalibrator at cell level
+};
+
+/// \brief Create CellRecalibrator processor spec
+/// \param inputSubsepc Subspecification used for the input objects (cells and trigger records)
+/// \param outputSubspec Subspecification used for the output objects (cells and trigger records)
+/// \param ledsettings Settings of LED handling (keep/drop/redirect)
+/// \param badChannelCalib If true the bad channel calibration is enabled
+/// \param timeCalib If true the time calibration is enabled
+/// \param gainCalib If true the gain (energy) calibration is enabled
+/// \param isMC If true also the MC label container is adapted (relevant only for bad channel masking)
+framework::DataProcessorSpec getCellRecalibratorSpec(uint32_t inputSubspec, uint32_t outputSubspec, uint32_t ledsettings, bool badChannelCalib, bool timeCalib, bool gainCalib, bool isMC);
+
+} // namespace emcal
+
+} // namespace o2
+
+#endif // O2_EMCAL_CELLRECALIBRATOR_SPEC
diff --git a/Detectors/EMCAL/workflow/include/EMCALWorkflow/ClusterizerSpec.h b/Detectors/EMCAL/workflow/include/EMCALWorkflow/ClusterizerSpec.h
index 67c60fab6655c..6fd204acb161f 100644
--- a/Detectors/EMCAL/workflow/include/EMCALWorkflow/ClusterizerSpec.h
+++ b/Detectors/EMCAL/workflow/include/EMCALWorkflow/ClusterizerSpec.h
@@ -9,6 +9,9 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 
+#ifndef O2_EMCAL_CLUSTERIZER_SPEC
+#define O2_EMCAL_CLUSTERIZER_SPEC
+
 #include 
 
 #include "DataFormatsEMCAL/Cluster.h"
@@ -85,3 +88,5 @@ framework::DataProcessorSpec getClusterizerSpec(bool useDigits);
 } // namespace emcal
 
 } // namespace o2
+
+#endif // O2_EMCAL_CLUSTERIZER_SPEC
diff --git a/Detectors/EMCAL/workflow/include/EMCALWorkflow/DigitsPrinterSpec.h b/Detectors/EMCAL/workflow/include/EMCALWorkflow/DigitsPrinterSpec.h
index 6ee0b7143c186..378da6e14dc7d 100644
--- a/Detectors/EMCAL/workflow/include/EMCALWorkflow/DigitsPrinterSpec.h
+++ b/Detectors/EMCAL/workflow/include/EMCALWorkflow/DigitsPrinterSpec.h
@@ -9,6 +9,9 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 
+#ifndef O2_EMCAL_DIGITSPRINTER_SPEC
+#define O2_EMCAL_DIGITSPRINTER_SPEC
+
 #include 
 #include "Framework/DataProcessorSpec.h"
 #include "Framework/Task.h"
@@ -61,9 +64,11 @@ class DigitsPrinterSpec : public framework::Task
 /// specs
 o2::framework::DataProcessorSpec getEmcalDigitsPrinterSpec(std::string inputtype);
 
-//using DigitsPrinterSpecDigit = o2::emcal::reco_workflow::DigitsPrinterSpec;
-//using DigitsPrinterSpecCell = o2::emcal::reco_workflow::DigitsPrinterSpec;
+// using DigitsPrinterSpecDigit = o2::emcal::reco_workflow::DigitsPrinterSpec;
+// using DigitsPrinterSpecCell = o2::emcal::reco_workflow::DigitsPrinterSpec;
 } // namespace reco_workflow
 } // namespace emcal
 
 } // namespace o2
+
+#endif // O2_EMCAL_DIGITSPRINTER_SPEC
diff --git a/Detectors/EMCAL/workflow/include/EMCALWorkflow/EMCALDCSDataProcessorSpec.h b/Detectors/EMCAL/workflow/include/EMCALWorkflow/EMCALDCSDataProcessorSpec.h
index 274db888e3b0d..99dd66e20e64d 100644
--- a/Detectors/EMCAL/workflow/include/EMCALWorkflow/EMCALDCSDataProcessorSpec.h
+++ b/Detectors/EMCAL/workflow/include/EMCALWorkflow/EMCALDCSDataProcessorSpec.h
@@ -127,20 +127,27 @@ class EMCALDCSDataProcessor : public o2::framework::Task
   void run(o2::framework::ProcessingContext& pc) final
   {
     TStopwatch sw;
+    static size_t runCount = 0;
+    const size_t logPrescale = 10;
     if (mCheckRunStartStop) {
       const auto* grp = mRunChecker.check(); // check if there is a run with EMC
       // this is an example of what it will return
       if (mRunChecker.getRunStatus() == RunStatus::NONE) {
-        LOGP(info, "No run with is ongoing or finished");
+        if ((runCount % logPrescale) == 0) {
+          LOGP(info, "No run with is ongoing or finished");
+        }
       } else if (mRunChecker.getRunStatus() == RunStatus::START) { // saw new run with wanted detectors
         LOGP(info, "Run {} has started", mRunChecker.getFollowedRun());
         grp->print();
         mProcessor->setRunNumberFromGRP(mRunChecker.getFollowedRun());
       } else if (mRunChecker.getRunStatus() == RunStatus::ONGOING) { // run which was already seen is still ongoing
-        LOGP(info, "Run {} is still ongoing", mRunChecker.getFollowedRun());
+        if ((runCount % logPrescale) == 0) {
+          LOGP(info, "Run {} is still ongoing", mRunChecker.getFollowedRun());
+        }
       } else if (mRunChecker.getRunStatus() == RunStatus::STOP) { // run which was already seen was stopped (EOR seen)
         LOGP(info, "Run {} was stopped", mRunChecker.getFollowedRun());
       }
+      runCount++;
     } else {
       mProcessor->setRunNumberFromGRP(-2);
     }
diff --git a/Detectors/EMCAL/workflow/include/EMCALWorkflow/EMCALDigitizerSpec.h b/Detectors/EMCAL/workflow/include/EMCALWorkflow/EMCALDigitizerSpec.h
index e817fed6105d4..ee50dfb03bb6d 100644
--- a/Detectors/EMCAL/workflow/include/EMCALWorkflow/EMCALDigitizerSpec.h
+++ b/Detectors/EMCAL/workflow/include/EMCALWorkflow/EMCALDigitizerSpec.h
@@ -12,6 +12,7 @@
 #ifndef STEER_DIGITIZERWORKFLOW_EMCALDIGITIZER_H_
 #define STEER_DIGITIZERWORKFLOW_EMCALDIGITIZER_H_
 
+#include 
 #include 
 
 #include "Framework/DataProcessorSpec.h"
@@ -22,13 +23,27 @@
 #include "EMCALSimulation/SDigitizer.h"
 #include "SimulationDataFormat/MCTruthContainer.h"
 #include 
+#include "EMCALSimulation/DigitizerTRU.h"
 
 class TChain;
 
 namespace o2
 {
+
+namespace steer
+{
+class MCKinematicsReader;
+}
+
+namespace ctp
+{
+class CTPConfiguration;
+}
+
 namespace emcal
 {
+class CalibLoader;
+
 /// \brief Create new digitizer spec
 /// \return Digitizer spec
 
@@ -37,11 +52,12 @@ namespace emcal
 /// \author Anders Garritt Knospe , University of Houston
 /// \author Markus Fasel  Oak Ridge National laboratory
 /// \since Nov 12, 2018
-class DigitizerSpec final : public o2::base::BaseDPLDigitizer
+class DigitizerSpec final : public o2::base::BaseDPLDigitizer, public o2::framework::Task
 {
  public:
+  using o2::base::BaseDPLDigitizer::init;
   /// \brief Constructor
-  DigitizerSpec() : o2::base::BaseDPLDigitizer(o2::base::InitServices::GEOM) {}
+  DigitizerSpec(std::shared_ptr calibloader, bool requireCTPInput) : o2::base::BaseDPLDigitizer(o2::base::InitServices::GEOM), o2::framework::Task(), mRequireCTPInput(requireCTPInput), mCalibHandler(calibloader) {}
 
   /// \brief Destructor
   ~DigitizerSpec() final = default;
@@ -50,6 +66,10 @@ class DigitizerSpec final : public o2::base::BaseDPLDigitizer
   /// \param ctx Init context
   void initDigitizerTask(framework::InitContext& ctx) final;
 
+  void finaliseCCDB(o2::framework::ConcreteDataMatcher& matcher, void* obj) final;
+
+  void configure();
+
   /// \brief run digitizer
   /// \param ctx Processing context
   ///
@@ -57,21 +77,31 @@ class DigitizerSpec final : public o2::base::BaseDPLDigitizer
   /// - Open readout window when the event sets a trigger
   /// - Accumulate digits sampled via the time response from different bunch crossings
   /// - Retrieve digits when the readout window closes
-  void run(framework::ProcessingContext& ctx);
+  void run(framework::ProcessingContext& ctx) override;
 
  private:
-  Bool_t mFinished = false;            ///< Flag for digitization finished
-  Digitizer mDigitizer;                ///< Digitizer object
-  o2::emcal::SDigitizer mSumDigitizer; ///< Summed digitizer
-  std::vector mHits;              ///< Vector with input hits
+  Bool_t mFinished = false;                   ///< Flag for digitization finished
+  bool mIsConfigured = false;                 ///< Initialization status of the digitizer
+  bool mRunSDitizer = false;                  ///< Run SDigitization
+  bool mRequireCTPInput = false;              ///< Require CTP min. bias input
+  Digitizer mDigitizer;                       ///< Digitizer object
+  o2::emcal::SDigitizer mSumDigitizer;        ///< Summed digitizer
+  std::shared_ptr mCalibHandler; ///< Handler of calibration objects
+  std::vector mHits;                     ///< Vector with input hits
   std::vector mSimChains;
+  o2::ctp::CTPConfiguration* mCTPConfig; ///< CTP configuration
+  o2::steer::MCKinematicsReader* mcReader; ///< reader to access MC collision information
+
+  DigitizerTRU mDigitizerTRU;             ///< Digitizer object TRU
+  o2::emcal::SDigitizer mSumDigitizerTRU; ///< Summed digitizer TRU
+  bool mRunDigitizerTRU = true;           ///< Run Digitizer TRU?
 };
 
 /// \brief Create new digitizer spec
 /// \return Digitizer spec
-o2::framework::DataProcessorSpec getEMCALDigitizerSpec(int channel, bool mctruth = true);
+o2::framework::DataProcessorSpec getEMCALDigitizerSpec(int channel, bool requireCTPInput, bool mctruth = true, bool useccdb = true);
 
-} // end namespace emcal
+} // namespace emcal
 } // end namespace o2
 
 #endif /* STEER_DIGITIZERWORKFLOW_EMCALDIGITIZER_H_ */
diff --git a/Detectors/EMCAL/workflow/include/EMCALWorkflow/EntropyDecoderSpec.h b/Detectors/EMCAL/workflow/include/EMCALWorkflow/EntropyDecoderSpec.h
index 7a80a459050fc..9cc5ba7887473 100644
--- a/Detectors/EMCAL/workflow/include/EMCALWorkflow/EntropyDecoderSpec.h
+++ b/Detectors/EMCAL/workflow/include/EMCALWorkflow/EntropyDecoderSpec.h
@@ -28,7 +28,7 @@ namespace emcal
 class EntropyDecoderSpec : public o2::framework::Task
 {
  public:
-  EntropyDecoderSpec(int verbosity);
+  EntropyDecoderSpec(int verbosity, unsigned int sspecOut, const std::string& ctfdictOpt = "none");
   ~EntropyDecoderSpec() override = default;
   void run(o2::framework::ProcessingContext& pc) final;
   void init(o2::framework::InitContext& ic) final;
@@ -37,11 +37,12 @@ class EntropyDecoderSpec : public o2::framework::Task
 
  private:
   o2::emcal::CTFCoder mCTFCoder;
+  unsigned int mSSpecOut = 0;
   TStopwatch mTimer;
 };
 
 /// create a processor spec
-framework::DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec);
+framework::DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspecInp, unsigned int sspecOut = 0, const std::string& ctfdictOpt = "none");
 
 } // namespace emcal
 } // namespace o2
diff --git a/Detectors/EMCAL/workflow/include/EMCALWorkflow/EntropyEncoderSpec.h b/Detectors/EMCAL/workflow/include/EMCALWorkflow/EntropyEncoderSpec.h
index 68db6b37d4649..df502beef30df 100644
--- a/Detectors/EMCAL/workflow/include/EMCALWorkflow/EntropyEncoderSpec.h
+++ b/Detectors/EMCAL/workflow/include/EMCALWorkflow/EntropyEncoderSpec.h
@@ -28,7 +28,7 @@ namespace emcal
 class EntropyEncoderSpec : public o2::framework::Task
 {
  public:
-  EntropyEncoderSpec();
+  EntropyEncoderSpec(bool selIR, const std::string& ctfdictOpt = "none");
   ~EntropyEncoderSpec() override = default;
   void run(o2::framework::ProcessingContext& pc) final;
   void init(o2::framework::InitContext& ic) final;
@@ -37,11 +37,12 @@ class EntropyEncoderSpec : public o2::framework::Task
 
  private:
   o2::emcal::CTFCoder mCTFCoder;
+  bool mSelIR = false;
   TStopwatch mTimer;
 };
 
 /// create a processor spec
-framework::DataProcessorSpec getEntropyEncoderSpec();
+framework::DataProcessorSpec getEntropyEncoderSpec(bool selIR = false, const std::string& ctfdictOpt = "none");
 
 } // namespace emcal
 } // namespace o2
diff --git a/Detectors/EMCAL/workflow/include/EMCALWorkflow/OfflineCalibSpec.h b/Detectors/EMCAL/workflow/include/EMCALWorkflow/OfflineCalibSpec.h
new file mode 100644
index 0000000000000..556b4e1040bda
--- /dev/null
+++ b/Detectors/EMCAL/workflow/include/EMCALWorkflow/OfflineCalibSpec.h
@@ -0,0 +1,104 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef O2_EMCAL_OFFLINECALIB_SPEC
+#define O2_EMCAL_OFFLINECALIB_SPEC
+
+#include 
+#include "Framework/DataProcessorSpec.h"
+#include "EMCALWorkflow/CalibLoader.h"
+#include "EMCALCalib/GainCalibrationFactors.h"
+#include "EMCALCalibration/EMCALCalibParams.h"
+#include "DataFormatsCTP/Digits.h"
+#include "DataFormatsCTP/Configuration.h"
+#include "Framework/Task.h"
+
+#include "TFile.h"
+#include "TH1.h"
+#include "TH2.h"
+#include "THnSparse.h"
+
+namespace o2
+{
+
+namespace emcal
+{
+
+/// \class OfflineCalibSpec
+/// \brief Task for producing offline calibration objects
+/// \ingroup EMCALworkflow
+/// \author Hannah Bossi , Yale University
+/// \since August 16th, 2022
+///
+/// This task fills offline calibration objects for the EMCAL.
+class OfflineCalibSpec : public framework::Task
+{
+  using EMCALCalibParams = o2::emcal::EMCALCalibParams;
+
+ public:
+  /// \brief Constructor
+  /// \param makeCellIDTimeEnergy If true the THnSparseF of cell ID, time, and energy is made
+  /// \param rejectCalibTriggers if true, only events which have the o2::trigger::PhT flag will be taken into account
+  OfflineCalibSpec(bool makeCellIDTimeEnergy, bool rejectCalibTriggers, bool rejectL0Trigger, std::shared_ptr calibHandler) : mMakeCellIDTimeEnergy(makeCellIDTimeEnergy), mRejectCalibTriggers(rejectCalibTriggers), mRejectL0Triggers(rejectL0Trigger), mCalibrationHandler(calibHandler){};
+
+  /// \brief Destructor
+  ~OfflineCalibSpec() override = default;
+
+  /// \brief Initializing the offline calib task
+  /// \param ctx Init context
+  void init(framework::InitContext& ctx) final;
+
+  /// Loading CCDB object into internal cache for the 3 supported CCDB entries (bad
+  /// channel map, time calibration params, gain calibration params). Objects are only
+  /// loaded in case the calibration type was enabled.
+  void finaliseCCDB(framework::ConcreteDataMatcher& matcher, void* obj) final;
+
+  /// \brief Fill histograms needed for the offline calibration
+  /// \param ctx Processing context
+  ///
+  void run(framework::ProcessingContext& ctx) final;
+
+  /// \brief Write histograms to an output root file
+  /// \param ec end of stream context
+  ///
+  void endOfStream(o2::framework::EndOfStreamContext& ec) final;
+
+  static const char* getCTPDigitsBinding() { return "CTPDigits"; }
+  static const char* getCTPConfigBinding() { return "CTPConfig"; }
+
+ private:
+  /// \brief Update calibration objects (if changed)
+  void updateCalibObjects();
+  std::unique_ptr mCellAmplitude;              ///< Cell energy vs. cell ID
+  std::unique_ptr mCellTime;                   ///< Cell time vs. cell ID
+  std::unique_ptr mCellTimeLG;                 ///< Cell time vs. cell ID for low gain cells
+  std::unique_ptr mCellTimeHG;                 ///< Cell time vs. cell ID for high gain cells
+  std::unique_ptr mNevents;                    ///< Number of events
+  std::unique_ptr mCellTimeEnergy;      ///< ID, time, energy
+  std::shared_ptr mCalibrationHandler; ///< Handler loading calibration objects
+  GainCalibrationFactors* mGainCalibFactors;        ///< cell gain calibration factors
+  bool mMakeCellIDTimeEnergy = true;                ///< Switch whether or not to make a THnSparseF of cell ID, time, and energy
+  bool mRejectCalibTriggers = true;                 ///< Switch to select if calib triggerred events should be rejected
+  bool mEnableGainCalib = false;                    ///< Switch weather gain calibration should be applied or not when filling the hsitograms
+  bool mRejectL0Triggers = false;                   ///< Switch to select if L0 triggerred events should be rejected
+  std::vector mSelectedClassMasks = {};   ///< vector with selected trigger masks that are accepted for processing
+};
+
+/// \brief Creating offline calib spec
+/// \ingroup EMCALworkflow
+///
+o2::framework::DataProcessorSpec getEmcalOfflineCalibSpec(bool makeCellIDTimeEnergy, bool rejectCalibTriggers, bool rejectL0Trigger, uint32_t inputsubspec, bool enableGainCalib, bool ctpcfgperrun);
+
+} // namespace emcal
+
+} // namespace o2
+
+#endif // O2_EMCAL_OFFLINECALIB_SPEC
diff --git a/Detectors/EMCAL/workflow/include/EMCALWorkflow/PublisherSpec.h b/Detectors/EMCAL/workflow/include/EMCALWorkflow/PublisherSpec.h
index 74f0d7c1663ed..468e66c43a0ab 100644
--- a/Detectors/EMCAL/workflow/include/EMCALWorkflow/PublisherSpec.h
+++ b/Detectors/EMCAL/workflow/include/EMCALWorkflow/PublisherSpec.h
@@ -9,6 +9,9 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 
+#ifndef O2_EMCAL_PUBLISHER_SPEC
+#define O2_EMCAL_PUBLISHER_SPEC
+
 #include "DPLUtils/RootTreeReader.h"
 #include "Framework/DataProcessorSpec.h"
 #include "Framework/DataSpecUtils.h"
@@ -46,7 +49,7 @@ struct PublisherConf {
 };
 
 template 
-framework::DataProcessorSpec getPublisherSpec(PublisherConf const& config, bool propagateMC = true)
+framework::DataProcessorSpec getPublisherSpec(PublisherConf const& config, uint32_t subspec = 0, bool propagateMC = true)
 {
   using Reader = o2::framework::RootTreeReader;
   using Output = o2::framework::Output;
@@ -56,28 +59,27 @@ framework::DataProcessorSpec getPublisherSpec(PublisherConf const& config, bool
   auto mco = o2::framework::DataSpecUtils::asConcreteDataTypeMatcher(config.mcoutput);
 
   // a creator callback for the actual reader instance
-  auto creator = [dto, tro, mco, propagateMC](const char* treename, const char* filename, int nofEvents, Reader::PublishingMode publishingMode, const char* branchname, const char* triggerbranchname, const char* mcbranchname) {
-    constexpr auto persistency = o2::framework::Lifetime::Timeframe;
+  auto creator = [dto, tro, mco, subspec, propagateMC](const char* treename, const char* filename, int nofEvents, Reader::PublishingMode publishingMode, const char* branchname, const char* triggerbranchname, const char* mcbranchname) {
     if (propagateMC) {
       return std::make_shared(treename,
                                       filename,
                                       nofEvents,
                                       publishingMode,
-                                      Output{mco.origin, mco.description, 0, persistency},
+                                      Output{mco.origin, mco.description, subspec},
                                       mcbranchname,
-                                      Reader::BranchDefinition{Output{dto.origin, dto.description, 0, persistency}, branchname},
-                                      Reader::BranchDefinition{Output{tro.origin, tro.description, 0, persistency}, triggerbranchname});
+                                      Reader::BranchDefinition{Output{dto.origin, dto.description, subspec}, branchname},
+                                      Reader::BranchDefinition{Output{tro.origin, tro.description, subspec}, triggerbranchname});
     } else {
       return std::make_shared(treename,
                                       filename,
                                       nofEvents,
                                       publishingMode,
-                                      Reader::BranchDefinition{Output{dto.origin, dto.description, 0, persistency}, branchname},
-                                      Reader::BranchDefinition{Output{tro.origin, tro.description, 0, persistency}, triggerbranchname});
+                                      Reader::BranchDefinition{Output{dto.origin, dto.description, subspec}, branchname},
+                                      Reader::BranchDefinition{Output{tro.origin, tro.description, subspec}, triggerbranchname});
     }
   };
 
-  return createPublisherSpec(config, propagateMC, creator);
+  return createPublisherSpec(config, subspec, propagateMC, creator);
 }
 
 inline framework::DataProcessorSpec getCellReaderSpec(bool propagateMC)
@@ -92,6 +94,7 @@ inline framework::DataProcessorSpec getCellReaderSpec(bool propagateMC)
                                                        o2::framework::OutputSpec{"EMC", "CELLS"},
                                                        o2::framework::OutputSpec{"EMC", "CELLSTRGR"},
                                                        o2::framework::OutputSpec{"EMC", "CELLSMCTR"}},
+                                         0,
                                          propagateMC);
 }
 
@@ -101,7 +104,9 @@ using Reader = o2::framework::RootTreeReader;
 using Creator = std::function(const char*, const char*, int, Reader::PublishingMode, const char*, const char*, const char*)>;
 } // namespace workflow_reader
 
-framework::DataProcessorSpec createPublisherSpec(PublisherConf const& config, bool propagateMC, workflow_reader::Creator creator);
+framework::DataProcessorSpec createPublisherSpec(PublisherConf const& config, uint32_t subspec, bool propagateMC, workflow_reader::Creator creator);
 
 } // namespace emcal
 } // end namespace o2
+
+#endif // O2_EMCAL_PUBLISHER_SPEC
diff --git a/Detectors/EMCAL/workflow/include/EMCALWorkflow/RawToCellConverterSpec.h b/Detectors/EMCAL/workflow/include/EMCALWorkflow/RawToCellConverterSpec.h
index 1fef5ecea12fc..00eb030e470d9 100644
--- a/Detectors/EMCAL/workflow/include/EMCALWorkflow/RawToCellConverterSpec.h
+++ b/Detectors/EMCAL/workflow/include/EMCALWorkflow/RawToCellConverterSpec.h
@@ -9,17 +9,29 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 
+#ifndef O2_EMCAL_RAWTOCELLCONVERTER_SPEC
+#define O2_EMCAL_RAWTOCELLCONVERTER_SPEC
+
 #include 
+#include 
 #include 
 
+#include "Framework/ConcreteDataMatcher.h"
 #include "Framework/DataProcessorSpec.h"
 #include "Framework/Task.h"
 #include "DataFormatsEMCAL/Cell.h"
+#include "DataFormatsEMCAL/CompressedTriggerData.h"
 #include "DataFormatsEMCAL/TriggerRecord.h"
 #include "Headers/DataHeader.h"
 #include "EMCALBase/Geometry.h"
 #include "EMCALBase/Mapper.h"
+#include "EMCALBase/TriggerMappingV2.h"
 #include "EMCALReconstruction/CaloRawFitter.h"
+#include "EMCALReconstruction/RawReaderMemory.h"
+#include "EMCALReconstruction/RecoContainer.h"
+#include "EMCALReconstruction/ReconstructionErrors.h"
+#include "EMCALReconstruction/TRUDecodingErrors.h"
+#include "EMCALWorkflow/CalibLoader.h"
 
 namespace o2
 {
@@ -27,21 +39,93 @@ namespace o2
 namespace emcal
 {
 
+class AltroDecoderError;
+class Channel;
+class MinorAltroDecodingError;
+class RawDecodingError;
+class FastORIndexException;
+class TRUIndexException;
+class FastOrStartTimeInvalidException;
+
 namespace reco_workflow
 {
 
 /// \class RawToCellConverterSpec
-/// \brief Coverter task for Raw data to EMCAL cells
+/// \brief Coverter task for Raw data to EMCAL cells and trigger objects
 /// \author Hadi Hassan , Oak Ridge National Laboratory
+/// \author Markus Fasel , Oak Ridge National Laboratory
+/// \ingroup EMCALworkflow
 /// \since December 10, 2019
+////
+/// General reconstruction task of EMCAL raw data of FEC and trigger sources, running during the synchronous
+/// reconstruction. The task decodees pages from ALTRO, Fake ALTRO and STU sources, and mergees the data according
+/// to events (triggers). The data is then further processed based on the origin of the data:
+///
+/// - in case of FEC data (cells or LEDMONS) a raw fit is performed, and data from high- and low-gain channels are
+///   merged, always preferring the high-gain data if not saturated (better resolution)
+/// - in case of Fake ALTRO data (TRU) the L0 timesums, trigger patches and TRU information are reconstructed. For
+///   the trigger patches a small peak finder selects the time sample with the max. patch energy. For the L0 timesums
+///   a fixed L0 time (8) is used for all event types
+///
+/// FEC and trigger information are ordered according to events and streamed to their output buffers, where corresponding
+/// trigger records mark ranges in the buffer belonging to the same trigger. Tasks subscribing to outputs from this
+/// task must always subscribe to teh trigger records in addition.
+///
+/// Several components of the task (raw parsing, ALTRO decoding, channel mapping and geometry, raw fit) can end in error
+/// states, particularly due to unexpected data. Error handling is performed internally at different stages. Errors are
+/// cathegoriezed as major or minor errors, where major errors prevent decoding the page while minor errors only lead
+/// to loss of certain segments. For monitoring tasks must subscribe to EMC/DECODERERR.
+///
+/// In order to guarantee data consistency a link checker compares the links contributing to data from a certain trigger
+/// to the links activated in the DCS. If not all links are present the timeframe is discarded. In order to switch off
+/// that feature the option --no-checkactivelinks must be activated.
+///
+/// Inputs:
+/// | Input spec           | Optional | CCDB | Purpose                                      |
+/// |----------------------|----------|------|----------------------------------------------|
+/// | EMC/RAWDATA          | no       | no   | EMCAL raw data                               |
+/// | FLP/DISTSUBTIMEFRAME | yes      | no   | Message send when no data was received in TF |
+/// | EMC/RECOPARAM        | no       | yes  | Reconstruction parameters                    |
+/// | EMC/FEEDCS           | no       | yes  | FEE DCS information                          |
+///
+/// Outputs:
+/// | Input spec           | Subspec (default) | Optional  | Purpose                                            |
+/// |----------------------|-------------------|-----------|----------------------------------------------------|
+/// | EMC/CELLS            | 1                 | no        | EMCAL cell (tower) data                            |
+/// | EMC/CELLSTRGR        | 1                 | no        | Trigger records related to cell data               |
+/// | EMC/DECODERERR       | 1                 | yes       | Decoder errors (for QC), if enabled                |
+/// | EMC/TRUS             | 1                 | yes       | TRU information, if trigger reconstruction enabled |
+/// | EMC/TRUSTRGR         | 1                 | yes       | Trigger reconrds related to TRU information        |
+/// | EMC/PATCHES          | 1                 | yes       | Trigger patches, if trigger reconstruction enabled |
+/// | EMC/PATCHESTRGR      | 1                 | yes       | Trigger reconrds related to trigger patches        |
+/// | EMC/FASTORS          | 1                 | yes       | L0 timesums, if trigger reconstruction enabled     |
+/// | EMC/FASTORSTRGR      | 1                 | yes       | Trigger reconrds related to L0 timesums            |
+///
+/// Workflow options (via --EMCALRawToCellConverterSpec ...):
+/// | Option              | Default | Possible values | Purpose                                        |
+/// |---------------------|---------|-----------------|------------------------------------------------|
+/// | fitmethod           | gamma2  | gamma2,standard | Raw fit method                                 |
+/// | maxmessage          | 100     | any int         | Max. amount of error messages on infoLogger    |
+/// | printtrailer        | false   | set (bool)      | Print RCU trailer (for debugging)              |
+/// | no-mergeHGLG        | false   | set (bool)      | Do not merge HG and LG channels for same tower |
+/// | no-checkactivelinks | false   | set (bool)      | Do not check for active links per BC           |
+/// | no-evalpedestal     | false   | set (bool)      | Disable pedestal evaluation                    |
 ///
+/// Global switches of the EMCAL reco workflow related to the RawToCellConverter:
+/// | Option                         | Default | Purpose                                       |
+/// | -------------------------------|---------|-----------------------------------------------|
+/// | disable-decoding-errors        | false   | Disable sending decoding errors               |
+/// | disable-trigger-reconstruction | false   | Disable trigger reconstruction                |
+/// | ignore-dist-stf                | false   | disable subscribing to FLP/DISTSUBTIMEFRAME/0 |
 class RawToCellConverterSpec : public framework::Task
 {
  public:
   /// \brief Constructor
   /// \param subspecification Output subspecification for parallel running on multiple nodes
   /// \param hasDecodingErrors Option to swich on/off creating raw decoding error objects for later monitoring
-  RawToCellConverterSpec(int subspecification, bool hasDecodingErrors) : framework::Task(), mSubspecification(subspecification), mCreateRawDataErrors(hasDecodingErrors){};
+  /// \param hasTriggerReconstruction Perform trigger reconstruction and add trigger-related outputs
+  /// \param calibhandler Calibration object handler
+  RawToCellConverterSpec(int subspecification, bool hasDecodingErrors, bool hasTriggerReconstruction, std::shared_ptr calibhandler) : framework::Task(), mSubspecification(subspecification), mCreateRawDataErrors(hasDecodingErrors), mDoTriggerReconstruction(hasTriggerReconstruction), mCalibHandler(calibhandler){};
 
   /// \brief Destructor
   ~RawToCellConverterSpec() override;
@@ -59,13 +143,26 @@ class RawToCellConverterSpec : public framework::Task
   /// Output cells trigger record: {"EMC", "CELLSTR", 0, Lifetime::Timeframe}
   void run(framework::ProcessingContext& ctx) final;
 
+  /// \brief Handle objects obtained from the CCDB
+  /// \param matcher Matcher providing the CCDB path of the object
+  /// \param obj CCDB object loaded by the CCDB interface
+  void finaliseCCDB(framework::ConcreteDataMatcher& matcher, void* obj) final;
+
   /// \brief Set max number of error messages printed
   /// \param maxMessages Max. amount of messages printed
   ///
   /// Error messages will be suppressed once the maximum is reached
-  void setMaxErrorMessages(int maxMessages) { mMaxErrorMessages = maxMessages; }
+  void setMaxErrorMessages(int maxMessages)
+  {
+    mMaxErrorMessages = maxMessages;
+  }
+
+  /// \brief Set noise threshold for gain type errors
+  /// \param threshold Noise threshold
+  void setNoiseThreshold(int threshold) { mNoiseThreshold = threshold; }
 
-  void setNoiseThreshold(int thresold) { mNoiseThreshold = thresold; }
+  /// \brief Get the noise threshold for gain type errors
+  /// \return Noise threshold
   int getNoiseThreshold() const { return mNoiseThreshold; }
 
   /// \brief Set ID of the subspecification
@@ -83,37 +180,326 @@ class RawToCellConverterSpec : public framework::Task
   header::DataHeader::SubSpecificationType getSubspecification() const { return mSubspecification; }
 
  private:
-  /// \struct RecCellInfo
-  /// \brief Internal bookkeeping for cell double counting
-  ///
-  /// In case the energy is in the overlap region between the
-  /// two digitizers 2 channels exist for the same cell. In this
-  /// case the low gain cells are used above a certain threshold.
-  /// In certain error cases the information from the other digitizer
-  /// might be missing. Such cases must be fitered out, however this
-  /// can be done only after all cells are processed. The overlap information
-  /// needs to be propagated for the filtering but is not part of the
-  /// final cell object
-  struct RecCellInfo {
-    o2::emcal::Cell mCellData; ///< Cell information
-    bool mIsLGnoHG;            ///< Cell has only LG digits
-    bool mHGOutOfRange;        ///< Cell has only HG digits which are out of range
-    int mFecID;                ///< FEC ID of the channel (for monitoring)
-    int mDDLID;                ///< DDL of the channel (for monitoring)
-    int mHWAddressLG;          ///< HW address of LG (for monitoring)
-    int mHWAddressHG;          ///< HW address of HG (for monitoring)
+  /// \class ModuleIndexException
+  /// \brief Exception handling errors in calculation of the absolute module ID
+  class ModuleIndexException : public std::exception
+  {
+   public:
+    /// \enum ModuleType_t
+    /// \brief Type of module raising the exception
+    enum class ModuleType_t {
+      CELL_MODULE,  ///< Cell module type
+      LEDMON_MODULE ///< LEDMON module type
+    };
+
+    /// \brief Constructor for cell indices
+    /// \param moduleIndex Index of the module raising the exception
+    /// \param column Column of the cell
+    /// \param row Row of the cell
+    /// \param columnshifted Shifted column index
+    /// \param rowshifted Shifted row index
+    ModuleIndexException(int moduleIndex, int column, int row, int columnshifted, int rowshifted);
+
+    /// \brief Constructor for LEDMON indices
+    /// \param moduleIndex Index of the module raising the exception
+    ModuleIndexException(int moduleIndex);
+
+    /// \brief Destructor
+    ~ModuleIndexException() noexcept final = default;
+
+    /// \brief Access to error message
+    /// \return Error message
+    const char* what() const noexcept final { return "Invalid cell / LEDMON index"; }
+
+    /// \brief Get type of module raising the exception
+    /// \return Module type
+    ModuleType_t getModuleType() const { return mModuleType; }
+
+    /// \brief Get index of the module raising the exception
+    /// \return Index of the module
+    int getIndex() const { return mIndex; }
+
+    /// \brief Get column raising the exception (cell-case)
+    /// \return Column
+    int getColumn() const { return mColumn; }
+
+    /// \brief Get row raising the exception (cell-case)
+    /// \return Row
+    int getRow() const { return mRow; }
+
+    /// \brief Get shifted column raising the exception (cell-case)
+    /// \return Shifted column
+    int getColumnShifted() const { return mColumnShifted; }
+
+    /// \brief Get shifted row raising the exception (cell-case)
+    /// \return Shifted row
+    int getRowShifted() const { return mRowShifted; }
+
+   private:
+    ModuleType_t mModuleType; ///< Type of the module raising the exception
+    int mIndex = -1;          ///< Index raising the exception
+    int mColumn = -1;         ///< Column of the module (cell-case)
+    int mRow = -1;            ///< Row of the module (cell-case)
+    int mColumnShifted = -1;  ///< shifted column of the module (cell-case)
+    int mRowShifted = -1;     /// << shifted row of the module (cell-case)
+  };
+
+  /// \struct CellTimeCorrection
+  /// \brief  Correction for cell time
+  struct CellTimeCorrection {
+    double mTimeShift; ///< Constant time shift
+    int mBcMod4;       ///< BC-dependent shift
+
+    /// \brief Get the corrected cell time
+    /// \param rawtime Raw time from fit
+    /// \return Corrected time
+    ///
+    /// The time is corrected for an average shift and the BC phase
+    double getCorrectedTime(double rawtime) const { return rawtime - mTimeShift - 25. * mBcMod4; }
+  };
+
+  /// \struct LocalPosition
+  /// \brief Position in the supermodule coordinate system
+  struct LocalPosition {
+    uint16_t mSupermoduleID; ///< Supermodule ID
+    uint16_t mFeeID;         ///< FEE ID
+    uint8_t mColumn;         ///< Column in supermodule
+    uint8_t mRow;            ///< Row in supermodule
   };
+
+  using TRUContainer = std::vector;
+  using PatchContainer = std::vector;
+
+  /// \brief Check if the timeframe is empty
+  /// \param ctx Processing context of timeframe
+  /// \return True if the timeframe is empty, false otherwise
+  ///
+  /// Emtpy timeframes do not have RAWDATA from any physical link in the
+  /// processing context, instead they contain RAWDATA from link 0xDEADBEEF
+  /// and a message in FLP/DISTSUBTIMEFRAME
   bool isLostTimeframe(framework::ProcessingContext& ctx) const;
 
+  /// \brief Update calibration objects
+  void updateCalibrationObjects();
+
+  /// \brief Select cells and put them on the output container
+  /// \param cells Cells to select
+  /// \param isLEDMON Distinction whether input is cell or LEDMON
+  /// \return Number of accepted cells
+  ///
+  /// Cells or LEDMONs are rejected if
+  /// - They have a low gain but no high gain channel
+  /// - They only have a high gain channel which is out of range
+  int bookEventCells(const gsl::span& cells, bool isLELDMON);
+
   /// \brief Send data to output channels
-  /// \param cells Container with output cells for timeframe
-  /// \param triggers Container with trigger records for timeframe
-  /// \param decodingErrors Container with decoding errors for timeframe
+  /// \param ctx target processing context
   ///
   /// Send data to all output channels for the given subspecification. The subspecification
   /// is determined on the fly in the run method and therefore used as parameter. Consumers
   /// must use wildcard subspecification via ConcreteDataTypeMatcher.
-  void sendData(framework::ProcessingContext& ctx, const std::vector& cells, const std::vector& triggers, const std::vector& decodingErrors) const;
+  void sendData(framework::ProcessingContext& ctx) const;
+
+  /// \brief Get absolute Cell ID from column/row in supermodule
+  /// \param supermoduleID Index of the supermodule
+  /// \param column Column of the tower within the supermodule
+  /// \param row Row of the tower within the supermodule
+  /// \return Cell absolute ID
+  /// \throw ModuleIndexException in case of invalid module indices
+  int getCellAbsID(int supermoduleID, int column, int row);
+
+  /// \brief Get the absoulte ID of LEDMON from the module ID in supermodule
+  /// \param supermoduleID Index of the supermodule
+  /// \param module Index of the module within the supermodule
+  /// \return LEDMON absolute ID
+  /// \throw ModuleIndexException in case of invalid module indices
+  int geLEDMONAbsID(int supermoduleID, int module);
+
+  /// \brief Add FEE channel to the current evnet
+  /// \param currentEvent Event to add the channel to
+  /// \param currentchannel Current FEE channel
+  /// \param timeCorrector Handler for correction of the time
+  /// \param position Channel coordinates
+  /// \param chantype Channel type (High Gain, Low Gain, LEDMON)
+  ///
+  /// Performing a raw fit of the bunches in the channel to extract energy and time, and
+  /// adding them to the container for FEE data of the given event.
+  void addFEEChannelToEvent(o2::emcal::EventContainer& currentEvent, const o2::emcal::Channel& currentchannel, const CellTimeCorrection& timeCorrector, const LocalPosition& position, ChannelType_t chantype);
+
+  /// \brief Add TRU channel to the event
+  /// \param currentEvent Event to add the channel to
+  /// \param currentchannel Current TRU channel
+  /// \param position Channel coordinates
+  ///
+  /// TRU channels are encoded in colums:
+  /// - 0-95: FastOR timeseries (time-reversed)
+  /// - 96-105: bitmap with fired patches and TRU header
+  /// The TRU index is taken from the hardware address, while the FastOR index is taken from the
+  /// column number. The TRU and patch times are taken from the time sample in which the corresponding
+  /// bit is found.
+  void addTRUChannelToEvent(o2::emcal::EventContainer& currentEvent, const o2::emcal::Channel& currentchannel, const LocalPosition& position);
+
+  /// @brief Build L0 patches from FastOR time series and TRU data of the current event
+  /// @param currentevent Current event to process
+  /// @return Compressed patches
+  ///
+  /// Only reconstruct patches which were decoded as fired from the raw data. The patch energy and time
+  /// are calculated from the corresponding FastOR time series as the energy and time with the largest
+  /// patch energy extracted from all possible time integrals (see reconstructTriggerPatch)
+  std::tuple buildL0Patches(const EventContainer& currentevent) const;
+
+  /// @brief Build L0 timesums with respect to a given L0 time
+  /// @param currentevent Current event with FastOR time series
+  /// @param l0time L0 time of the event
+  /// @return Container with time series
+  std::vector buildL0Timesums(const o2::emcal::EventContainer& currentevent, uint8_t l0time) const;
+
+  /// \brief Reconstruct trigger patch energy and time from its FastOR time series
+  /// \param fastors FastORs contributing to the patch (only present)
+  /// \return Tuple with 0 - patch ADC, 1 - patch time
+  ///
+  /// For all possible combinations reconstruct the 4-time integral of the patches from
+  /// its contributing FastORs. The patch is reconstructed when the ADC reached its
+  /// maximum. The patch time is the start time of the 4-integral
+  std::tuple reconstructTriggerPatch(const gsl::span fastors) const;
+
+  /// \brief Handling of mapper hardware address errors
+  /// \param error Exception raised by the mapper
+  /// \param ddlID DDL ID of the segment raising the exception
+  /// \param hwaddress Hardware address raising the exception
+  ///
+  /// Errors are printed to the infoLogger until a user-defiened
+  /// threshold is reached. In case the export of decoder errors
+  /// is activated an error object with additional information is
+  /// produced.
+  void handleAddressError(const Mapper::AddressNotFoundException& error, int ddlID, int hwaddress);
+
+  /// \brief Handler function for major ALTRO decoder errors
+  /// \param altroerror Exception raised by the ALTRO decoder
+  /// \param ddlID DDL ID of the segment raising the exception
+  /// \param hwaddress Hardware address raising the exception
+  ///
+  /// Errors are printed to the infoLogger until a user-defiened
+  /// threshold is reached. In case the export of decoder errors
+  /// is activated an error object with additional information is
+  /// produced.
+  void handleAltroError(const o2::emcal::AltroDecoderError& altroerror, int ddlID);
+
+  /// \brief Handler function for minor ALTRO errors
+  /// \param altroerror Minor errors created by the ALTRO decoder
+  /// \param ddlID DDL ID of the segment raising the exception
+  ///
+  /// Errors are printed to the infoLogger until a user-defiened
+  /// threshold is reached. In case the export of decoder errors
+  /// is activated an error object with additional information is
+  /// produced.
+  void handleMinorAltroError(const o2::emcal::MinorAltroDecodingError& altroerror, int ddlID);
+
+  /// \brief Handler function of mapper errors related to invalid DDL
+  /// \param error Exception raised by the mapper
+  /// \param feeID FEE ID (DDL ID) of the segment raising the exception
+  ///
+  /// Errors are printed to the infoLogger until a user-defiened
+  /// threshold is reached. In case the export of decoder errors
+  /// is activated an error object with additional information is
+  /// produced.
+  void handleDDLError(const MappingHandler::DDLInvalid& error, int feeID);
+
+  /// \brief Handler function of errors related to geometry (invalid supermodule / module/ tower ...)
+  /// \param error Geometry exception
+  /// \param supermoduleID Supermodule ID of the exception
+  /// \param cellID Cell (Tower) ID of the exception
+  /// \param hwaddress Hardware address raising the exception
+  /// \param chantype Channel type of the exception
+  ///
+  /// Errors are printed to the infoLogger until a user-defiened
+  /// threshold is reached. In case the export of decoder errors
+  /// is activated an error object with additional information is
+  /// produced.
+  void handleGeometryError(const ModuleIndexException& error, int supermoduleID, int cellID, int hwaddress, ChannelType_t chantype);
+
+  /// \brief Handler function of mapper errors related to invalid DDL
+  /// \param error Exception raised by the mapper
+  /// \param feeID FEE ID (DDL ID) of the segment raising the exception
+  ///
+  /// Errors are printed to the infoLogger until a user-defiened
+  /// threshold is reached. In case the export of decoder errors
+  /// is activated an error object with additional information is
+  /// produced.
+  void handleFitError(const o2::emcal::CaloRawFitter::RawFitterError_t& fiterror, int ddlID, int cellID, int hwaddress);
+
+  /// \brief Handler function for gain type errors
+  /// \param errortype Gain error type
+  /// \param ddlID DDL ID of the segment raising the exception
+  /// \param hwaddress Hardware address raising the exception
+  ///
+  /// Errors are printed to the infoLogger until a user-defiened
+  /// threshold is reached. In case the export of decoder errors
+  /// is activated an error object with additional information is
+  /// produced.
+  void handleGainError(const o2::emcal::reconstructionerrors::GainError_t& errortype, int ddlID, int hwaddress);
+
+  /// \brief Handler function for raw page decoding errors (i.e. header/trailer corruptions)
+  /// \param error Raw page error
+  ///
+  /// Errors are printed to the infoLogger until a user-defiened
+  /// threshold is reached. In case the export of decoder errors
+  /// is activated an error object with additional information is
+  /// produced.
+  void handlePageError(const RawDecodingError& error);
+
+  /// \brief Handler function for minor raw page decoding errors (i.e. header/trailer corruptions)
+  /// \param error Raw page error
+  ///
+  /// Errors are printed to the infoLogger until a user-defiened
+  /// threshold is reached. In case the export of decoder errors
+  /// is activated an error object with additional information is
+  /// produced.
+  void handleMinorPageError(const RawReaderMemory::MinorError& error);
+
+  /// \brief Handler function for FastOR indexing errors
+  /// \param error FastOR index error
+  /// \param linkID DDL raising the exception
+  /// \param indexTRU TRU raising the exception
+  ///
+  /// Errors are printed to the infoLogger until a user-defiened
+  /// threshold is reached. In case the export of decoder errors
+  /// is activated an error object with additional information is
+  /// produced.
+  void handleFastORErrors(const FastORIndexException& error, unsigned int linkID, unsigned int indexTRU);
+
+  /// \brief Handler function for FastOR start time errors
+  /// \param error FastOR index error
+  /// \param linkID DDL raising the exception
+  /// \param indexTRU TRU raising the exception
+  ///
+  /// Errors are printed to the infoLogger until a user-defiened
+  /// threshold is reached. In case the export of decoder errors
+  /// is activated an error object with additional information is
+  /// produced.
+  void handleFastORStartTimeErrors(const FastOrStartTimeInvalidException& e, unsigned int linkID, unsigned int indexTRU);
+
+  /// \brief Handler function patch index exception
+  /// \param error Patch index error
+  /// \param linkID DDL raising the exception
+  /// \param indexTRU TRU raising the exception
+  ///
+  /// Errors are printed to the infoLogger until a user-defiened
+  /// threshold is reached. In case the export of decoder errors
+  /// is activated an error object with additional information is
+  /// produced.
+  void handlePatchError(const TRUDataHandler::PatchIndexException& error, unsigned int linkID, unsigned int indexTRU);
+
+  /// \brief Handler function for TRU index exception
+  /// \param error TRU index error
+  /// \param linkID DDL raising the exception
+  /// \param hwaddress Hardware address of the channel raising the exception
+  ///
+  /// Errors are printed to the infoLogger until a user-defiened
+  /// threshold is reached. In case the export of decoder errors
+  /// is activated an error object with additional information is
+  /// produced.
+  void handleTRUIndexError(const TRUIndexException& error, unsigned int linkID, unsigned int hwaddress);
 
   header::DataHeader::SubSpecificationType mSubspecification = 0;    ///< Subspecification for output channels
   int mNoiseThreshold = 0;                                           ///< Noise threshold in raw fit
@@ -121,25 +507,42 @@ class RawToCellConverterSpec : public framework::Task
   int mErrorMessagesSuppressed = 0;                                  ///< Counter of suppressed error messages
   int mMaxErrorMessages = 100;                                       ///< Max. number of error messages
   bool mMergeLGHG = true;                                            ///< Merge low and high gain cells
+  bool mActiveLinkCheck = true;                                      ///< Run check for active links
   bool mPrintTrailer = false;                                        ///< Print RCU trailer
   bool mDisablePedestalEvaluation = false;                           ///< Disable pedestal evaluation independent of settings in the RCU trailer
   bool mCreateRawDataErrors = false;                                 ///< Create raw data error objects for monitoring
+  bool mDoTriggerReconstruction = false;                             ///< Do trigger reconstruction
   std::chrono::time_point mReferenceTime; ///< Reference time for muting messages
   Geometry* mGeometry = nullptr;                                     ///! mCalibHandler;                        ///< Handler for calibration objects
   std::unique_ptr mMapper = nullptr;                 ///! mTriggerMapping;                 ///! mRawFitter;                         ///! mOutputCells;                                    ///< Container with output cells
-  std::vector mOutputTriggerRecords;                  ///< Container with output cells
+  std::vector mOutputTriggerRecords;                  ///< Container with output trigger records for cells
   std::vector mOutputDecoderErrors;                    ///< Container with decoder errors
+  std::vector mOutputTRUs;                            ///< Compressed output TRU information
+  std::vector mOutputTRUTriggerRecords;               ///< Container with trigger records for TRU data
+  std::vector mOutputPatches;                ///< Compressed trigger patch information
+  std::vector mOutputPatchTriggerRecords;             ///< Container with trigger records for Patch data
+  std::vector mOutputTimesums;                  ///< Compressed L0 timesum information
+  std::vector mOutputTimesumTriggerRecords;           ///< Trigger records for L0 timesum
 };
 
 /// \brief Creating DataProcessorSpec for the EMCAL Cell Converter Spec
+/// \param askDISTSTF Include input spec FLP/DISTSUBTIMEFRAME
+/// \param disableDecodingErrors Obtain reco params from the CCDB
+/// \param disableTriggerReconstruction Do not run trigger reconstruction
+/// \param subspecification Subspecification used in the output spec
 ///
 /// Refer to RawToCellConverterSpec::run for input and output specs
-framework::DataProcessorSpec getRawToCellConverterSpec(bool askDISTSTF, bool disableDecodingErrors, int subspecification);
+framework::DataProcessorSpec getRawToCellConverterSpec(bool askDISTSTF, bool disableDecodingError, bool disableTriggerReconstruction, int subspecification);
 
 } // namespace reco_workflow
 
 } // namespace emcal
 
 } // namespace o2
+
+#endif // O2_EMCAL_RAWTOCELLCONVERTER_SPEC
diff --git a/Detectors/EMCAL/workflow/include/EMCALWorkflow/RecoWorkflow.h b/Detectors/EMCAL/workflow/include/EMCALWorkflow/RecoWorkflow.h
index 2684c4f887bb8..909e356297095 100644
--- a/Detectors/EMCAL/workflow/include/EMCALWorkflow/RecoWorkflow.h
+++ b/Detectors/EMCAL/workflow/include/EMCALWorkflow/RecoWorkflow.h
@@ -48,20 +48,27 @@ enum struct OutputType { Digits,          ///< EMCAL digits
 /// \param propagateMC If true MC labels are propagated to the output files
 /// \param askDISTSTF If true the Raw->Cell converter subscribes to FLP/DISTSUBTIMEFRAME
 /// \param enableDigitsPrinter If true then the simple digits printer is added as dummy task
-/// \param subspecification Subspecification in case of running on different FLPs
+/// \param subspecificationIn Subspecification of input in case of running on different FLPs
+/// \param subspecificationOut Subspecification if output in case of running on different FLPs
 /// \param cfgInput Input objects processed in the workflow
 /// \param cfgOutput Output objects created in the workflow
+/// \param disableRootInput Disable reading from ROOT file (raw mode)
+/// \param disableRootOutput Disable writing ROOT files (sync reco)
+/// \param disableDecodingErrors Diable streaming raw decoding errors (async reco)
+/// \param disableTriggerReconstruction Disable trigger reconstrction
 /// \return EMCAL reconstruction workflow for the configuration provided
 /// \ingroup EMCALwokflow
 framework::WorkflowSpec getWorkflow(bool propagateMC = true,
                                     bool askDISTSTF = true,
                                     bool enableDigitsPrinter = false,
-                                    int subspecification = 0,
+                                    int subspecificationIn = 0,
+                                    int subspecificationOut = 0,
                                     std::string const& cfgInput = "digits",
                                     std::string const& cfgOutput = "clusters",
                                     bool disableRootInput = false,
                                     bool disableRootOutput = false,
-                                    bool disableDecodingErrors = false);
+                                    bool disableDecodingErrors = false,
+                                    bool disableTriggerReconstruction = false);
 } // namespace reco_workflow
 
 } // namespace emcal
diff --git a/Detectors/EMCAL/workflow/src/AnalysisClusterSpec.cxx b/Detectors/EMCAL/workflow/src/AnalysisClusterSpec.cxx
index 9d84a1767c9b6..7b81fec681c14 100644
--- a/Detectors/EMCAL/workflow/src/AnalysisClusterSpec.cxx
+++ b/Detectors/EMCAL/workflow/src/AnalysisClusterSpec.cxx
@@ -60,8 +60,11 @@ template 
 void AnalysisClusterSpec::init(framework::InitContext& ctx)
 {
   o2::base::GRPGeomHelper::instance().setRequest(mGGCCDBRequest);
-  auto& ilctx = ctx.services().get();
-  ilctx.setField(AliceO2::InfoLogger::InfoLoggerContext::FieldName::Detector, "EMC");
+
+  if (ctx.services().active()) {
+    auto& ilctx = ctx.services().get();
+    ilctx.setField(AliceO2::InfoLogger::InfoLoggerContext::FieldName::Detector, "EMC");
+  }
 
   LOG(debug) << "[EMCALClusterizer - init] Initialize clusterizer ...";
 
@@ -135,9 +138,7 @@ void AnalysisClusterSpec::run(framework::ProcessingContext& ctx)
     auto inputEvent = mEventHandler->buildEvent(iev);
 
     mClusterFactory->reset();
-    mClusterFactory->setClustersContainer(inputEvent.mClusters);
-    mClusterFactory->setCellsContainer(Inputs);
-    mClusterFactory->setCellsIndicesContainer(inputEvent.mCellIndices);
+    mClusterFactory->setContainer(inputEvent.mClusters, Inputs, inputEvent.mCellIndices);
 
     //for (const auto& analysisCluster : mClusterFactory) {
     for (int icl = 0; icl < mClusterFactory->getNumberOfClusters(); icl++) {
@@ -147,7 +148,7 @@ void AnalysisClusterSpec::run(framework::ProcessingContext& ctx)
   }
 
   LOG(debug) << "[EMCALClusterizer - run] Writing " << mOutputAnaClusters->size() << " clusters ...";
-  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "ANALYSISCLUSTERS", 0, o2::framework::Lifetime::Timeframe}, *mOutputAnaClusters);
+  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "ANALYSISCLUSTERS", 0}, *mOutputAnaClusters);
 }
 
 o2::framework::DataProcessorSpec o2::emcal::reco_workflow::getAnalysisClusterSpec(bool useDigits)
diff --git a/Detectors/EMCAL/workflow/src/CalibLoader.cxx b/Detectors/EMCAL/workflow/src/CalibLoader.cxx
new file mode 100644
index 0000000000000..a07c649cf9301
--- /dev/null
+++ b/Detectors/EMCAL/workflow/src/CalibLoader.cxx
@@ -0,0 +1,170 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include 
+#include 
+#include "EMCALCalib/CalibDB.h"
+#include "EMCALCalib/BadChannelMap.h"
+#include "EMCALCalib/FeeDCS.h"
+#include "EMCALCalib/EMCALChannelScaleFactors.h"
+#include "EMCALCalib/TempCalibrationParams.h"
+#include "EMCALCalib/TimeCalibrationParams.h"
+#include "EMCALCalib/GainCalibrationFactors.h"
+#include "EMCALReconstruction/RecoParam.h"
+#include "EMCALSimulation/SimParam.h"
+#include "EMCALWorkflow/CalibLoader.h"
+#include "Framework/CCDBParamSpec.h"
+#include "Framework/InputRecord.h"
+#include "TFile.h"
+
+using namespace o2::emcal;
+
+void CalibLoader::defineInputSpecs(std::vector& inputs)
+{
+  if (hasBadChannelMap()) {
+    inputs.push_back({"badChannelMap", o2::header::gDataOriginEMC, "BADCHANNELMAP", 0, o2::framework::Lifetime::Condition, o2::framework::ccdbParamSpec(CalibDB::getCDBPathBadChannelMap())});
+  }
+  if (hasBCMScaleFactors()) {
+    inputs.push_back({"bcmScaleFactors", o2::header::gDataOriginEMC, "BCMSCALEFACTORS", 0, o2::framework::Lifetime::Condition, o2::framework::ccdbParamSpec(CalibDB::getCDBPathChannelScaleFactors())});
+  }
+  if (hasFEEDCS()) {
+    inputs.push_back({"feeDCS", o2::header::gDataOriginEMC, "FEEDCS", 0, o2::framework::Lifetime::Condition, o2::framework::ccdbParamSpec(CalibDB::getCDBPathFeeDCS())});
+  }
+  if (hasTimeCalib()) {
+    inputs.push_back({"timeCalibParams", o2::header::gDataOriginEMC, "TIMECALIBPARAMS", 0, o2::framework::Lifetime::Condition, o2::framework::ccdbParamSpec(CalibDB::getCDBPathTimeCalibrationParams())});
+  }
+  if (hasGainCalib()) {
+    inputs.push_back({"gainCalibParams", o2::header::gDataOriginEMC, "GAINCALIBPARAMS", 0, o2::framework::Lifetime::Condition, o2::framework::ccdbParamSpec(CalibDB::getCDBPathGainCalibrationParams())});
+  }
+  if (hasTemperatureCalib()) {
+    inputs.push_back({"tempCalibParams", o2::header::gDataOriginEMC, "TEMPCALIBPARAMS", 0, o2::framework::Lifetime::Condition, o2::framework::ccdbParamSpec(CalibDB::getCDBPathTemperatureCalibrationParams())});
+  }
+  if (hasRecoParams()) {
+    inputs.push_back({"recoParams", o2::header::gDataOriginEMC, "RECOPARAM", 0, o2::framework::Lifetime::Condition, o2::framework::ccdbParamSpec("EMC/Config/RecoParam")});
+  }
+  if (hasSimParams()) {
+    inputs.push_back({"simParams", o2::header::gDataOriginEMC, "SIMPARAM", 0, o2::framework::Lifetime::Condition, o2::framework::ccdbParamSpec("EMC/Config/SimParam")});
+  }
+}
+
+void CalibLoader::checkUpdates(o2::framework::ProcessingContext& ctx)
+{
+  resetUpdateStatus();
+  if (hasBadChannelMap()) {
+    ctx.inputs().get("badChannelMap");
+  }
+  if (hasBCMScaleFactors()) {
+    ctx.inputs().get("bcmScaleFactors");
+  }
+  if (hasFEEDCS()) {
+    ctx.inputs().get("feeDCS");
+  }
+  if (hasTimeCalib()) {
+    ctx.inputs().get("timeCalibParams");
+  }
+  if (hasGainCalib()) {
+    ctx.inputs().get("gainCalibParams");
+  }
+  if (hasTemperatureCalib()) {
+    ctx.inputs().get("tempCalibParams");
+  }
+  if (hasRecoParams()) {
+    ctx.inputs().get("recoParams");
+  }
+  if (hasSimParams()) {
+    ctx.inputs().get("simParams");
+  }
+}
+
+bool CalibLoader::finalizeCCDB(o2::framework::ConcreteDataMatcher& matcher, void* obj)
+{
+  if (matcher == o2::framework::ConcreteDataMatcher("EMC", "BADCHANNELMAP", 0)) {
+    if (hasBadChannelMap()) {
+      LOG(info) << "New bad channel map loaded";
+      mBadChannelMap = reinterpret_cast(obj);
+      setUpdateBadChannelMap();
+    } else {
+      LOG(error) << "New bad channel map available even though bad channel calibration was not enabled, not loading";
+    }
+    return true;
+  }
+  if (matcher == o2::framework::ConcreteDataMatcher("EMC", "BCMSCALEFACTORS", 0)) {
+    if (hasBCMScaleFactors()) {
+      LOG(info) << "New BCM scale factors loaded";
+      mBCMScaleFactors = reinterpret_cast(obj);
+      setUpdateBCMScaleFactors();
+    } else {
+      LOG(error) << "New BCSM scale factors available even though not enabled, not loading";
+    }
+    return true;
+  }
+  if (matcher == o2::framework::ConcreteDataMatcher("EMC", "FEEDCS", 0)) {
+    if (hasFEEDCS()) {
+      LOG(info) << "New FEE DCS params loaded";
+      mFeeDCS = reinterpret_cast(obj);
+      setUpdateFEEDCS();
+    } else {
+      LOG(error) << "New FEE DCS params available even though FEE DCS was not enabled, not loading";
+    }
+    return true;
+  }
+  if (matcher == o2::framework::ConcreteDataMatcher("EMC", "TIMECALIBPARAMS", 0)) {
+    if (hasTimeCalib()) {
+      LOG(info) << "New time calibration paramset loaded";
+      mTimeCalibParams = reinterpret_cast(obj);
+      setUpdateTimeCalib();
+    } else {
+      LOG(error) << "New time calibration paramset available even though time calibration was not enabled, not loading";
+    }
+    return true;
+  }
+  if (matcher == o2::framework::ConcreteDataMatcher("EMC", "GAINCALIBPARAMS", 0)) {
+    if (hasGainCalib()) {
+      LOG(info) << "New gain calibration paramset loaded";
+      mGainCalibParams = reinterpret_cast(obj);
+      setUpdateGainCalib();
+    } else {
+      LOG(error) << "New gain calibration paramset available even though the gain calibration was not enabled, not loading";
+    }
+    return true;
+  }
+  if (matcher == o2::framework::ConcreteDataMatcher("EMC", "TEMPCALIBPARAMS", 0)) {
+    if (hasTemperatureCalib()) {
+      LOG(info) << "New temperature calibration paramset loaded";
+      mTempCalibParams = reinterpret_cast(obj);
+      setUpdateTemperatureCalib();
+    } else {
+      LOG(error) << "New temperature calibration paramset available even though the temperature calibration was not enabled, not loading";
+    }
+    return true;
+  }
+  if (matcher == o2::framework::ConcreteDataMatcher("EMC", "RECOPARAM", 0)) {
+    if (hasRecoParams()) {
+      LOG(info) << "New reconstruction parameters loaded";
+      mRecoParam = reinterpret_cast(obj);
+      setUpdateRecoParams();
+    } else {
+      LOG(error) << "New reconstruction parameters available even though reconstruction parameters were not requested, not loading";
+    }
+    return true;
+  }
+  if (matcher == o2::framework::ConcreteDataMatcher("EMC", "SIMPARAM", 0)) {
+    if (hasSimParams()) {
+      LOG(info) << "New simulation parameters loaded";
+      mSimParam = reinterpret_cast(obj);
+      setUpdateSimParams();
+    } else {
+      LOG(error) << "New simulation parameters available even though simulation parameters were not requested, not loading";
+    }
+    return true;
+  }
+  return false;
+}
\ No newline at end of file
diff --git a/Detectors/EMCAL/workflow/src/CellConverterSpec.cxx b/Detectors/EMCAL/workflow/src/CellConverterSpec.cxx
index bc944bdc4762b..5fa7353e907e2 100644
--- a/Detectors/EMCAL/workflow/src/CellConverterSpec.cxx
+++ b/Detectors/EMCAL/workflow/src/CellConverterSpec.cxx
@@ -10,10 +10,11 @@
 // or submit itself to any jurisdiction.
 #include 
 #include 
-#include "FairLogger.h"
+#include 
 
 #include "DataFormatsEMCAL/Digit.h"
 #include "EMCALWorkflow/CellConverterSpec.h"
+#include "Framework/CCDBParamSpec.h"
 #include "Framework/ControlService.h"
 #include "Framework/ConfigParamRegistry.h"
 #include "DataFormatsEMCAL/MCLabel.h"
@@ -46,20 +47,27 @@ void CellConverterSpec::init(framework::InitContext& ctx)
   }
   mRawFitter->setAmpCut(0.);
   mRawFitter->setL1Phase(0.);
-  LOG(info) << "Using time shift: " << RecoParam::Instance().getCellTimeShiftNanoSec() << " ns";
 }
 
 void CellConverterSpec::run(framework::ProcessingContext& ctx)
 {
   LOG(debug) << "[EMCALCellConverter - run] called";
+
+  mCalibHandler->checkUpdates(ctx);
+  updateCalibrationObjects();
+
   double timeshift = RecoParam::Instance().getCellTimeShiftNanoSec(); // subtract offset in ns in order to center the time peak around the nominal delay
+  timeshift = int(timeshift / 100) * 100;                             // This is a cheat to make the time multiple of 100, since the digitizer takes the delay only as mutiple of 100
 
   mOutputCells.clear();
   mOutputLabels.clear();
   mOutputTriggers.clear();
   auto digitsAll = ctx.inputs().get>("digits");
   auto triggers = ctx.inputs().get>("triggers");
-  auto truthcont = ctx.inputs().get*>("digitsmctr");
+  std::unique_ptr> truthcont = nullptr;
+  if (mPropagateMC) {
+    truthcont = ctx.inputs().get*>("digitsmctr");
+  }
 
   LOG(debug) << "[EMCALCellConverter - run]  Received " << digitsAll.size() << " digits from " << triggers.size() << " trigger ...";
   int currentstart = mOutputCells.size(), ncellsTrigger = 0;
@@ -69,6 +77,8 @@ void CellConverterSpec::run(framework::ProcessingContext& ctx)
       continue;
     }
 
+    int bcmod4 = (trg.getBCData().bc + RecoParam::Instance().getPhaseBCmod4()) % 4;
+
     gsl::span digits(digitsAll.data() + trg.getFirstEntry(), trg.getNumberOfObjects());
     std::vector> mcLabels;
 
@@ -107,7 +117,7 @@ void CellConverterSpec::run(framework::ProcessingContext& ctx)
           if (fitResults.getTime() < 0) {
             fitResults.setTime(0.);
           }
-          mOutputCells.emplace_back(tower, fitResults.getAmp() * o2::emcal::constants::EMCAL_ADCENERGY, fitResults.getTime() - timeshift, channelType);
+          mOutputCells.emplace_back(tower, fitResults.getAmp() * o2::emcal::constants::EMCAL_ADCENERGY, fitResults.getTime() - timeshift - 25 * bcmod4, channelType);
 
           if (mPropagateMC) {
             Int_t LabelIndex = mOutputLabels.getIndexedSize();
@@ -151,10 +161,25 @@ void CellConverterSpec::run(framework::ProcessingContext& ctx)
     ncellsTrigger = 0;
   }
   LOG(debug) << "[EMCALCellConverter - run] Writing " << mOutputCells.size() << " cells ...";
-  ctx.outputs().snapshot(o2::framework::Output{"EMC", "CELLS", 0, o2::framework::Lifetime::Timeframe}, mOutputCells);
-  ctx.outputs().snapshot(o2::framework::Output{"EMC", "CELLSTRGR", 0, o2::framework::Lifetime::Timeframe}, mOutputTriggers);
+  ctx.outputs().snapshot(o2::framework::Output{"EMC", "CELLS", mSubspecificationOut}, mOutputCells);
+  ctx.outputs().snapshot(o2::framework::Output{"EMC", "CELLSTRGR", mSubspecificationOut}, mOutputTriggers);
   if (mPropagateMC) {
-    ctx.outputs().snapshot(o2::framework::Output{"EMC", "CELLSMCTR", 0, o2::framework::Lifetime::Timeframe}, mOutputLabels);
+    ctx.outputs().snapshot(o2::framework::Output{"EMC", "CELLSMCTR", mSubspecificationOut}, mOutputLabels);
+  }
+}
+
+void CellConverterSpec::finaliseCCDB(o2::framework::ConcreteDataMatcher& matcher, void* obj)
+{
+  if (mCalibHandler->finalizeCCDB(matcher, obj)) {
+    return;
+  }
+}
+
+void CellConverterSpec::updateCalibrationObjects()
+{
+  if (mCalibHandler->hasUpdateRecoParam()) {
+    LOG(info) << "RecoParams updated";
+    o2::emcal::RecoParam::Instance().printKeyValues(true, true);
   }
 }
 
@@ -178,45 +203,81 @@ std::vector CellConverterSpec::digitsToBunches(gsl
   std::vector>* bunchLabels;
   int lasttower = -1;
   // for (auto& dig : digits) {
-  for (const auto& [dig, labels] : boost::combine(digits, mcLabels)) {
-    auto tower = dig.getTower();
-    if (tower != lasttower) {
-      lasttower = tower;
-      if (tower > 20000) {
-        std::cout << "Wrong cell ID " << tower << std::endl;
-      }
-
-      auto onlineindices = mGeometry->getOnlineID(tower);
-      int sruID = std::get<0>(onlineindices);
 
-      auto towerdata = sruDigitContainer[sruID].mChannelsDigits.find(tower);
-      if (towerdata == sruDigitContainer[sruID].mChannelsDigits.end()) {
-        sruDigitContainer[sruID].mChannelsDigits[tower] = {dig.getType(), std::vector(o2::emcal::constants::EMCAL_MAXTIMEBINS), mPropagateMC ? std::vector>(o2::emcal::constants::EMCAL_MAXTIMEBINS) : std::vector>()};
-        bunchDigits = &(sruDigitContainer[sruID].mChannelsDigits[tower].mChannelDigits);
-        memset(bunchDigits->data(), 0, sizeof(o2::emcal::Digit*) * o2::emcal::constants::EMCAL_MAXTIMEBINS);
-        if (mPropagateMC) {
-          bunchLabels = &(sruDigitContainer[sruID].mChannelsDigits[tower].mChannelLabels);
-          memset(bunchLabels->data(), 0, sizeof(gsl::span) * o2::emcal::constants::EMCAL_MAXTIMEBINS);
+  if (!mPropagateMC) {
+    for (const auto& dig : digits) {
+      auto tower = dig.getTower();
+      if (tower != lasttower) {
+        lasttower = tower;
+        if (tower > 20000) {
+          std::cout << "Wrong cell ID " << tower << std::endl;
         }
-      } else {
-        bunchDigits = &(towerdata->second.mChannelDigits);
-        if (mPropagateMC) {
-          bunchLabels = &(towerdata->second.mChannelLabels);
+
+        auto onlineindices = mGeometry->getOnlineID(tower);
+        int sruID = std::get<0>(onlineindices);
+
+        auto towerdata = sruDigitContainer[sruID].mChannelsDigits.find(tower);
+        if (towerdata == sruDigitContainer[sruID].mChannelsDigits.end()) {
+          sruDigitContainer[sruID].mChannelsDigits[tower] = {dig.getType(), std::vector(o2::emcal::constants::EMCAL_MAXTIMEBINS), std::vector>()};
+          bunchDigits = &(sruDigitContainer[sruID].mChannelsDigits[tower].mChannelDigits);
+          memset(bunchDigits->data(), 0, sizeof(o2::emcal::Digit*) * o2::emcal::constants::EMCAL_MAXTIMEBINS);
+        } else {
+          bunchDigits = &(towerdata->second.mChannelDigits);
         }
       }
-    }
 
-    // Get time sample of the digit:
-    // Digitizer stores the time sample in ns, needs to be converted to time sample dividing
-    // by the length of the time sample
-    auto timesample = int(dig.getTimeStamp() / emcal::constants::EMCAL_TIMESAMPLE);
-    if (timesample >= o2::emcal::constants::EMCAL_MAXTIMEBINS) {
-      LOG(error) << "Digit time sample " << timesample << " outside range [0," << o2::emcal::constants::EMCAL_MAXTIMEBINS << "]";
-      continue;
+      // Get time sample of the digit:
+      // Digitizer stores the time sample in ns, needs to be converted to time sample dividing
+      // by the length of the time sample
+      auto timesample = int(dig.getTimeStamp() / emcal::constants::EMCAL_TIMESAMPLE);
+      if (timesample >= o2::emcal::constants::EMCAL_MAXTIMEBINS) {
+        LOG(error) << "Digit time sample " << timesample << " outside range [0," << o2::emcal::constants::EMCAL_MAXTIMEBINS << "]";
+        continue;
+      }
+      (*bunchDigits)[timesample] = &dig;
     }
-    (*bunchDigits)[timesample] = &dig;
-    if (mPropagateMC) {
-      (*bunchLabels)[timesample] = labels;
+  } else {
+
+    for (const auto& [dig, labels] : boost::combine(digits, mcLabels)) {
+      auto tower = dig.getTower();
+      if (tower != lasttower) {
+        lasttower = tower;
+        if (tower > 20000) {
+          std::cout << "Wrong cell ID " << tower << std::endl;
+        }
+
+        auto onlineindices = mGeometry->getOnlineID(tower);
+        int sruID = std::get<0>(onlineindices);
+
+        auto towerdata = sruDigitContainer[sruID].mChannelsDigits.find(tower);
+        if (towerdata == sruDigitContainer[sruID].mChannelsDigits.end()) {
+          sruDigitContainer[sruID].mChannelsDigits[tower] = {dig.getType(), std::vector(o2::emcal::constants::EMCAL_MAXTIMEBINS), mPropagateMC ? std::vector>(o2::emcal::constants::EMCAL_MAXTIMEBINS) : std::vector>()};
+          bunchDigits = &(sruDigitContainer[sruID].mChannelsDigits[tower].mChannelDigits);
+          memset(bunchDigits->data(), 0, sizeof(o2::emcal::Digit*) * o2::emcal::constants::EMCAL_MAXTIMEBINS);
+          if (mPropagateMC) {
+            bunchLabels = &(sruDigitContainer[sruID].mChannelsDigits[tower].mChannelLabels);
+            memset(bunchLabels->data(), 0, sizeof(gsl::span) * o2::emcal::constants::EMCAL_MAXTIMEBINS);
+          }
+        } else {
+          bunchDigits = &(towerdata->second.mChannelDigits);
+          if (mPropagateMC) {
+            bunchLabels = &(towerdata->second.mChannelLabels);
+          }
+        }
+      }
+
+      // Get time sample of the digit:
+      // Digitizer stores the time sample in ns, needs to be converted to time sample dividing
+      // by the length of the time sample
+      auto timesample = int(dig.getTimeStamp() / emcal::constants::EMCAL_TIMESAMPLE);
+      if (timesample >= o2::emcal::constants::EMCAL_MAXTIMEBINS) {
+        LOG(error) << "Digit time sample " << timesample << " outside range [0," << o2::emcal::constants::EMCAL_MAXTIMEBINS << "]";
+        continue;
+      }
+      (*bunchDigits)[timesample] = &dig;
+      if (mPropagateMC) {
+        (*bunchLabels)[timesample] = labels;
+      }
     }
   }
 
@@ -278,7 +339,10 @@ std::vector CellConverterSpec::findBunches(const std::vec
   int itime;
   for (itime = channelDigits.size() - 1; itime >= 0; itime--) {
     auto dig = channelDigits[itime];
-    auto labels = mcLabels[itime];
+    gsl::span labels;
+    if (mPropagateMC) {
+      labels = mcLabels[itime];
+    }
     if (!dig) {
       if (bunchStarted) {
         // we have a bunch which is started and needs to be closed
@@ -390,22 +454,25 @@ int CellConverterSpec::selectMaximumBunch(const gsl::span& bunchvec
   return bunchindex;
 }
 
-o2::framework::DataProcessorSpec o2::emcal::reco_workflow::getCellConverterSpec(bool propagateMC)
+o2::framework::DataProcessorSpec o2::emcal::reco_workflow::getCellConverterSpec(bool propagateMC, int inputSubspec, int outputSubspec)
 {
   std::vector inputs;
   std::vector outputs;
-  inputs.emplace_back("digits", o2::header::gDataOriginEMC, "DIGITS", 0, o2::framework::Lifetime::Timeframe);
-  inputs.emplace_back("triggers", "EMC", "DIGITSTRGR", 0, o2::framework::Lifetime::Timeframe);
-  outputs.emplace_back("EMC", "CELLS", 0, o2::framework::Lifetime::Timeframe);
-  outputs.emplace_back("EMC", "CELLSTRGR", 0, o2::framework::Lifetime::Timeframe);
+  inputs.emplace_back("digits", o2::header::gDataOriginEMC, "DIGITS", inputSubspec, o2::framework::Lifetime::Timeframe);
+  inputs.emplace_back("triggers", "EMC", "DIGITSTRGR", inputSubspec, o2::framework::Lifetime::Timeframe);
+  outputs.emplace_back("EMC", "CELLS", outputSubspec, o2::framework::Lifetime::Timeframe);
+  outputs.emplace_back("EMC", "CELLSTRGR", outputSubspec, o2::framework::Lifetime::Timeframe);
   if (propagateMC) {
-    inputs.emplace_back("digitsmctr", "EMC", "DIGITSMCTR", 0, o2::framework::Lifetime::Timeframe);
-    outputs.emplace_back("EMC", "CELLSMCTR", 0, o2::framework::Lifetime::Timeframe);
+    inputs.emplace_back("digitsmctr", "EMC", "DIGITSMCTR", inputSubspec, o2::framework::Lifetime::Timeframe);
+    outputs.emplace_back("EMC", "CELLSMCTR", outputSubspec, o2::framework::Lifetime::Timeframe);
   }
+  auto calibhandler = std::make_shared();
+  calibhandler->enableRecoParams(true);
+  calibhandler->defineInputSpecs(inputs);
   return o2::framework::DataProcessorSpec{"EMCALCellConverterSpec",
                                           inputs,
                                           outputs,
-                                          o2::framework::adaptFromTask(propagateMC),
+                                          o2::framework::adaptFromTask(propagateMC, inputSubspec, outputSubspec, calibhandler),
                                           o2::framework::Options{
                                             {"fitmethod", o2::framework::VariantType::String, "gamma2", {"Fit method (standard or gamma2)"}}}};
 }
diff --git a/Detectors/EMCAL/workflow/src/CellRecalibratorSpec.cxx b/Detectors/EMCAL/workflow/src/CellRecalibratorSpec.cxx
new file mode 100644
index 0000000000000..3f19b49cd73bd
--- /dev/null
+++ b/Detectors/EMCAL/workflow/src/CellRecalibratorSpec.cxx
@@ -0,0 +1,215 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include 
+#include 
+#include 
+#include "CommonConstants/Triggers.h"
+#include "DataFormatsEMCAL/Cell.h"
+#include "DataFormatsEMCAL/TriggerRecord.h"
+#include "EMCALCalib/BadChannelMap.h"
+#include "EMCALCalib/TimeCalibrationParams.h"
+#include "EMCALCalib/GainCalibrationFactors.h"
+#include "EMCALWorkflow/CellRecalibratorSpec.h"
+#include "Framework/CCDBParamSpec.h"
+
+using namespace o2::emcal;
+
+CellRecalibratorSpec::CellRecalibratorSpec(uint32_t outputspec, LEDEventSettings ledsettings, bool badChannelCalib, bool timeCalib, bool gainCalib, bool isMC, std::shared_ptr calibHandler) : mOutputSubspec(outputspec), mIsMC(isMC), mLEDsettings(ledsettings), mCalibrationHandler(calibHandler)
+{
+  setRunBadChannelCalibration(badChannelCalib);
+  setRunTimeCalibration(timeCalib);
+  setRunGainCalibration(gainCalib);
+}
+
+void CellRecalibratorSpec::init(framework::InitContext& ctx)
+{
+  // Display calibrations which are enabled:
+  LOG(info) << "Bad channel calibration: " << (isRunBadChannlCalibration() ? "enabled" : "disabled");
+  LOG(info) << "Time calibration:        " << (isRunTimeCalibration() ? "enabled" : "disabled");
+  LOG(info) << "Gain calibration:        " << (isRunGainCalibration() ? "enabled" : "disabled");
+  std::string ledsettingstitle;
+  switch (mLEDsettings) {
+    case LEDEventSettings::KEEP:
+      ledsettingstitle = "keep";
+      break;
+    case LEDEventSettings::DROP:
+      ledsettingstitle = "drop";
+      break;
+    case LEDEventSettings::REDIRECT:
+      ledsettingstitle = "redirect to EMC/CELLS/10";
+      break;
+  };
+  LOG(info) << "Handling of LED events: " << ledsettingstitle;
+  if (mIsMC) {
+    LOG(info) << "MC mode - removing labels for masked cells from MC label container";
+  }
+}
+
+void CellRecalibratorSpec::run(framework::ProcessingContext& ctx)
+{
+  auto inputcells = ctx.inputs().get>("cells");
+  auto intputtriggers = ctx.inputs().get>("triggerrecords");
+  LOG(info) << "Received " << inputcells.size() << " cells from " << intputtriggers.size() << " triggers";
+  std::unique_ptr> inputMCLabels = nullptr;
+  std::optional> outputMCLabels;
+  if (mIsMC) {
+    inputMCLabels = ctx.inputs().get*>("cellmclabels");
+    outputMCLabels = std::optional>(o2::dataformats::MCTruthContainer());
+    LOG(info) << "Received " << inputMCLabels->getIndexedSize() << " MC Label entries";
+  }
+
+  mCalibrationHandler->checkUpdates(ctx);
+  updateCalibObjects();
+
+  std::vector outputcells;
+  std::vector outputtriggers;
+
+  // Prepare containers for LED triggers (in case they should be redirected to a different output)
+  std::vector ledcells;
+  std::vector ledtriggers;
+
+  uint32_t currentfirst = outputcells.size(),
+           currentledfirst = ledcells.size();
+  for (const auto& trg : intputtriggers) {
+    if (trg.getTriggerBits() & o2::trigger::Cal) {
+      switch (mLEDsettings) {
+        case LEDEventSettings::KEEP:
+          // LED events stay always uncalibrated
+          writeTrigger(inputcells.subspan(trg.getFirstEntry(), trg.getNumberOfObjects()), trg, outputcells, outputtriggers);
+          continue;
+        case LEDEventSettings::DROP:
+          // LED events will be dropped, simply discard them
+          continue;
+        case LEDEventSettings::REDIRECT: {
+          // LED events stay always uncalibrated
+          writeTrigger(inputcells.subspan(trg.getFirstEntry(), trg.getNumberOfObjects()), trg, ledcells, ledtriggers);
+          continue;
+        }
+      };
+    }
+    if (!trg.getNumberOfObjects()) {
+      outputtriggers.emplace_back(trg.getBCData(), outputcells.size(), trg.getNumberOfObjects()).setTriggerBits(trg.getTriggerBits());
+      continue;
+    }
+    auto [calibratedCells, cellIndices] = mCellRecalibrator.getCalibratedCells(gsl::span(inputcells.data() + trg.getFirstEntry(), trg.getNumberOfObjects()));
+    writeTrigger(calibratedCells, trg, outputcells, outputtriggers);
+    if (mIsMC) {
+      writeMCLabels(*inputMCLabels, outputMCLabels.value(), cellIndices, trg.getFirstEntry());
+    }
+  }
+
+  LOG(info) << "Timeframe: " << inputcells.size() << " cells read, " << outputcells.size() << " cells kept";
+  if (mLEDsettings == LEDEventSettings::REDIRECT) {
+    LOG(info) << "Redirecting " << ledcells.size() << " LED cells from " << ledtriggers.size() << " LED triggers";
+  }
+
+  // send recalibrated objects
+  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "CELLS", mOutputSubspec}, outputcells);
+  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "CELLSTRGR", mOutputSubspec}, outputtriggers);
+  if (outputMCLabels.has_value()) {
+    LOG(info) << "Timeframe: " << inputMCLabels->getIndexedSize() << " label entries read, " << outputMCLabels->getIndexedSize() << " label entries kept";
+    ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "CELLSMCTR", mOutputSubspec}, outputMCLabels.value());
+  }
+  if (mLEDsettings == LEDEventSettings::REDIRECT) {
+    ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "CELLS", 10}, ledcells);
+    ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "CELLSTRGR", 10}, ledtriggers);
+  }
+}
+
+void CellRecalibratorSpec::writeTrigger(const gsl::span selectedCells, const o2::emcal::TriggerRecord& currenttrigger, std::vector& outputcontainer, std::vector& outputtriggers)
+{
+  std::size_t currentfirst = outputcontainer.size();
+  if (selectedCells.size()) {
+    std::copy(selectedCells.begin(), selectedCells.end(), std::back_inserter(outputcontainer));
+  }
+  outputtriggers.emplace_back(currenttrigger.getBCData(), currentfirst, selectedCells.size()).setTriggerBits(currenttrigger.getTriggerBits());
+}
+
+void CellRecalibratorSpec::writeMCLabels(const o2::dataformats::MCTruthContainer& inputlabels, o2::dataformats::MCTruthContainer& outputContainer, const std::vector& keptIndices, int firstindex)
+{
+  for (auto keptindex : keptIndices) {
+    int globalInputIndex = firstindex + keptindex;
+    auto labelsToKeep = inputlabels.getLabels(globalInputIndex);
+    outputContainer.addElements(outputContainer.getIndexedSize(), labelsToKeep);
+  }
+}
+
+void CellRecalibratorSpec::finaliseCCDB(o2::framework::ConcreteDataMatcher& matcher, void* obj)
+{
+  LOG(info) << "Handling new Calibration objects";
+  if (mCalibrationHandler->finalizeCCDB(matcher, obj)) {
+    return;
+  }
+}
+
+void CellRecalibratorSpec::updateCalibObjects()
+{
+  if (isRunBadChannlCalibration()) {
+    if (mCalibrationHandler->hasUpdateBadChannelMap()) {
+      LOG(info) << "updateCalibObjects: Bad channel map changed";
+      mCellRecalibrator.setBadChannelMap(mCalibrationHandler->getBadChannelMap());
+    }
+  }
+  if (isRunTimeCalibration()) {
+    if (mCalibrationHandler->hasUpdateTimeCalib()) {
+      LOG(info) << "updateCalibObjects: Time calib params changed";
+      mCellRecalibrator.setTimeCalibration(mCalibrationHandler->getTimeCalibration());
+    }
+  }
+  if (isRunGainCalibration()) {
+    if (mCalibrationHandler->hasUpdateGainCalib()) {
+      LOG(info) << "updateCalibObjects: Time calib params changed";
+      mCellRecalibrator.setGainCalibration(mCalibrationHandler->getGainCalibration());
+    }
+  }
+}
+
+o2::framework::DataProcessorSpec o2::emcal::getCellRecalibratorSpec(uint32_t inputSubspec, uint32_t outputSubspec, uint32_t ledsettings, bool badChannelCalib, bool timeCalib, bool gainCalib, bool isMC)
+{
+  auto calibhandler = std::make_shared();
+  calibhandler->enableBadChannelMap(badChannelCalib);
+  calibhandler->enableTimeCalib(timeCalib);
+  calibhandler->enableGainCalib(gainCalib);
+  std::vector
+    inputs = {{"cells", o2::header::gDataOriginEMC, "CELLS", inputSubspec, o2::framework::Lifetime::Timeframe},
+              {"triggerrecords", o2::header::gDataOriginEMC, "CELLSTRGR", inputSubspec, o2::framework::Lifetime::Timeframe}};
+  CellRecalibratorSpec::LEDEventSettings taskledsettings = CellRecalibratorSpec::LEDEventSettings::KEEP;
+  switch (ledsettings) {
+    case 0:
+      taskledsettings = CellRecalibratorSpec::LEDEventSettings::KEEP;
+      break;
+    case 1:
+      taskledsettings = CellRecalibratorSpec::LEDEventSettings::DROP;
+      break;
+    case 2:
+      taskledsettings = CellRecalibratorSpec::LEDEventSettings::REDIRECT;
+      break;
+    default:
+      LOG(fatal) << "Undefined handling of LED events";
+  }
+  std::vector outputs = {{o2::header::gDataOriginEMC, "CELLS", outputSubspec, o2::framework::Lifetime::Timeframe},
+                                                    {o2::header::gDataOriginEMC, "CELLSTRGR", outputSubspec, o2::framework::Lifetime::Timeframe}};
+  if (taskledsettings == CellRecalibratorSpec::LEDEventSettings::REDIRECT) {
+    outputs.push_back({o2::header::gDataOriginEMC, "CELLS", 10, o2::framework::Lifetime::Timeframe});
+    outputs.push_back({o2::header::gDataOriginEMC, "CELLSTRGR", 10, o2::framework::Lifetime::Timeframe});
+  }
+  if (isMC) {
+    inputs.push_back({"cellmclabels", o2::header::gDataOriginEMC, "CELLSMCTR", inputSubspec, o2::framework::Lifetime::Timeframe});
+    outputs.push_back({o2::header::gDataOriginEMC, "CELLSMCTR", outputSubspec, o2::framework::Lifetime::Timeframe});
+  }
+  calibhandler->defineInputSpecs(inputs);
+
+  return o2::framework::DataProcessorSpec{"EMCALCellRecalibrator",
+                                          inputs,
+                                          outputs,
+                                          o2::framework::adaptFromTask(outputSubspec, taskledsettings, badChannelCalib, timeCalib, gainCalib, isMC, calibhandler)};
+}
diff --git a/Detectors/EMCAL/workflow/src/ClusterizerSpec.cxx b/Detectors/EMCAL/workflow/src/ClusterizerSpec.cxx
index e044d1051b024..f938d02ce7e3f 100644
--- a/Detectors/EMCAL/workflow/src/ClusterizerSpec.cxx
+++ b/Detectors/EMCAL/workflow/src/ClusterizerSpec.cxx
@@ -24,8 +24,11 @@ using namespace o2::emcal::reco_workflow;
 template 
 void ClusterizerSpec::init(framework::InitContext& ctx)
 {
-  auto& ilctx = ctx.services().get();
-  ilctx.setField(AliceO2::InfoLogger::InfoLoggerContext::FieldName::Detector, "EMC");
+  // Check if InfoLoggerContext is active and if so set the Detector field
+  if (ctx.services().active()) {
+    auto& ilctx = ctx.services().get();
+    ilctx.setField(AliceO2::InfoLogger::InfoLoggerContext::FieldName::Detector, "EMC");
+  }
 
   LOG(debug) << "[EMCALClusterizer - init] Initialize clusterizer ...";
 
@@ -104,11 +107,11 @@ void ClusterizerSpec::run(framework::ProcessingContext& ctx)
     currentStartIndices = mOutputCellDigitIndices->size();
   }
   LOG(debug) << "[EMCALClusterizer - run] Writing " << mOutputClusters->size() << " clusters ...";
-  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "CLUSTERS", 0, o2::framework::Lifetime::Timeframe}, *mOutputClusters);
-  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "INDICES", 0, o2::framework::Lifetime::Timeframe}, *mOutputCellDigitIndices);
+  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "CLUSTERS", 0}, *mOutputClusters);
+  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "INDICES", 0}, *mOutputCellDigitIndices);
 
-  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "CLUSTERSTRGR", 0, o2::framework::Lifetime::Timeframe}, *mOutputTriggerRecord);
-  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "INDICESTRGR", 0, o2::framework::Lifetime::Timeframe}, *mOutputTriggerRecordIndices);
+  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "CLUSTERSTRGR", 0}, *mOutputTriggerRecord);
+  ctx.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginEMC, "INDICESTRGR", 0}, *mOutputTriggerRecordIndices);
   mTimer.Stop();
 }
 
diff --git a/Detectors/EMCAL/workflow/src/DigitsPrinterSpec.cxx b/Detectors/EMCAL/workflow/src/DigitsPrinterSpec.cxx
index ec03aba6ce002..7dcf25b737f66 100644
--- a/Detectors/EMCAL/workflow/src/DigitsPrinterSpec.cxx
+++ b/Detectors/EMCAL/workflow/src/DigitsPrinterSpec.cxx
@@ -14,7 +14,7 @@
 #include 
 #include 
 
-#include "FairLogger.h"
+#include 
 
 #include "Framework/ControlService.h"
 #include "Framework/DataRefUtils.h"
diff --git a/Detectors/EMCAL/workflow/src/EMCALDigitizerSpec.cxx b/Detectors/EMCAL/workflow/src/EMCALDigitizerSpec.cxx
index 13d09faf62a3f..cabdb2c74d818 100644
--- a/Detectors/EMCAL/workflow/src/EMCALDigitizerSpec.cxx
+++ b/Detectors/EMCAL/workflow/src/EMCALDigitizerSpec.cxx
@@ -9,8 +9,10 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 
+#include "EMCALWorkflow/CalibLoader.h"
 #include "EMCALWorkflow/EMCALDigitizerSpec.h"
 #include "CommonConstants/Triggers.h"
+#include "Framework/CCDBParamSpec.h"
 #include "Framework/ConfigParamRegistry.h"
 #include "Framework/ControlService.h"
 #include "Framework/DataProcessorSpec.h"
@@ -23,8 +25,14 @@
 #include 
 
 #include "CommonDataFormat/EvIndex.h"
+#include "DetectorsCommonDataFormats/DetID.h"
 #include "DataFormatsParameters/GRPObject.h"
+#include "DataFormatsCTP/Configuration.h"
+#include "DataFormatsCTP/Digits.h"
 #include "DataFormatsEMCAL/TriggerRecord.h"
+#include "DataFormatsFT0/Digit.h"
+#include "DataFormatsFV0/Digit.h"
+#include 
 
 using namespace o2::framework;
 using SubSpecificationType = o2::framework::DataAllocator::SubSpecificationType;
@@ -45,11 +53,17 @@ void DigitizerSpec::initDigitizerTask(framework::InitContext& ctx)
   // init digitizer
 
   mSumDigitizer.setGeometry(geom);
+  mSumDigitizerTRU.setGeometry(geom);
+  mDigitizerTRU.setGeometry(geom);
 
   if (ctx.options().get("debug-stream")) {
     mDigitizer.setDebugStreaming(true);
+    mDigitizerTRU.setDebugStreaming(true);
+  }
+  // mDigitizer.init();
+  if (ctx.options().get("disable-dig-tru")) {
+    mRunDigitizerTRU = false;
   }
-  mDigitizer.init();
 
   mFinished = false;
 }
@@ -59,11 +73,33 @@ void DigitizerSpec::run(framework::ProcessingContext& ctx)
   if (mFinished) {
     return;
   }
+  if (mCalibHandler) {
+    // Load CCDB object (sim params)
+    mCalibHandler->checkUpdates(ctx);
+  }
+
+  if (!mIsConfigured) {
+    configure();
+    mIsConfigured = true;
+  }
+
+  o2::emcal::SimParam::Instance().printKeyValues(true, true);
+
   mDigitizer.flush();
+  mDigitizerTRU.flush();
 
   // read collision context from input
   auto context = ctx.inputs().get("collisioncontext");
+
+  // get interaction rate ... so that it can be used in digitization
+  auto intRate = context->getDigitizerInteractionRate();
   context->initSimChains(o2::detectors::DetID::EMC, mSimChains);
+
+  // init the MCKinematicsReader from the digitization context
+  if (mcReader == nullptr) {
+    mcReader = new o2::steer::MCKinematicsReader(context.get());
+  }
+
   auto& timesview = context->getEventRecords();
   LOG(debug) << "GOT " << timesview.size() << " COLLISSION TIMES";
 
@@ -75,21 +111,239 @@ void DigitizerSpec::run(framework::ProcessingContext& ctx)
   TStopwatch timer;
   timer.Start();
 
+  auto& eventParts = context->getEventParts();
+
+  // ------------------------------
+  // For the TRIGGER Simulation
+  // ------------------------------
+  // Load the masked fastOr
+  // This impact the acceptance
+  // of the detector, and thus the
+  // overall efficiency of the L0
+  if (mCalibHandler) {
+    mDigitizerTRU.setFEE(mCalibHandler->getFEEDCS());
+    mDigitizerTRU.setMaskedFastOrsInLZERO();
+  }
+
+  // ------------------------------
+  // TRIGGER Simulation
+  // ------------------------------
+  // 1. Run SDigitizer separate loop -> map of vector of SDigits
+  // 2. Run Trigger simulation chain (Digitizer -> Digits writeout buffer -> L0 Simulation)
+  // 3. Run Loop SDigits -> Digits, set live only if the trigger is accepted.
+  //
+  // loop over all composite collisions given from context
+  // (aka loop over all the interaction records)
+  int collisionN = 0;
+  for (int collID = 0; collID < timesview.size(); ++collID) {
+
+    if (mRunDigitizerTRU == false) {
+      break;
+    }
+
+    if (intRate < 10000.) {
+      break;
+    }
+
+    collisionN++;
+
+    mDigitizerTRU.setEventTime(timesview[collID]);
+
+    // for each collision, loop over the constituents event and source IDs
+    // (background signal merging is basically taking place here)
+    for (auto& part : eventParts[collID]) {
+
+      mSumDigitizerTRU.setCurrEvID(part.entryID);
+      mSumDigitizerTRU.setCurrSrcID(part.sourceID);
+
+      // get the hits for this event and this source
+      mHits.clear();
+      context->retrieveHits(mSimChains, "EMCHit", part.sourceID, part.entryID, &mHits);
+
+      // LOG(info) << "DIG TRU For collision " << collID << " eventID " << part.entryID << " found " << mHits.size() << " hits ";
+
+      // std::vector summedLabeledDigits = mSumDigitizerTRU.process(mHits);
+      // std::vector summedDigits;
+      // for (auto labeledsummeddigit : summedLabeledDigits) {
+      //   summedDigits.push_back(labeledsummeddigit.getDigit());
+      // }
+
+      std::vector summedLabeledDigits;
+      std::vector summedDigits;
+      if (mRunSDitizer) {
+        summedLabeledDigits = mSumDigitizerTRU.process(mHits);
+        for (auto labeledsummeddigit : summedLabeledDigits) {
+          summedDigits.push_back(labeledsummeddigit.getDigit());
+        }
+      } else {
+        for (auto& hit : mHits) {
+          summedDigits.emplace_back(hit.GetDetectorID(), hit.GetEnergyLoss(), hit.GetTime());
+        }
+      }
+
+      // call actual digitization procedure
+      mDigitizerTRU.process(summedDigits);
+    }
+  }
+  mDigitizerTRU.printMaskedFastOrsInLZERO();
+  mDigitizerTRU.finish();
+  // Result of the trigger simulation
+  // -> Set of BCs with triggering patches
+  auto emcalTriggers = mDigitizerTRU.getTriggerInputs();
+
+  // Load FIT triggers if not running in self-triggered mode
+  std::vector mbtriggers;
+  if (mRequireCTPInput) {
+    // Hopefully at some point we can replace it by CTP input digits
+    // In case of CTP digits react only to trigger inputs activated in trigger configuration
+    // For the moment react to FIT vertex, cent and semicent triggers
+    ctx.inputs().get("ctpconfig");
+    std::vector inputmasks;
+    for (const auto& trg : mCTPConfig->getCTPClasses()) {
+      if (trg.cluster->maskCluster[o2::detectors::DetID::EMC]) {
+        // Class triggering EMCAL cluster
+        LOG(debug) << "Found trigger class for EMCAL cluster: " << trg.name << " with input mask " << std::bitset<64>(trg.descriptor->getInputsMask());
+        inputmasks.emplace_back(trg.descriptor->getInputsMask());
+      }
+    }
+    unsigned long ft0mask = 0, fv0mask = 0;
+    std::map detInputName2Mask =
+      {{"MVBA", 1}, {"MVOR", 2}, {"MVNC", 4}, {"MVCH", 8}, {"MVIR", 0x10}, {"MT0A", 1}, {"MT0C", 2}, {"MTSC", 4}, {"MTCE", 8}, {"MTVX", 0x10}};
+    // Translation 2022: EMCAL cluster received CTP input masks, need to track it to FIT trigger masks
+    std::map> ctpInput2DetInput = {
+      {"0VBA", {o2::detectors::DetID::FV0, "MVBA"}}, {"0VOR", {o2::detectors::DetID::FV0, "MVOR"}}, {"0VNC", {o2::detectors::DetID::FV0, "MVNC"}}, {"0VCH", {o2::detectors::DetID::FV0, "MVCH"}}, {"0VIR", {o2::detectors::DetID::FV0, "MVIR"}}, {"0T0A", {o2::detectors::DetID::FT0, "MT0A"}}, {"0T0C", {o2::detectors::DetID::FT0, "MT0C"}}, {"0TSC", {o2::detectors::DetID::FT0, "MTSC"}}, {"0TCE", {o2::detectors::DetID::FT0, "MTCE"}}, {"0TVX", {o2::detectors::DetID::FT0, "MTVX"}}};
+    for (const auto& [det, ctpinputs] : mCTPConfig->getDet2InputMap()) {
+      if (!(det == o2::detectors::DetID::FT0 || det == o2::detectors::DetID::FV0 || det == o2::detectors::DetID::CTP)) {
+        continue;
+      }
+      for (const auto& input : ctpinputs) {
+        LOG(debug) << "CTP det input: " << input.name << " with mask " << std::bitset<64>(input.inputMask);
+        bool isSelected = false;
+        for (auto testmask : inputmasks) {
+          if (testmask & input.inputMask) {
+            isSelected = true;
+          }
+        }
+        if (isSelected) {
+          std::string usedInputName = input.name;
+          o2::detectors::DetID usedDetID = det;
+          if (det == o2::detectors::DetID::CTP) {
+            auto found = ctpInput2DetInput.find(input.name);
+            if (found != ctpInput2DetInput.end()) {
+              usedInputName = found->second.second;
+              usedDetID = found->second.first;
+              LOG(debug) << "Decoded " << input.name << " -> " << usedInputName;
+            }
+          }
+          auto maskFound = detInputName2Mask.find(usedInputName);
+          if (maskFound != detInputName2Mask.end()) {
+            if (usedDetID == o2::detectors::DetID::FT0) {
+              ft0mask |= maskFound->second;
+            } else {
+              fv0mask |= maskFound->second;
+            }
+          }
+        }
+      }
+    }
+    LOG(debug) << "FTO mask: " << std::bitset<64>(ft0mask);
+    LOG(debug) << "FVO mask: " << std::bitset<64>(fv0mask);
+    for (const auto& trg : ctx.inputs().get>("ft0inputs")) {
+      if (trg.mInputs.to_ulong() & ft0mask) {
+        mbtriggers.emplace_back(trg.mIntRecord);
+      }
+    }
+    for (const auto& trg : ctx.inputs().get>("fv0inputs")) {
+      if (trg.mInputs.to_ulong() & fv0mask) {
+        if (std::find(mbtriggers.begin(), mbtriggers.end(), trg.mIntRecord) == mbtriggers.end()) {
+          mbtriggers.emplace_back(trg.mIntRecord);
+        }
+      }
+    }
+  }
+
   LOG(info) << " CALLING EMCAL DIGITIZATION ";
   o2::dataformats::MCTruthContainer labelAccum;
 
-  auto& eventParts = context->getEventParts();
+  // auto& eventParts = context->getEventParts();
+  std::vector>> acceptedTriggers;
+  enum EMCALTriggerBits { kMB,
+                          kEMC,
+                          kDMC }; // EMCAL trigger bits enum for CTP inputs
+  TRandom3 mRandomGenerator(std::chrono::high_resolution_clock::now().time_since_epoch().count());
 
   // loop over all composite collisions given from context
   // (aka loop over all the interaction records)
   for (int collID = 0; collID < timesview.size(); ++collID) {
 
-    mDigitizer.setEventTime(timesview[collID]);
+    std::bitset<5> trigger{0x1}; // Default: Self-triggered mode - all collisions treated as trigger
+    if (mRequireCTPInput) {
+      // ====================================================
+      // check if we have a trigger input from CTP
+      // Simulated L0 and downsampling
+      // 2 cases -> Check from CTP config
+      // 1) EMCAL trigger active -> MB downsampled
+      //    - Accept the event in 2 conditions
+      //      + MB downsampling (first condition) -> Set trigger bit kTVXinEMC
+      //      + EMCAL L0 trigger -> Set bit EMCAL L0
+      //      Always test both, set trigger 0x1 -> EMBA, 0x2 -> 0EMC, 0x4 -> 0DCMX, 0 -> No trigger
+      // 2) EMCAL triggers not active
+      // .  - Old logics kept, no downscaling, pure busy
+      // ----------------------------------------------------
+      // PRNG for downscaling check
+      auto mbtrigger = std::find(mbtriggers.begin(), mbtriggers.end(), timesview[collID]);
+      if (mbtrigger != mbtriggers.end()) {
 
-    if (!mDigitizer.isLive()) {
+        // ============================
+        // retrieve downscaling from
+        // configuration of CTP classes
+        int downscaling = 0;
+        for (const auto& trg : mCTPConfig->getCTPClasses()) {
+          if (trg.cluster->maskCluster[o2::detectors::DetID::EMC]) {
+            // Class triggering EMCAL cluster
+            downscaling = trg.downScale;
+          }
+        }
+        if (mRandomGenerator.Uniform(0., 1) < downscaling) {
+          // accept as minimum bias
+          trigger.set(EMCALTriggerBits::kMB, true);
+        }
+        // check for LO triggers in 12 BCs
+        for (auto emcalTrigger : emcalTriggers) {
+          auto bcTimingOfEmcalTrigger = emcalTrigger.mInterRecord.bc;
+          auto bcTimingOfMBTrigger = (*mbtrigger).bc;
+          if (std::abs(bcTimingOfEmcalTrigger - bcTimingOfMBTrigger) < 12) {
+            if (emcalTrigger.mTriggeredTRU < 32) {
+              trigger.set(EMCALTriggerBits::kEMC, true);
+            } else {
+              trigger.set(EMCALTriggerBits::kDMC, true);
+            }
+          }
+        }
+
+      } else {
+        // No trigger active
+        // Set busy
+        trigger.set(EMCALTriggerBits::kMB, false);
+      }
+    }
+    // Bitset
+    // Trigger sim: Select event
+    if (!trigger.any()) {
       continue;
     }
 
+    mDigitizer.setEventTime(timesview[collID], trigger.any());
+    if (!mDigitizer.isCurrentEventTriggered()) {
+      LOG(debug) << "reject collision";
+      continue;
+    }
+    LOG(debug) << "accept collision";
+
+    // Trigger sim: Prepare CTP input digit
+    acceptedTriggers.push_back(std::make_tuple(timesview[collID], trigger));
+    LOG(debug) << "EMCAL TRU simulation: Sending trg = " << trigger << " to CTP";
+
     // for each collision, loop over the constituents event and source IDs
     // (background signal merging is basically taking place here)
     for (auto& part : eventParts[collID]) {
@@ -97,13 +351,28 @@ void DigitizerSpec::run(framework::ProcessingContext& ctx)
       mSumDigitizer.setCurrEvID(part.entryID);
       mSumDigitizer.setCurrSrcID(part.sourceID);
 
+      // retrieve information about the MC collision via the MCEventHeader
+      auto& mcEventHeader = mcReader->getMCEventHeader(part.sourceID, part.entryID);
+      mcEventHeader.print();
+
       // get the hits for this event and this source
       mHits.clear();
       context->retrieveHits(mSimChains, "EMCHit", part.sourceID, part.entryID, &mHits);
 
       LOG(info) << "For collision " << collID << " eventID " << part.entryID << " found " << mHits.size() << " hits ";
 
-      std::vector summedDigits = mSumDigitizer.process(mHits);
+      std::vector summedDigits;
+      if (mRunSDitizer) {
+        summedDigits = mSumDigitizer.process(mHits);
+      } else {
+        for (auto& hit : mHits) {
+          o2::emcal::MCLabel digitlabel(hit.GetTrackID(), part.entryID, part.sourceID, false, 1.);
+          if (hit.GetEnergyLoss() < __DBL_EPSILON__) {
+            digitlabel.setAmplitudeFraction(0);
+          }
+          summedDigits.emplace_back(hit.GetDetectorID(), hit.GetEnergyLoss(), hit.GetTime(), digitlabel);
+        }
+      }
 
       // call actual digitization procedure
       mDigitizer.process(summedDigits);
@@ -113,15 +382,42 @@ void DigitizerSpec::run(framework::ProcessingContext& ctx)
   mDigitizer.finish();
 
   // here we have all digits and we can send them to consumer (aka snapshot it onto output)
-  ctx.outputs().snapshot(Output{"EMC", "DIGITS", 0, Lifetime::Timeframe}, mDigitizer.getDigits());
-  ctx.outputs().snapshot(Output{"EMC", "TRGRDIG", 0, Lifetime::Timeframe}, mDigitizer.getTriggerRecords());
+  ctx.outputs().snapshot(Output{"EMC", "DIGITS", 0}, mDigitizer.getDigits());
+  ctx.outputs().snapshot(Output{"EMC", "TRGRDIG", 0}, mDigitizer.getTriggerRecords());
   if (ctx.outputs().isAllowed({"EMC", "DIGITSMCTR", 0})) {
-    ctx.outputs().snapshot(Output{"EMC", "DIGITSMCTR", 0, Lifetime::Timeframe}, mDigitizer.getMCLabels());
+    ctx.outputs().snapshot(Output{"EMC", "DIGITSMCTR", 0}, mDigitizer.getMCLabels());
   }
   // EMCAL is always a triggering detector
   const o2::parameters::GRPObject::ROMode roMode = o2::parameters::GRPObject::TRIGGERING;
   LOG(info) << "EMCAL: Sending ROMode= " << roMode << " to GRPUpdater";
-  ctx.outputs().snapshot(Output{"EMC", "ROMode", 0, Lifetime::Timeframe}, roMode);
+  ctx.outputs().snapshot(Output{"EMC", "ROMode", 0}, roMode);
+  // Create CTP digits
+  std::vector triggerinputs;
+  // for (auto& trg : mDigitizer.getTriggerRecords()) {
+  //   // covert TriggerRecord into CTP trigger digit
+  //   o2::ctp::CTPInputDigit nextdigit;
+  //   nextdigit.intRecord = trg.getBCData();
+  //   nextdigit.detector = o2::detectors::DetID::EMC;
+  //   // Set min. bias accept trigger (input 0) as fake trigger
+  //   // Other inputs will be added once available
+  //   nextdigit.inputsMask.set(0);
+  //   triggerinputs.push_back(nextdigit);
+  // }
+  for (auto& trg : acceptedTriggers) {
+    // convert TriggerRecord into CTP trigger digit
+    o2::ctp::CTPInputDigit nextdigit;
+    nextdigit.intRecord = std::get<0>(trg);
+    nextdigit.detector = o2::detectors::DetID::EMC;
+    // nextdigit.inputsMask = std::get<1>(trg);
+    for (size_t i = 0; i < nextdigit.inputsMask.size(); i++) {
+      if (std::get<1>(trg)[i] != 0) {
+        nextdigit.inputsMask.set(i);
+      }
+    }
+    LOG(debug) << "EMCAL TRU simulation: assigned  = " << nextdigit.inputsMask << " as nextdigit for CTP, with IR = " << nextdigit.intRecord.bc << ", orbit = " << nextdigit.intRecord.orbit;
+    triggerinputs.push_back(nextdigit);
+  }
+  ctx.outputs().snapshot(Output{"EMC", "TRIGGERINPUT", 0}, triggerinputs);
 
   timer.Stop();
   LOG(info) << "Digitization took " << timer.CpuTime() << "s";
@@ -130,7 +426,41 @@ void DigitizerSpec::run(framework::ProcessingContext& ctx)
   mFinished = true;
 }
 
-o2::framework::DataProcessorSpec getEMCALDigitizerSpec(int channel, bool mctruth)
+void DigitizerSpec::configure()
+{
+  mDigitizer.init();
+  mDigitizerTRU.init();
+}
+
+void DigitizerSpec::finaliseCCDB(o2::framework::ConcreteDataMatcher& matcher, void* obj)
+{
+  if (mCalibHandler->finalizeCCDB(matcher, obj)) {
+    return;
+  }
+  if (matcher == o2::framework::ConcreteDataMatcher("CTP", "CTPCONFIG", 0)) {
+    std::cout << "Loading CTP configuration" << std::endl;
+    mCTPConfig = reinterpret_cast(obj);
+    for (const auto& trg : mCTPConfig->getCTPClasses()) {
+      if (trg.cluster->maskCluster[o2::detectors::DetID::EMC]) {
+        // Class triggering EMCAL cluster
+        LOG(debug) << "ENABLING Trigger simulation, found trigger class for EMCAL cluster: " << trg.name << " with input mask " << std::bitset<64>(trg.descriptor->getInputsMask());
+        for (const auto& [det, ctpinputs] : mCTPConfig->getDet2InputMap()) {
+          if (!(det == o2::detectors::DetID::EMC)) {
+            continue;
+          }
+          // if the detector ID is EMC AND the input mask is not empty, run the trigger simulation
+          for (const auto& input : ctpinputs) {
+            if (input.inputMask != 0) {
+              mRunDigitizerTRU = true;
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+o2::framework::DataProcessorSpec getEMCALDigitizerSpec(int channel, bool requireCTPInput, bool mctruth, bool useccdb)
 {
   // create the full data processor spec using
   //  a name identifier
@@ -144,13 +474,31 @@ o2::framework::DataProcessorSpec getEMCALDigitizerSpec(int channel, bool mctruth
     outputs.emplace_back("EMC", "DIGITSMCTR", 0, Lifetime::Timeframe);
   }
   outputs.emplace_back("EMC", "ROMode", 0, Lifetime::Timeframe);
+  outputs.emplace_back("EMC", "TRIGGERINPUT", 0, Lifetime::Timeframe);
+
+  std::vector inputs;
+  inputs.emplace_back("collisioncontext", "SIM", "COLLISIONCONTEXT", static_cast(channel), Lifetime::Timeframe);
+  std::shared_ptr calibloader;
+  if (useccdb) {
+    calibloader = std::make_shared();
+    calibloader->enableSimParams(true);
+    calibloader->enableFEEDCS(true);
+    calibloader->defineInputSpecs(inputs);
+  }
+  if (requireCTPInput) {
+    inputs.emplace_back("ft0inputs", "FT0", "TRIGGERINPUT", 0, Lifetime::Timeframe);
+    inputs.emplace_back("fv0inputs", "FV0", "TRIGGERINPUT", 0, Lifetime::Timeframe);
+    inputs.emplace_back("ctpconfig", "CTP", "CTPCONFIG", 0, Lifetime::Condition, ccdbParamSpec("CTP/Config/Config", true));
+  }
 
   return DataProcessorSpec{
-    "EMCALDigitizer", Inputs{InputSpec{"collisioncontext", "SIM", "COLLISIONCONTEXT", static_cast(channel), Lifetime::Timeframe}},
+    "EMCALDigitizer", // Inputs{InputSpec{"collisioncontext", "SIM", "COLLISIONCONTEXT", static_cast(channel), Lifetime::Timeframe}, InputSpec{"EMC_SimParam", o2::header::gDataOriginEMC, "SIMPARAM", 0, Lifetime::Condition, ccdbParamSpec("EMC/Config/SimParam")}},
+    inputs,
     outputs,
-    AlgorithmSpec{o2::framework::adaptFromTask()},
+    AlgorithmSpec{o2::framework::adaptFromTask(calibloader, requireCTPInput)},
     Options{
       {"pileup", VariantType::Int, 1, {"whether to run in continuous time mode"}},
+      {"disable-dig-tru", VariantType::Bool, false, {"Disable TRU digitisation"}},
       {"debug-stream", VariantType::Bool, false, {"Enable debug streaming"}}}
     // I can't use VariantType::Bool as it seems to have a problem
   };
diff --git a/Detectors/EMCAL/workflow/src/EntropyDecoderSpec.cxx b/Detectors/EMCAL/workflow/src/EntropyDecoderSpec.cxx
index 0d36668fd244f..ecc0e45492bea 100644
--- a/Detectors/EMCAL/workflow/src/EntropyDecoderSpec.cxx
+++ b/Detectors/EMCAL/workflow/src/EntropyDecoderSpec.cxx
@@ -24,12 +24,13 @@ namespace o2
 {
 namespace emcal
 {
-
-EntropyDecoderSpec::EntropyDecoderSpec(int verbosity) : mCTFCoder(o2::ctf::CTFCoderBase::OpType::Decoder)
+EntropyDecoderSpec::EntropyDecoderSpec(int verbosity, unsigned int sspecOut, const std::string& ctfdictOpt) : mCTFCoder(o2::ctf::CTFCoderBase::OpType::Decoder, ctfdictOpt), mSSpecOut(sspecOut)
 {
   mTimer.Stop();
   mTimer.Reset();
   mCTFCoder.setVerbosity(verbosity);
+  mCTFCoder.setSupportBCShifts(true);
+  mCTFCoder.setDictBinding("ctfdict_EMC");
 }
 
 void EntropyDecoderSpec::finaliseCCDB(o2::framework::ConcreteDataMatcher& matcher, void* obj)
@@ -50,11 +51,11 @@ void EntropyDecoderSpec::run(ProcessingContext& pc)
   mTimer.Start(false);
   o2::ctf::CTFIOSize iosize;
 
-  mCTFCoder.updateTimeDependentParams(pc);
-  auto buff = pc.inputs().get>("ctf");
+  mCTFCoder.updateTimeDependentParams(pc, true);
+  auto buff = pc.inputs().get>("ctf_EMC");
 
-  auto& triggers = pc.outputs().make>(OutputRef{"triggers"});
-  auto& cells = pc.outputs().make>(OutputRef{"cells"});
+  auto& triggers = pc.outputs().make>(OutputRef{"triggers", mSSpecOut});
+  auto& cells = pc.outputs().make>(OutputRef{"cells", mSSpecOut});
 
   // since the buff is const, we cannot use EncodedBlocks::relocate directly, instead we wrap its data to another flat object
   if (buff.size()) {
@@ -72,24 +73,27 @@ void EntropyDecoderSpec::endOfStream(EndOfStreamContext& ec)
        mTimer.CpuTime(), mTimer.RealTime(), mTimer.Counter() - 1);
 }
 
-DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec)
+DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspecInp, unsigned int sspecOut, const std::string& ctfdictOpt)
 {
   std::vector outputs{
-    OutputSpec{{"triggers"}, "EMC", "CELLSTRGR", 0, Lifetime::Timeframe},
-    OutputSpec{{"cells"}, "EMC", "CELLS", 0, Lifetime::Timeframe},
+    OutputSpec{{"triggers"}, "EMC", "CELLSTRGR", sspecOut, Lifetime::Timeframe},
+    OutputSpec{{"cells"}, "EMC", "CELLS", sspecOut, Lifetime::Timeframe},
     OutputSpec{{"ctfrep"}, "EMC", "CTFDECREP", 0, Lifetime::Timeframe}};
 
   std::vector inputs;
-  inputs.emplace_back("ctf", "EMC", "CTFDATA", sspec, Lifetime::Timeframe);
-  inputs.emplace_back("ctfdict", "EMC", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("EMC/Calib/CTFDictionary"));
+  inputs.emplace_back("ctf_EMC", "EMC", "CTFDATA", sspecInp, Lifetime::Timeframe);
+
+  if (ctfdictOpt.empty() || ctfdictOpt == "ccdb") {
+    inputs.emplace_back("ctfdict_EMC", "EMC", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("EMC/Calib/CTFDictionaryTree"));
+  }
+  inputs.emplace_back("trigoffset", "CTP", "Trig_Offset", 0, Lifetime::Condition, ccdbParamSpec("CTP/Config/TriggerOffsets"));
 
   return DataProcessorSpec{
     "emcal-entropy-decoder",
     inputs,
     outputs,
-    AlgorithmSpec{adaptFromTask(verbosity)},
-    Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}}};
+    AlgorithmSpec{adaptFromTask(verbosity, sspecOut, ctfdictOpt)},
+    Options{{"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}};
 }
-
 } // namespace emcal
 } // namespace o2
diff --git a/Detectors/EMCAL/workflow/src/EntropyEncoderSpec.cxx b/Detectors/EMCAL/workflow/src/EntropyEncoderSpec.cxx
index 1b6835d593cc7..2928a71a167bc 100644
--- a/Detectors/EMCAL/workflow/src/EntropyEncoderSpec.cxx
+++ b/Detectors/EMCAL/workflow/src/EntropyEncoderSpec.cxx
@@ -25,8 +25,7 @@ namespace o2
 {
 namespace emcal
 {
-
-EntropyEncoderSpec::EntropyEncoderSpec() : mCTFCoder(o2::ctf::CTFCoderBase::OpType::Encoder)
+EntropyEncoderSpec::EntropyEncoderSpec(bool selIR, const std::string& ctfdictOpt) : mCTFCoder(o2::ctf::CTFCoderBase::OpType::Encoder, ctfdictOpt), mSelIR(selIR)
 {
   mTimer.Stop();
   mTimer.Reset();
@@ -48,14 +47,20 @@ void EntropyEncoderSpec::run(ProcessingContext& pc)
 {
   auto cput = mTimer.CpuTime();
   mTimer.Start(false);
-  mCTFCoder.updateTimeDependentParams(pc);
+  mCTFCoder.updateTimeDependentParams(pc, true);
   auto triggers = pc.inputs().get>("triggers");
   auto cells = pc.inputs().get>("cells");
 
-  auto& buffer = pc.outputs().make>(Output{"EMC", "CTFDATA", 0, Lifetime::Timeframe});
+  auto& buffer = pc.outputs().make>(Output{"EMC", "CTFDATA", 0});
+  if (mSelIR) {
+    mCTFCoder.setSelectedIRFrames(pc.inputs().get>("selIRFrames"));
+  }
   auto iosize = mCTFCoder.encode(buffer, triggers, cells);
   pc.outputs().snapshot({"ctfrep", 0}, iosize);
   mTimer.Stop();
+  if (mSelIR) {
+    mCTFCoder.getIRFramesSelector().clear();
+  }
   LOG(info) << iosize.asString() << " in " << mTimer.CpuTime() - cput << " s";
 }
 
@@ -65,23 +70,28 @@ void EntropyEncoderSpec::endOfStream(EndOfStreamContext& ec)
        mTimer.CpuTime(), mTimer.RealTime(), mTimer.Counter() - 1);
 }
 
-DataProcessorSpec getEntropyEncoderSpec()
+DataProcessorSpec getEntropyEncoderSpec(bool selIR, const std::string& ctfdictOpt)
 {
   std::vector inputs;
   inputs.emplace_back("triggers", "EMC", "CELLSTRGR", 0, Lifetime::Timeframe);
   inputs.emplace_back("cells", "EMC", "CELLS", 0, Lifetime::Timeframe);
-  inputs.emplace_back("ctfdict", "EMC", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("EMC/Calib/CTFDictionary"));
 
+  if (ctfdictOpt.empty() || ctfdictOpt == "ccdb") {
+    inputs.emplace_back("ctfdict", "EMC", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("EMC/Calib/CTFDictionaryTree"));
+  }
+  if (selIR) {
+    inputs.emplace_back("selIRFrames", "CTF", "SELIRFRAMES", 0, Lifetime::Timeframe);
+  }
   return DataProcessorSpec{
     "emcal-entropy-encoder",
     inputs,
     Outputs{{"EMC", "CTFDATA", 0, Lifetime::Timeframe},
             {{"ctfrep"}, "EMC", "CTFENCREP", 0, Lifetime::Timeframe}},
-    AlgorithmSpec{adaptFromTask()},
-    Options{
-      {"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}},
-      {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}}}};
+    AlgorithmSpec{adaptFromTask(selIR, ctfdictOpt)},
+    Options{{"irframe-margin-bwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame lower boundary when selection is requested"}},
+            {"irframe-margin-fwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame upper boundary when selection is requested"}},
+            {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}},
+            {"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}};
 }
-
 } // namespace emcal
 } // namespace o2
diff --git a/Detectors/EMCAL/workflow/src/OfflineCalibSpec.cxx b/Detectors/EMCAL/workflow/src/OfflineCalibSpec.cxx
new file mode 100644
index 0000000000000..4ab574793876d
--- /dev/null
+++ b/Detectors/EMCAL/workflow/src/OfflineCalibSpec.cxx
@@ -0,0 +1,226 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include 
+#include 
+#include 
+#include 
+
+#include 
+
+#include "Framework/ControlService.h"
+#include "Framework/DataRefUtils.h"
+#include "Framework/CCDBParamSpec.h"
+#include "CommonConstants/Triggers.h"
+#include "DataFormatsEMCAL/Cell.h"
+#include "DataFormatsEMCAL/TriggerRecord.h"
+#include "EMCALWorkflow/OfflineCalibSpec.h"
+
+using namespace o2::emcal;
+
+void OfflineCalibSpec::init(o2::framework::InitContext& ctx)
+{
+  // energy vs. cell ID
+  mCellAmplitude = std::unique_ptr(new TH2F("mCellAmplitude", "Cell amplitude", 800, 0., 40., 17664, -0.5, 17663.5));
+  // time vs. cell ID
+  mCellTime = std::unique_ptr(new TH2F("mCellTime", "Cell time", 800, -200, 600, 17664, -0.5, 17663.5));
+  // time vs. cell ID
+  mCellTimeLG = std::unique_ptr(new TH2F("mCellTimeLG", "Cell time (low gain)", 800, -200, 600, 17664, -0.5, 17663.5));
+  // time vs. cell ID
+  mCellTimeHG = std::unique_ptr(new TH2F("mCellTimeHG", "Cell time (high gain)", 800, -200, 600, 17664, -0.5, 17663.5));
+  // number of events
+  mNevents = std::unique_ptr(new TH1F("mNevents", "Number of events", 1, 0.5, 1.5));
+  if (mMakeCellIDTimeEnergy) {
+    // cell time, cell energy, cell ID
+    std::array arrNBins = {17664, 100, 100}; // NCells, time, energy
+    std::array arrMin = {-0.5, -50, 0};
+    std::array arrMax = {17663.5, 50., 50};
+    mCellTimeEnergy = std::unique_ptr(new THnSparseF("CellIDvsTimevsEnergy", "CellIDvsTimevsEnergy", arrNBins.size(), arrNBins.data(), arrMin.data(), arrMax.data()));
+  }
+}
+
+void OfflineCalibSpec::finaliseCCDB(o2::framework::ConcreteDataMatcher& matcher, void* obj)
+{
+  LOG(info) << "Handling new Calibration objects";
+  mCalibrationHandler->finalizeCCDB(matcher, obj);
+
+  if (matcher == o2::framework::ConcreteDataMatcher("EMC", "EMCALCALIBPARAM", 0)) {
+    LOG(info) << "EMCal CalibParams updated";
+    EMCALCalibParams::Instance().printKeyValues(true, true);
+  }
+
+  if (mRejectL0Triggers && matcher == o2::framework::ConcreteDataMatcher("CTP", "CTPCONFIG", 0)) {
+    // clear current class mask and prepare to fill in the updated values
+    // The trigger names are seperated by a ":" in one string in the calib params
+    mSelectedClassMasks.clear();
+    std::string strSelClassMasks = EMCALCalibParams::Instance().selectedClassMasks;
+    std::string delimiter = ":";
+    size_t pos = 0;
+    std::vector vSelMasks;
+    while ((pos = strSelClassMasks.find(delimiter)) != std::string::npos) {
+      vSelMasks.push_back(strSelClassMasks.substr(0, pos));
+      strSelClassMasks.erase(0, pos + delimiter.length());
+    }
+    vSelMasks.push_back(strSelClassMasks);
+
+    auto ctpconf = reinterpret_cast(obj);
+
+    for (auto& cls : ctpconf->getCTPClasses()) {
+      LOG(debug) << "CTP class: " << cls.name << "\t " << cls.classMask;
+
+      if (std::find(vSelMasks.begin(), vSelMasks.end(), cls.name) != vSelMasks.end()) {
+        mSelectedClassMasks.push_back(cls.classMask);
+        LOG(info) << "Setting selected class mask " << cls.name << " to bit " << cls.classMask;
+      }
+    }
+  }
+}
+
+void OfflineCalibSpec::updateCalibObjects()
+{
+  if (mCalibrationHandler->hasUpdateGainCalib()) {
+    LOG(info) << "updateCalibObjects: Gain calib params changed";
+    mGainCalibFactors = mCalibrationHandler->getGainCalibration();
+  }
+}
+
+void OfflineCalibSpec::run(framework::ProcessingContext& pc)
+{
+  auto cells = pc.inputs().get>("cells");
+  auto triggerrecords = pc.inputs().get>("triggerrecord");
+
+  // prepare CTPConfiguration such that it can be loaded in finalise ccdb
+  if (mRejectL0Triggers) {
+    pc.inputs().get(getCTPConfigBinding());
+  }
+
+  using ctpDigitsType = std::decay_t>(getCTPDigitsBinding()))>;
+  std::optional ctpDigits;
+  if (mRejectL0Triggers) {
+    ctpDigits = pc.inputs().get>(getCTPDigitsBinding());
+  }
+
+  mCalibrationHandler->checkUpdates(pc);
+  updateCalibObjects();
+
+  LOG(debug) << "[EMCALOfflineCalib - run] received " << cells.size() << " cells from " << triggerrecords.size() << " triggers ...";
+  if (triggerrecords.size()) {
+    for (const auto& trg : triggerrecords) {
+      if (!trg.getNumberOfObjects()) {
+        LOG(debug) << "[EMCALOfflineCalib - run] Trigger does not contain cells, skipping ...";
+        continue;
+      }
+
+      // reject calibration triggers (EMCAL LED events etc.)
+      if (mRejectCalibTriggers) {
+        LOG(debug) << "Trigger: " << trg.getTriggerBits() << "   o2::trigger::Cal " << o2::trigger::Cal;
+        if (trg.getTriggerBits() & o2::trigger::Cal) {
+          LOG(debug) << "skipping triggered events due to wrong trigger (no Physics trigger)";
+          continue;
+        }
+      }
+
+      // reject all triggers that are not included in the classMask (typically only EMC min. bias should be accepted)
+      uint64_t classMaskCTP = 0;
+      if (mRejectL0Triggers) {
+        bool acceptEvent = false;
+        // Match the EMCal bc to the CTP bc
+        int64_t bcEMC = trg.getBCData().toLong();
+        for (auto& ctpDigit : *ctpDigits) {
+          int64_t bcCTP = ctpDigit.intRecord.toLong();
+          LOG(debug) << "bcEMC " << bcEMC << "   bcCTP " << bcCTP;
+          if (bcCTP == bcEMC) {
+            // obtain trigger mask that belongs to the selected bc
+            classMaskCTP = ctpDigit.CTPClassMask.to_ulong();
+            // now check if min bias trigger is not in mask
+            for (const uint64_t& selectedClassMask : mSelectedClassMasks) {
+              if ((classMaskCTP & selectedClassMask) != 0) {
+                LOG(debug) << "trigger " << selectedClassMask << " found! accepting event";
+                acceptEvent = true;
+                break;
+              }
+            }
+            break; // break as bc was matched
+          }
+        }
+        // if current event is not accepted (selected triggers not present), move on to next event
+        if (!acceptEvent) {
+          continue;
+        }
+      }
+
+      LOG(debug) << "[EMCALOfflineCalib - run] Trigger has " << trg.getNumberOfObjects() << " cells  ..." << std::endl;
+      gsl::span objectsTrigger(cells.data() + trg.getFirstEntry(), trg.getNumberOfObjects());
+      for (const auto& c : objectsTrigger) {
+        LOG(debug) << "[EMCALOfflineSpec - run] Channel: " << c.getTower();
+        LOG(debug) << "[EMCALOfflineSpec - run] Energy: " << c.getEnergy();
+        LOG(debug) << "[EMCALOfflineSpec - run] Time: " << c.getTimeStamp();
+        LOG(debug) << "[EMCALOfflineSpec - run] IsLowGain: " << c.getLowGain();
+        float cellE = c.getEnergy();
+        if (mGainCalibFactors && mCalibrationHandler->hasGainCalib()) {
+          LOG(debug) << "gain calib factor " << mGainCalibFactors->getGainCalibFactors(c.getTower());
+          cellE *= mGainCalibFactors->getGainCalibFactors(c.getTower());
+          LOG(debug) << "[EMCALOfflineSpec - run] corrected Energy: " << cellE;
+        }
+        mCellAmplitude->Fill(cellE, c.getTower());
+        if (cellE > 0.5) {
+          mCellTime->Fill(c.getTimeStamp(), c.getTower());
+          if (c.getLowGain()) {
+            mCellTimeLG->Fill(c.getTimeStamp(), c.getTower());
+          } else { // high gain cells
+            mCellTimeHG->Fill(c.getTimeStamp(), c.getTower());
+          }
+          if (mMakeCellIDTimeEnergy) {
+            mCellTimeEnergy->Fill(c.getTower(), c.getTimeStamp(), cellE);
+          }
+        }
+      }
+      mNevents->Fill(1);
+    }
+  }
+}
+
+void OfflineCalibSpec::endOfStream(o2::framework::EndOfStreamContext& ec)
+{
+  // write histograms to root file here
+  std::unique_ptr outputFile(new TFile("emcal-offline-calib.root", "RECREATE"));
+  outputFile->cd();
+  mCellAmplitude->Write();
+  mCellTime->Write();
+  mCellTimeLG->Write();
+  mCellTimeHG->Write();
+  mNevents->Write();
+  if (mMakeCellIDTimeEnergy) {
+    mCellTimeEnergy->Write();
+  }
+  outputFile->Close();
+}
+
+o2::framework::DataProcessorSpec o2::emcal::getEmcalOfflineCalibSpec(bool makeCellIDTimeEnergy, bool rejectCalibTriggers, bool rejectL0Trigger, uint32_t inputsubspec, bool enableGainCalib, bool ctpcfgperrun)
+{
+
+  std::vector
+    inputs = {{"cells", o2::header::gDataOriginEMC, "CELLS", inputsubspec, o2::framework::Lifetime::Timeframe},
+              {"triggerrecord", o2::header::gDataOriginEMC, "CELLSTRGR", inputsubspec, o2::framework::Lifetime::Timeframe}};
+
+  if (rejectL0Trigger) {
+    inputs.emplace_back(OfflineCalibSpec::getCTPConfigBinding(), "CTP", "CTPCONFIG", 0, o2::framework::Lifetime::Condition, o2::framework::ccdbParamSpec("CTP/Config/Config", ctpcfgperrun));
+    inputs.emplace_back(OfflineCalibSpec::getCTPDigitsBinding(), "CTP", "DIGITS", 0, o2::framework::Lifetime::Timeframe);
+  }
+  auto calibhandler = std::make_shared();
+  calibhandler->enableGainCalib(enableGainCalib);
+  calibhandler->defineInputSpecs(inputs);
+
+  return o2::framework::DataProcessorSpec{"EMCALOfflineCalib",
+                                          inputs,
+                                          {},
+                                          o2::framework::adaptFromTask(makeCellIDTimeEnergy, rejectCalibTriggers, rejectL0Trigger, calibhandler)};
+}
diff --git a/Detectors/EMCAL/workflow/src/PublisherSpec.cxx b/Detectors/EMCAL/workflow/src/PublisherSpec.cxx
index 3f4232cde1f40..1d49ea0f6ce4a 100644
--- a/Detectors/EMCAL/workflow/src/PublisherSpec.cxx
+++ b/Detectors/EMCAL/workflow/src/PublisherSpec.cxx
@@ -23,7 +23,7 @@ namespace o2
 namespace emcal
 {
 
-o2::framework::DataProcessorSpec createPublisherSpec(PublisherConf const& config, bool propagateMC, workflow_reader::Creator creator)
+o2::framework::DataProcessorSpec createPublisherSpec(PublisherConf const& config, uint32_t subspec, bool propagateMC, workflow_reader::Creator creator)
 {
   struct ProcessAttributes {
     std::shared_ptr reader;
@@ -32,7 +32,7 @@ o2::framework::DataProcessorSpec createPublisherSpec(PublisherConf const& config
     bool finished;
   };
 
-  auto initFunction = [config, propagateMC, creator](o2::framework::InitContext& ic) {
+  auto initFunction = [config, subspec, propagateMC, creator](o2::framework::InitContext& ic) {
     // get the option from the init context
     auto filename = o2::utils::Str::concat_string(o2::utils::Str::rectifyDirectory(ic.options().get("input-dir")),
                                                   ic.options().get("infile"));
@@ -84,15 +84,15 @@ o2::framework::DataProcessorSpec createPublisherSpec(PublisherConf const& config
     return processFunction;
   };
 
-  auto createOutputSpecs = [&config, propagateMC]() {
+  auto createOutputSpecs = [&config, subspec, propagateMC]() {
     std::vector outputSpecs;
     auto dto = o2::framework::DataSpecUtils::asConcreteDataTypeMatcher(config.dataoutput);
     auto tro = o2::framework::DataSpecUtils::asConcreteDataTypeMatcher(config.triggerrecordoutput);
     auto mco = o2::framework::DataSpecUtils::asConcreteDataTypeMatcher(config.mcoutput);
-    outputSpecs.emplace_back(o2::framework::OutputSpec{{"output"}, dto.origin, dto.description, 0, o2::framework::Lifetime::Timeframe});
-    outputSpecs.emplace_back(o2::framework::OutputSpec{{"outputTRG"}, tro.origin, tro.description, 0, o2::framework::Lifetime::Timeframe});
+    outputSpecs.emplace_back(o2::framework::OutputSpec{{"output"}, dto.origin, dto.description, subspec, o2::framework::Lifetime::Timeframe});
+    outputSpecs.emplace_back(o2::framework::OutputSpec{{"outputTRG"}, tro.origin, tro.description, subspec, o2::framework::Lifetime::Timeframe});
     if (propagateMC) {
-      outputSpecs.emplace_back(o2::framework::OutputSpec{{"outputMC"}, mco.origin, mco.description, 0, o2::framework::Lifetime::Timeframe});
+      outputSpecs.emplace_back(o2::framework::OutputSpec{{"outputMC"}, mco.origin, mco.description, subspec, o2::framework::Lifetime::Timeframe});
     }
     return std::move(outputSpecs);
   };
diff --git a/Detectors/EMCAL/workflow/src/RawToCellConverterSpec.cxx b/Detectors/EMCAL/workflow/src/RawToCellConverterSpec.cxx
index b80ef1f603f24..f2acc370a1bdc 100644
--- a/Detectors/EMCAL/workflow/src/RawToCellConverterSpec.cxx
+++ b/Detectors/EMCAL/workflow/src/RawToCellConverterSpec.cxx
@@ -12,11 +12,13 @@
 #include 
 #include 
 #include 
+#include 
 
 #include 
 
 #include "CommonConstants/Triggers.h"
 #include "CommonDataFormat/InteractionRecord.h"
+#include "Framework/CCDBParamSpec.h"
 #include "Framework/ConfigParamRegistry.h"
 #include "Framework/ControlService.h"
 #include "Framework/InputRecordWalker.h"
@@ -30,6 +32,8 @@
 #include "DetectorsRaw/RDHUtils.h"
 #include "EMCALBase/Geometry.h"
 #include "EMCALBase/Mapper.h"
+#include "EMCALBase/TriggerMappingErrors.h"
+#include "EMCALCalib/FeeDCS.h"
 #include "EMCALReconstruction/CaloFitResults.h"
 #include "EMCALReconstruction/Bunch.h"
 #include "EMCALReconstruction/CaloRawFitterStandard.h"
@@ -53,8 +57,10 @@ RawToCellConverterSpec::~RawToCellConverterSpec()
 
 void RawToCellConverterSpec::init(framework::InitContext& ctx)
 {
-  auto& ilctx = ctx.services().get();
-  ilctx.setField(AliceO2::InfoLogger::InfoLoggerContext::FieldName::Detector, "EMC");
+  if (ctx.services().active()) {
+    auto& ilctx = ctx.services().get();
+    ilctx.setField(AliceO2::InfoLogger::InfoLoggerContext::FieldName::Detector, "EMC");
+  }
 
   LOG(debug) << "[EMCALRawToCellConverter - init] Initialize converter ";
   if (!mGeometry) {
@@ -71,6 +77,10 @@ void RawToCellConverterSpec::init(framework::InitContext& ctx)
     LOG(error) << "Failed to initialize mapper";
   }
 
+  if (!mTriggerMapping) {
+    mTriggerMapping = std::make_unique(mGeometry);
+  }
+
   auto fitmethod = ctx.options().get("fitmethod");
   if (fitmethod == "standard") {
     LOG(info) << "Using standard raw fitter";
@@ -90,10 +100,11 @@ void RawToCellConverterSpec::init(framework::InitContext& ctx)
 
   mMergeLGHG = !ctx.options().get("no-mergeHGLG");
   mDisablePedestalEvaluation = ctx.options().get("no-evalpedestal");
+  mActiveLinkCheck = !ctx.options().get("no-checkactivelinks");
 
   LOG(info) << "Running gain merging mode: " << (mMergeLGHG ? "yes" : "no");
-  LOG(info) << "Using time shift: " << RecoParam::Instance().getCellTimeShiftNanoSec() << " ns";
-  LOG(info) << "Using BCshfit phase:" << RecoParam::Instance().getPhaseBCmod4() << " BCs";
+  LOG(info) << "Checking for active links: " << (mActiveLinkCheck ? "yes" : "no");
+  LOG(info) << "Calculate pedestals:       " << (mDisablePedestalEvaluation ? "no" : "yes");
   LOG(info) << "Using L0LM delay: " << o2::ctp::TriggerOffsetsParam::Instance().LM_L0 << " BCs";
 
   mRawFitter->setAmpCut(mNoiseThreshold);
@@ -103,10 +114,16 @@ void RawToCellConverterSpec::init(framework::InitContext& ctx)
 void RawToCellConverterSpec::run(framework::ProcessingContext& ctx)
 {
   LOG(debug) << "[EMCALRawToCellConverter - run] called";
-  double timeshift = RecoParam::Instance().getCellTimeShiftNanoSec(); // subtract offset in ns in order to center the time peak around the nominal delay
+  mCalibHandler->checkUpdates(ctx);
+  updateCalibrationObjects();
+
+  // container with BCid and feeID
+  std::unordered_map> bcFreq;
+
+  double timeshift = RecoParam::Instance().getCellTimeShiftNanoSec();       // subtract offset in ns in order to center the time peak around the nominal delay
+  auto maxBunchLengthRP = RecoParam::Instance().getMaxAllowedBunchLength(); // exclude bunches where either the start time or the bunch length is above the expected maximum
   constexpr auto originEMC = o2::header::gDataOriginEMC;
   constexpr auto descRaw = o2::header::gDataDescriptionRawData;
-  double noiseThresholLGnoHG = RecoParam::Instance().getNoiseThresholdLGnoHG();
 
   // reset message counter after 10 minutes
   auto currenttime = std::chrono::system_clock::now();
@@ -120,25 +137,28 @@ void RawToCellConverterSpec::run(framework::ProcessingContext& ctx)
   mOutputCells.clear();
   mOutputTriggerRecords.clear();
   mOutputDecoderErrors.clear();
+  mOutputTRUs.clear();
+  mOutputTRUTriggerRecords.clear();
+  mOutputPatches.clear();
+  mOutputPatchTriggerRecords.clear();
+  mOutputTimesums.clear();
+  mOutputTimesumTriggerRecords.clear();
 
   if (isLostTimeframe(ctx)) {
-    sendData(ctx, mOutputCells, mOutputTriggerRecords, mOutputDecoderErrors);
+    sendData(ctx);
     return;
   }
 
+  mCellHandler.reset();
+
   // Get the first orbit of the timeframe later used to check whether the corrected
   // BC is within the timeframe
   const auto tfOrbitFirst = ctx.services().get().firstTForbit;
   auto lml0delay = o2::ctp::TriggerOffsetsParam::Instance().LM_L0;
 
-  // Cache cells from for bunch crossings as the component reads timeframes from many links consecutively
-  std::map>> cellBuffer; // Internal cell buffer
-  std::map triggerBuffer;
-
   std::vector filter{{"filter", framework::ConcreteDataTypeMatcher(originEMC, descRaw)}};
   int firstEntry = 0;
   for (const auto& rawData : framework::InputRecordWalker(ctx.inputs(), filter)) {
-
     // Skip SOX headers
     auto rdhblock = reinterpret_cast(rawData.payload);
     if (o2::raw::RDHUtils::getHeaderSize(rdhblock) == static_cast(o2::framework::DataRefUtils::getPayloadSize(rawData))) {
@@ -153,23 +173,21 @@ void RawToCellConverterSpec::run(framework::ProcessingContext& ctx)
       try {
         rawreader.next();
       } catch (RawDecodingError& e) {
-        if (mCreateRawDataErrors) {
-          mOutputDecoderErrors.emplace_back(e.getFECID(), ErrorTypeFEE::ErrorSource_t::PAGE_ERROR, RawDecodingError::ErrorTypeToInt(e.getErrorType()), -1, -1);
-        }
-        if (mNumErrorMessages < mMaxErrorMessages) {
-          LOG(warning) << " Page decoding: " << e.what() << " in FEE ID " << e.getFECID() << std::endl;
-          mNumErrorMessages++;
-          if (mNumErrorMessages == mMaxErrorMessages) {
-            LOG(warning) << "Max. amount of error messages (" << mMaxErrorMessages << " reached, further messages will be suppressed";
-          }
-        } else {
-          mErrorMessagesSuppressed++;
+        handlePageError(e);
+        if (e.getErrorType() == RawDecodingError::ErrorType_t::HEADER_DECODING || e.getErrorType() == RawDecodingError::ErrorType_t::HEADER_INVALID) {
+          // We must break in case of header decoding as the offset to the next payload is lost
+          // consequently the parser does not know where to continue leading to an infinity loop
+          break;
         }
         // We must skip the page as payload is not consistent
         // otherwise the next functions will rethrow the exceptions as
         // the page format does not follow the expected format
         continue;
       }
+      for (auto& e : rawreader.getMinorErrors()) {
+        handleMinorPageError(e);
+        // For minor errors we do not need to skip the page, just print and send the error to the QC
+      }
 
       auto& header = rawreader.getRawHeader();
       auto triggerBC = raw::RDHUtils::getTriggerBC(header);
@@ -191,6 +209,9 @@ void RawToCellConverterSpec::run(framework::ProcessingContext& ctx)
           continue;
         }
       }
+
+      bcFreq[currentIR.toLong()].set(feeID, true);
+
       // Correct the cell time for the bc mod 4 (LHC: 40 MHz clock - ALTRO: 10 MHz clock)
       // Convention: All times shifted with respect to BC % 4 = 0 for trigger BC
       // Attention: Correction only works for the permutation (0 1 2 3) of the BC % 4, if the permutation is
@@ -203,16 +224,11 @@ void RawToCellConverterSpec::run(framework::ProcessingContext& ctx)
       LOG(debug) << "Applying correction for LM delay: " << correctionShiftBCmod4;
       LOG(debug) << "BC mod original: " << triggerBC % 4 << ", corrected " << bcmod4;
       LOG(debug) << "Applying time correction: " << -1 * 25 * bcmod4;
-      std::shared_ptr> currentCellContainer;
-      auto found = cellBuffer.find(currentIR);
-      if (found == cellBuffer.end()) {
-        currentCellContainer = std::make_shared>();
-        cellBuffer[currentIR] = currentCellContainer;
-        // also add trigger bits
-        triggerBuffer[currentIR] = triggerbits;
-      } else {
-        currentCellContainer = found->second;
+      auto& currentEvent = mCellHandler.getEventContainer(currentIR);
+      if (!currentEvent.getTriggerBits()) {
+        currentEvent.setTriggerBits(triggerbits);
       }
+      CellTimeCorrection timeCorrector{timeshift, bcmod4};
 
       if (feeID >= 40) {
         continue; // skip STU ddl
@@ -222,79 +238,19 @@ void RawToCellConverterSpec::run(framework::ProcessingContext& ctx)
 
       // use the altro decoder to decode the raw data, and extract the RCU trailer
       AltroDecoder decoder(rawreader);
+      if (maxBunchLengthRP) {
+        // apply user-defined max. bunch length
+        decoder.setMaxBunchLength(maxBunchLengthRP);
+      }
       // check the words of the payload exception in altrodecoder
       try {
         decoder.decode();
       } catch (AltroDecoderError& e) {
-        if (mNumErrorMessages < mMaxErrorMessages) {
-          std::string errormessage;
-          using AltroErrType = AltroDecoderError::ErrorType_t;
-          switch (e.getErrorType()) {
-            case AltroErrType::RCU_TRAILER_ERROR:
-              errormessage = " RCU Trailer Error ";
-              break;
-            case AltroErrType::RCU_VERSION_ERROR:
-              errormessage = " RCU Version Error ";
-              break;
-            case AltroErrType::RCU_TRAILER_SIZE_ERROR:
-              errormessage = " RCU Trailer Size Error ";
-              break;
-            case AltroErrType::ALTRO_BUNCH_HEADER_ERROR:
-              errormessage = " ALTRO Bunch Header Error ";
-              break;
-            case AltroErrType::ALTRO_BUNCH_LENGTH_ERROR:
-              errormessage = " ALTRO Bunch Length Error ";
-              break;
-            case AltroErrType::ALTRO_PAYLOAD_ERROR:
-              errormessage = " ALTRO Payload Error ";
-              break;
-            case AltroErrType::ALTRO_MAPPING_ERROR:
-              errormessage = " ALTRO Mapping Error ";
-              break;
-            case AltroErrType::CHANNEL_ERROR:
-              errormessage = " Channel Error ";
-              break;
-            default:
-              break;
-          };
-          LOG(warning) << " EMCAL raw task: " << errormessage << " in DDL " << feeID;
-          mNumErrorMessages++;
-          if (mNumErrorMessages == mMaxErrorMessages) {
-            LOG(warning) << "Max. amount of error messages (" << mMaxErrorMessages << " reached, further messages will be suppressed";
-          }
-        } else {
-          mErrorMessagesSuppressed++;
-        }
-        if (mCreateRawDataErrors) {
-          // fill histograms  with error types
-          ErrorTypeFEE errornum(feeID, ErrorTypeFEE::ErrorSource_t::ALTRO_ERROR, AltroDecoderError::errorTypeToInt(e.getErrorType()), -1, -1);
-          mOutputDecoderErrors.push_back(errornum);
-        }
+        handleAltroError(e, feeID);
         continue;
       }
-      for (auto minorerror : decoder.getMinorDecodingErrors()) {
-        if (mNumErrorMessages < mMaxErrorMessages) {
-          LOG(warning) << " EMCAL raw task - Minor error in DDL " << feeID << ": " << minorerror.what();
-          mNumErrorMessages++;
-          if (mNumErrorMessages == mMaxErrorMessages) {
-            LOG(warning) << "Max. amount of error messages (" << mMaxErrorMessages << " reached, further messages will be suppressed";
-          }
-        } else {
-          mErrorMessagesSuppressed++;
-        }
-        if (mCreateRawDataErrors) {
-          int fecID = -1, hwaddress = -1;
-          try {
-            hwaddress = Channel::getHardwareAddressFromChannelHeader(minorerror.getChannelHeader());
-            fecID = mMapper->getFEEForChannelInDDL(feeID, Channel::getFecIndexFromHwAddress(hwaddress), Channel::getBranchIndexFromHwAddress(hwaddress));
-          } catch (Mapper::AddressNotFoundException& e) {
-            // Unfortunately corrupted FEC IDs will not have useful information, so we need to initalize with -1
-          } catch (MappingHandler::DDLInvalid& e) {
-            // Unfortunately corrupted FEC IDs will not have useful information, so we need to initalize with -1
-          }
-          ErrorTypeFEE errornum(feeID, ErrorTypeFEE::ErrorSource_t::MINOR_ALTRO_ERROR, MinorAltroDecodingError::errorTypeToInt(minorerror.getErrorType()), fecID, hwaddress);
-          mOutputDecoderErrors.push_back(errornum);
-        }
+      for (const auto& minorerror : decoder.getMinorDecodingErrors()) {
+        handleMinorAltroError(minorerror, feeID);
       }
 
       if (mPrintTrailer) {
@@ -325,270 +281,146 @@ void RawToCellConverterSpec::run(framework::ProcessingContext& ctx)
         // Loop over all the channels
         int nBunchesNotOK = 0;
         for (auto& chan : decoder.getChannels()) {
-          int iRow, iCol;
-          ChannelType_t chantype;
-          try {
-            iRow = map.getRow(chan.getHardwareAddress());
-            iCol = map.getColumn(chan.getHardwareAddress());
-            chantype = map.getChannelType(chan.getHardwareAddress());
-          } catch (Mapper::AddressNotFoundException& ex) {
-            if (mNumErrorMessages < mMaxErrorMessages) {
-              LOG(warning) << "Mapping error DDL " << feeID << ": " << ex.what();
-              mNumErrorMessages++;
-              if (mNumErrorMessages == mMaxErrorMessages) {
-                LOG(warning) << "Max. amount of error messages (" << mMaxErrorMessages << " reached, further messages will be suppressed";
-              }
-            } else {
-              mErrorMessagesSuppressed++;
-            }
-            continue;
-          }
-
-          if (!(chantype == o2::emcal::ChannelType_t::HIGH_GAIN || chantype == o2::emcal::ChannelType_t::LOW_GAIN)) {
-            continue;
-          }
-
-          auto [phishift, etashift] = mGeometry->ShiftOnlineToOfflineCellIndexes(iSM, iRow, iCol);
-          int CellID = mGeometry->GetAbsCellIdFromCellIndexes(iSM, phishift, etashift);
-          if (CellID > 17664) {
-            if (mNumErrorMessages < mMaxErrorMessages) {
-              std::string celltypename;
-              switch (chantype) {
-                case o2::emcal::ChannelType_t::HIGH_GAIN:
-                  celltypename = "high gain";
-                  break;
-                case o2::emcal::ChannelType_t::LOW_GAIN:
-                  celltypename = "low-gain";
-                  break;
-                case o2::emcal::ChannelType_t::TRU:
-                  celltypename = "TRU";
-                  break;
-                case o2::emcal::ChannelType_t::LEDMON:
-                  celltypename = "LEDMON";
-                  break;
-              };
-              LOG(warning) << "Sending invalid cell ID " << CellID << "(SM " << iSM << ", row " << iRow << " - shift " << phishift << ", col " << iCol << " - shift " << etashift << ") of type " << celltypename;
-              mNumErrorMessages++;
-              if (mNumErrorMessages == mMaxErrorMessages) {
-                LOG(warning) << "Max. amount of error messages (" << mMaxErrorMessages << " reached, further messages will be suppressed";
-              }
-            } else {
-              mErrorMessagesSuppressed++;
-            }
-            if (mCreateRawDataErrors) {
-              mOutputDecoderErrors.emplace_back(feeID, ErrorTypeFEE::ErrorSource_t::GEOMETRY_ERROR, 0, CellID, chan.getHardwareAddress()); // 0 -> Cell ID out of range
-            }
-            continue;
-          }
-          if (CellID < 0) {
-            if (mNumErrorMessages < mMaxErrorMessages) {
-              std::string celltypename;
-              switch (chantype) {
-                case o2::emcal::ChannelType_t::HIGH_GAIN:
-                  celltypename = "high gain";
-                  break;
-                case o2::emcal::ChannelType_t::LOW_GAIN:
-                  celltypename = "low-gain";
-                  break;
-                case o2::emcal::ChannelType_t::TRU:
-                  celltypename = "TRU";
-                  break;
-                case o2::emcal::ChannelType_t::LEDMON:
-                  celltypename = "LEDMON";
-                  break;
-              };
-              LOG(warning) << "Sending negative cell ID " << CellID << "(SM " << iSM << ", row " << iRow << " - shift " << phishift << ", col " << iCol << " - shift " << etashift << ") of type " << celltypename;
-              mNumErrorMessages++;
-              if (mNumErrorMessages == mMaxErrorMessages) {
-                LOG(warning) << "Max. amount of error messages (" << mMaxErrorMessages << " reached, further messages will be suppressed";
-              }
-            } else {
-              mErrorMessagesSuppressed++;
-            }
-            if (mCreateRawDataErrors) {
-              mOutputDecoderErrors.emplace_back(feeID, ErrorTypeFEE::ErrorSource_t::GEOMETRY_ERROR, 2, CellID, chan.getHardwareAddress()); // Geometry error codes will start from 100
-            }
-            continue;
-          }
-
-          // define the conatiner for the fit results, and perform the raw fitting using the stadnard raw fitter
-          CaloFitResults fitResults;
           try {
-            fitResults = mRawFitter->evaluate(chan.getBunches());
-            // Prevent negative entries - we should no longer get here as the raw fit usually will end in an error state
-            if (fitResults.getAmp() < 0) {
-              fitResults.setAmp(0.);
-            }
-            if (fitResults.getTime() < 0) {
-              fitResults.setTime(0.);
-            }
-            // apply correction for bc mod 4
-            double celltime = fitResults.getTime() - timeshift - 25 * bcmod4;
-            double amp = fitResults.getAmp() * o2::emcal::constants::EMCAL_ADCENERGY;
-            if (mMergeLGHG) {
-              // Handling of HG/LG for ceratin cells
-              // Keep the high gain if it is below the threshold, otherwise
-              // change to the low gain
-              auto res = std::find_if(currentCellContainer->begin(), currentCellContainer->end(), [CellID](const RecCellInfo& test) { return test.mCellData.getTower() == CellID; });
-              if (res != currentCellContainer->end()) {
-                // Cell already existing, store LG if HG is larger then the overflow cut
-                if (chantype == o2::emcal::ChannelType_t::LOW_GAIN) {
-                  res->mHWAddressLG = chan.getHardwareAddress();
-                  res->mHGOutOfRange = false; // LG is found so it can replace the HG if the HG is out of range
-                  if (res->mCellData.getHighGain()) {
-                    double ampOld = res->mCellData.getEnergy() / o2::emcal::constants::EMCAL_ADCENERGY; // cut applied on ADC and not on energy
-                    if (ampOld > o2::emcal::constants::OVERFLOWCUT) {
-                      // High gain digit has energy above overflow cut, use low gain instead
-                      res->mCellData.setEnergy(amp * o2::emcal::constants::EMCAL_HGLGFACTOR);
-                      res->mCellData.setTimeStamp(celltime);
-                      res->mCellData.setLowGain();
-                    }
-                    res->mIsLGnoHG = false;
-                  }
-                } else {
-                  // new channel would be HG use that if it is belpw ADC cut
-                  // as the channel existed before it must have been a LG channel,
-                  /// whixh would be used in case the HG is out-of-range
-                  res->mIsLGnoHG = false;
-                  res->mHGOutOfRange = false;
-                  res->mHWAddressHG = chan.getHardwareAddress();
-                  if (amp / o2::emcal::constants::EMCAL_ADCENERGY <= o2::emcal::constants::OVERFLOWCUT) {
-                    res->mCellData.setEnergy(amp);
-                    res->mCellData.setTimeStamp(celltime);
-                    res->mCellData.setHighGain();
-                  }
-                }
-              } else {
-                // New cell
-                bool lgNoHG = false;       // Flag for filter of cells which have only low gain but no high gain
-                bool hgOutOfRange = false; // Flag if only a HG is present which is out-of-range
-                int hwAddressLG = -1,      // Hardware address of the LG of the tower (for monitoring)
-                  hwAddressHG = -1;        // Hardware address of the HG of the tower (for monitoring)
-                if (chantype == o2::emcal::ChannelType_t::LOW_GAIN) {
-                  lgNoHG = true;
-                  amp *= o2::emcal::constants::EMCAL_HGLGFACTOR;
-                  hwAddressLG = chan.getHardwareAddress();
-                } else {
-                  // High gain cell: Flag as low gain if above threshold
-                  if (amp / o2::emcal::constants::EMCAL_ADCENERGY > o2::emcal::constants::OVERFLOWCUT) {
-                    hgOutOfRange = true;
-                  }
-                  hwAddressHG = chan.getHardwareAddress();
+            auto iRow = map.getRow(chan.getHardwareAddress());
+            auto iCol = map.getColumn(chan.getHardwareAddress());
+            auto chantype = map.getChannelType(chan.getHardwareAddress());
+            LocalPosition channelPosition{iSM, feeID, iCol, iRow};
+            switch (chantype) {
+              case o2::emcal::ChannelType_t::HIGH_GAIN:
+              case o2::emcal::ChannelType_t::LOW_GAIN:
+                addFEEChannelToEvent(currentEvent, chan, timeCorrector, channelPosition, chantype);
+                break;
+              case o2::emcal::ChannelType_t::LEDMON:
+                // Drop LEDMON reconstruction in case of physics triggers
+                if (triggerbits & o2::trigger::Cal) {
+                  addFEEChannelToEvent(currentEvent, chan, timeCorrector, channelPosition, chantype);
                 }
-                int fecID = mMapper->getFEEForChannelInDDL(feeID, chan.getFECIndex(), chan.getBranchIndex());
-                currentCellContainer->push_back({o2::emcal::Cell(CellID, amp, celltime, chantype),
-                                                 lgNoHG,
-                                                 hgOutOfRange,
-                                                 fecID, feeID, hwAddressLG, hwAddressHG});
-              }
-            } else {
-              // No merge of HG/LG cells (usually MC where either
-              // of the two is simulated)
-              int hwAddressLG = chantype == ChannelType_t::LOW_GAIN ? chan.getHardwareAddress() : -1,
-                  hwAddressHG = chantype == ChannelType_t::HIGH_GAIN ? chan.getHardwareAddress() : -1;
-              // New cell
-              if (chantype == o2::emcal::ChannelType_t::LOW_GAIN) {
-                amp *= o2::emcal::constants::EMCAL_HGLGFACTOR;
-              }
-              int fecID = mMapper->getFEEForChannelInDDL(feeID, chan.getFECIndex(), chan.getBranchIndex());
-              currentCellContainer->push_back({o2::emcal::Cell(CellID, amp, celltime, chantype),
-                                               false,
-                                               false,
-                                               fecID, feeID, hwAddressLG, hwAddressHG});
-            }
-          } catch (CaloRawFitter::RawFitterError_t& fiterror) {
-            if (fiterror != CaloRawFitter::RawFitterError_t::BUNCH_NOT_OK) {
-              // Display
-              if (mNumErrorMessages < mMaxErrorMessages) {
-                LOG(warning) << "Failure in raw fitting: " << CaloRawFitter::createErrorMessage(fiterror);
-                mNumErrorMessages++;
-                if (mNumErrorMessages == mMaxErrorMessages) {
-                  LOG(warning) << "Max. amount of error messages (" << mMaxErrorMessages << " reached, further messages will be suppressed";
-                }
-              } else {
-                mErrorMessagesSuppressed++;
-              }
-              // Exclude BUNCH_NOT_OK also from raw error objects
-              if (mCreateRawDataErrors) {
-                mOutputDecoderErrors.emplace_back(feeID, ErrorTypeFEE::ErrorSource_t::FIT_ERROR, CaloRawFitter::getErrorNumber(fiterror), CellID, chan.getHardwareAddress());
-              }
-            } else {
-              LOG(debug2) << "Failure in raw fitting: " << CaloRawFitter::createErrorMessage(fiterror);
-              nBunchesNotOK++;
+                break;
+              case o2::emcal::ChannelType_t::TRU:
+                addTRUChannelToEvent(currentEvent, chan, channelPosition);
+                break;
+              default:
+                LOG(error) << "Unknown channel type for HW address " << chan.getHardwareAddress();
+                break;
             }
+          } catch (Mapper::AddressNotFoundException& ex) {
+            handleAddressError(ex, feeID, chan.getHardwareAddress());
+            continue;
           }
         }
       } catch (o2::emcal::MappingHandler::DDLInvalid& ddlerror) {
         // Unable to catch mapping
-        if (mNumErrorMessages < mMaxErrorMessages) {
-          LOG(error) << "Failed obtaining mapping for DDL " << ddlerror.getDDDL();
-          mNumErrorMessages++;
-          if (mNumErrorMessages == mMaxErrorMessages) {
-            LOG(error) << "Max. amount of error messages (" << mMaxErrorMessages << " reached, further messages will be suppressed";
-          }
-        }
-        if (mCreateRawDataErrors) {
-          mOutputDecoderErrors.emplace_back(feeID, ErrorTypeFEE::ErrorSource_t::ALTRO_ERROR, AltroDecoderError::errorTypeToInt(AltroDecoderError::ErrorType_t::ALTRO_MAPPING_ERROR), -1, -1);
-        }
+        handleDDLError(ddlerror, feeID);
       }
     }
   }
 
+  std::bitset<46> bitSetActiveLinks;
+  if (mActiveLinkCheck) {
+    // build expected active mask from DCS
+    FeeDCS* feedcs = mCalibHandler->getFEEDCS();
+    auto list0 = feedcs->getDDLlist0();
+    auto list1 = feedcs->getDDLlist1();
+    //  links 21 and 39 do not exist, but they are set active in DCS
+    list0.set(21, false);
+    list1.set(7, false);
+    // must be 0x307FFFDFFFFF if all links are active
+    bitSetActiveLinks = std::bitset<46>((list1.to_ullong() << 32) + list0.to_ullong());
+  }
+
   // Loop over BCs, sort cells with increasing tower ID and write to output containers
-  for (const auto& [bc, cells] : cellBuffer) {
-    int ncellsEvent = 0;
+  RecoContainerReader eventIterator(mCellHandler);
+  while (eventIterator.hasNext()) {
+    int ncellsEvent = 0, nLEDMONsEvent = 0;
     int eventstart = mOutputCells.size();
-    if (cells->size()) {
-      LOG(debug) << "Event has " << cells->size() << " cells";
-      // Sort cells according to cell ID
-      std::sort(cells->begin(), cells->end(), [](const RecCellInfo& lhs, const RecCellInfo& rhs) { return lhs.mCellData.getTower() < rhs.mCellData.getTower(); });
-      for (const auto& cell : *cells) {
-        if (cell.mIsLGnoHG) {
-          // Treat error only in case the LG is above the noise threshold
-          // no HG cell found, we can assume the cell amplitude is the LG amplitude
-          int ampLG = cell.mCellData.getAmplitude() / (o2::emcal::constants::EMCAL_ADCENERGY * o2::emcal::constants::EMCAL_HGLGFACTOR);
-          // use cut at 3 sigma where sigma for the LG digitizer is 0.4 ADC counts (EMCAL-502)
-          if (ampLG > noiseThresholLGnoHG) {
-            if (mNumErrorMessages < mMaxErrorMessages) {
-              LOG(warning) << "FEC " << cell.mFecID << ": 0x" << std::hex << cell.mHWAddressLG << std::dec << " (DDL " << cell.mDDLID << ") has low gain but no high-gain";
-              mNumErrorMessages++;
-              if (mNumErrorMessages == mMaxErrorMessages) {
-                LOG(warning) << "Max. amount of error messages (" << mMaxErrorMessages << " reached, further messages will be suppressed";
-              }
-            } else {
-              mErrorMessagesSuppressed++;
-            }
-            if (mCreateRawDataErrors) {
-              mOutputDecoderErrors.emplace_back(cell.mDDLID, ErrorTypeFEE::GAIN_ERROR, 0, cell.mFecID, cell.mHWAddressLG);
-            }
-          }
-          continue;
-        }
-        if (cell.mHGOutOfRange) {
-          if (mNumErrorMessages < mMaxErrorMessages) {
-            LOG(warning) << "FEC " << cell.mFecID << ": 0x" << std::hex << cell.mHWAddressHG << std::dec << " (DDL " << cell.mDDLID << ") has only high-gain out-of-range";
-            mNumErrorMessages++;
-            if (mNumErrorMessages == mMaxErrorMessages) {
-              LOG(warning) << "Max. amount of error messages (" << mMaxErrorMessages << " reached, further messages will be suppressed";
-            }
-          } else {
-            mErrorMessagesSuppressed++;
+    auto& currentevent = eventIterator.nextEvent();
+    const auto interaction = currentevent.getInteractionRecord();
+    if (mActiveLinkCheck) {
+      // check for current event if all links are present
+      // discard event if not all links are present
+      auto bcfreqFound = bcFreq.find(interaction.toLong());
+      if (bcfreqFound != bcFreq.end()) {
+        const auto& activelinks = bcfreqFound->second;
+        if (activelinks != bitSetActiveLinks) {
+          static int nErrors = 0;
+          if (nErrors++ < 3) {
+            LOG(error) << "Not all EMC active links contributed in global BCid=" << interaction.toLong() << ": mask=" << (activelinks ^ bitSetActiveLinks) << (nErrors == 3 ? " (not reporting further errors to avoid spamming)" : "");
           }
           if (mCreateRawDataErrors) {
-            mOutputDecoderErrors.emplace_back(cell.mDDLID, ErrorTypeFEE::GAIN_ERROR, 1, cell.mFecID, cell.mHWAddressHG);
+            for (std::size_t ilink = 0; ilink < bitSetActiveLinks.size(); ilink++) {
+              if (!bitSetActiveLinks.test(ilink)) {
+                continue;
+              }
+              if (!activelinks.test(ilink)) {
+                mOutputDecoderErrors.emplace_back(ilink, ErrorTypeFEE::ErrorSource_t::LINK_ERROR, 0, -1, -1);
+              }
+            }
           }
+          // discard event
+          // create empty trigger record with dedicated trigger bit marking as rejected
+          mOutputTriggerRecords.emplace_back(interaction, currentevent.getTriggerBits() | o2::emcal::triggerbits::Inc, eventstart, 0);
           continue;
         }
-        ncellsEvent++;
-        mOutputCells.push_back(cell.mCellData);
       }
-      LOG(debug) << "Next event [Orbit " << bc.orbit << ", BC (" << bc.bc << "]: Accepted " << ncellsEvent << " cells";
     }
-    mOutputTriggerRecords.emplace_back(bc, triggerBuffer[bc], eventstart, ncellsEvent);
+    // Add cells
+    if (currentevent.getNumberOfCells()) {
+      LOG(debug) << "Event has " << currentevent.getNumberOfCells() << " cells";
+      currentevent.sortCells(false);
+      ncellsEvent = bookEventCells(currentevent.getCells(), false);
+    }
+    // Add LEDMONs (if present)
+    if (currentevent.getNumberOfLEDMONs()) {
+      LOG(debug) << "Event has " << currentevent.getNumberOfLEDMONs() << " LEDMONs";
+      currentevent.sortCells(true);
+      nLEDMONsEvent = bookEventCells(currentevent.getLEDMons(), true);
+    }
+    LOG(debug) << "Next event [Orbit " << interaction.orbit << ", BC (" << interaction.bc << "]: Accepted " << ncellsEvent << " cells and " << nLEDMONsEvent << " LEDMONS";
+    mOutputTriggerRecords.emplace_back(interaction, currentevent.getTriggerBits(), eventstart, ncellsEvent + nLEDMONsEvent);
+
+    // Add trigger data
+    if (mDoTriggerReconstruction) {
+      auto [trus, patches] = buildL0Patches(currentevent);
+      LOG(debug) << "Found " << patches.size() << " L0 patches from " << trus.size() << " TRUs";
+      auto trusstart = mOutputTRUs.size();
+      std::copy(trus.begin(), trus.end(), std::back_inserter(mOutputTRUs));
+      mOutputTRUTriggerRecords.emplace_back(interaction, currentevent.getTriggerBits(), trusstart, trus.size());
+      auto patchesstart = mOutputPatches.size();
+      std::copy(patches.begin(), patches.end(), std::back_inserter(mOutputPatches));
+      mOutputPatchTriggerRecords.emplace_back(interaction, currentevent.getTriggerBits(), patchesstart, patches.size());
+      // For L0 timesums use fixed time, across TRUs and triggers, determined from the patch time QC
+      // average found to be - will be made configurable
+      auto timesumsstart = mOutputTimesums.size();
+      auto timesums = buildL0Timesums(currentevent, 8);
+      std::copy(timesums.begin(), timesums.end(), std::back_inserter(mOutputTimesums));
+      mOutputTimesumTriggerRecords.emplace_back(interaction, currentevent.getTriggerBits(), timesumsstart, timesums.size());
+    }
   }
 
   LOG(info) << "[EMCALRawToCellConverter - run] Writing " << mOutputCells.size() << " cells from " << mOutputTriggerRecords.size() << " events ...";
-  sendData(ctx, mOutputCells, mOutputTriggerRecords, mOutputDecoderErrors);
+  if (mDoTriggerReconstruction) {
+    LOG(info) << "[EMCALRawToCellConverter - run] Writing " << mOutputTRUs.size() << " TRU infos and " << mOutputPatches.size() << " trigger patches and " << mOutputTimesums.size() << " timesums from " << mOutputTRUTriggerRecords.size() << " events ...";
+  }
+  sendData(ctx);
+}
+
+void RawToCellConverterSpec::finaliseCCDB(o2::framework::ConcreteDataMatcher& matcher, void* obj)
+{
+  if (mCalibHandler->finalizeCCDB(matcher, obj)) {
+    return;
+  }
+}
+
+void RawToCellConverterSpec::updateCalibrationObjects()
+{
+  if (mCalibHandler->hasUpdateRecoParam()) {
+    LOG(info) << "RecoParams updated";
+    o2::emcal::RecoParam::Instance().printKeyValues(true, true);
+  }
+  if (mCalibHandler->hasUpdateFEEDCS()) {
+    LOG(info) << "DCS params updated";
+  }
 }
 
 bool RawToCellConverterSpec::isLostTimeframe(framework::ProcessingContext& ctx) const
@@ -605,7 +437,7 @@ bool RawToCellConverterSpec::isLostTimeframe(framework::ProcessingContext& ctx)
     if (payloadSize == 0) {
       auto maxWarn = o2::conf::VerbosityConfig::Instance().maxWarnDeadBeef;
       if (++contDeadBeef <= maxWarn) {
-        LOGP(alarm, "Found input [{}/{}/{:#x}] TF#{} 1st_orbit:{} Payload {} : assuming no payload for all links in this TF{}",
+        LOGP(warning, "Found input [{}/{}/{:#x}] TF#{} 1st_orbit:{} Payload {} : assuming no payload for all links in this TF{}",
              dh->dataOrigin.str, dh->dataDescription.str, dh->subSpecification, dh->tfCounter, dh->firstTForbit, payloadSize,
              contDeadBeef == maxWarn ? fmt::format(". {} such inputs in row received, stopping reporting", contDeadBeef) : "");
       }
@@ -616,18 +448,582 @@ bool RawToCellConverterSpec::isLostTimeframe(framework::ProcessingContext& ctx)
   return false;
 }
 
-void RawToCellConverterSpec::sendData(framework::ProcessingContext& ctx, const std::vector& cells, const std::vector& triggers, const std::vector& decodingErrors) const
+void RawToCellConverterSpec::addFEEChannelToEvent(o2::emcal::EventContainer& currentEvent, const o2::emcal::Channel& currentchannel, const CellTimeCorrection& timeCorrector, const LocalPosition& position, ChannelType_t chantype)
+{
+  int CellID = -1;
+  bool isLowGain = false;
+  try {
+    if (chantype == o2::emcal::ChannelType_t::HIGH_GAIN || chantype == o2::emcal::ChannelType_t::LOW_GAIN) {
+      // high- / low-gain cell
+      CellID = getCellAbsID(position.mSupermoduleID, position.mColumn, position.mRow);
+
+      isLowGain = chantype == o2::emcal::ChannelType_t::LOW_GAIN;
+    } else {
+      CellID = geLEDMONAbsID(position.mSupermoduleID, position.mColumn); // Module index encoded in colum for LEDMONs
+      isLowGain = position.mRow == 0;                                    // For LEDMONs gain type is encoded in the row (0 - low gain, 1 - high gain)
+    }
+  } catch (ModuleIndexException& e) {
+    handleGeometryError(e, position.mSupermoduleID, CellID, currentchannel.getHardwareAddress(), chantype);
+    return;
+  }
+
+  // define the conatiner for the fit results, and perform the raw fitting using the stadnard raw fitter
+  CaloFitResults fitResults;
+  try {
+    fitResults = mRawFitter->evaluate(currentchannel.getBunches());
+    // Prevent negative entries - we should no longer get here as the raw fit usually will end in an error state
+    if (fitResults.getAmp() < 0) {
+      fitResults.setAmp(0.);
+    }
+    if (fitResults.getTime() < 0) {
+      fitResults.setTime(0.);
+    }
+    // apply correction for bc mod 4
+    double celltime = timeCorrector.getCorrectedTime(fitResults.getTime());
+    double amp = fitResults.getAmp() * o2::emcal::constants::EMCAL_ADCENERGY;
+    if (isLowGain) {
+      amp *= o2::emcal::constants::EMCAL_HGLGFACTOR;
+    }
+    if (chantype == o2::emcal::ChannelType_t::LEDMON) {
+      // Mark LEDMONs as HIGH_GAIN/LOW_GAIN for gain type merging - will be flagged as LEDMON later when pushing to the output container
+      currentEvent.setLEDMONCell(CellID, amp, celltime, isLowGain ? o2::emcal::ChannelType_t::LOW_GAIN : o2::emcal::ChannelType_t::HIGH_GAIN, currentchannel.getHardwareAddress(), position.mFeeID, mMergeLGHG);
+    } else {
+      currentEvent.setCell(CellID, amp, celltime, chantype, currentchannel.getHardwareAddress(), position.mFeeID, mMergeLGHG);
+    }
+  } catch (CaloRawFitter::RawFitterError_t& fiterror) {
+    handleFitError(fiterror, position.mFeeID, CellID, currentchannel.getHardwareAddress());
+  }
+}
+
+void RawToCellConverterSpec::addTRUChannelToEvent(o2::emcal::EventContainer& currentEvent, const o2::emcal::Channel& currentchannel, const LocalPosition& position)
+{
+  try {
+    auto tru = mTriggerMapping->getTRUIndexFromOnlineHardareAddree(currentchannel.getHardwareAddress(), position.mFeeID, position.mSupermoduleID);
+    if (position.mColumn >= 96 && position.mColumn <= 105) {
+      auto& trudata = currentEvent.getTRUData(tru);
+      // Trigger patch information encoded columns 95-105
+      for (auto& bunch : currentchannel.getBunches()) {
+        LOG(debug) << "Found bunch of length " << static_cast(bunch.getBunchLength()) << " with start time " << static_cast(bunch.getStartTime()) << " (column " << static_cast(position.mColumn) << ")";
+        auto l0time = bunch.getStartTime();
+        int isample = 0;
+        for (auto& adc : bunch.getADC()) {
+          // patch word might be in any of the samples, need to check all of them
+          // in case of colum 105 the first 6 bits are the patch word, the remaining 4 bits are the header word
+          if (adc == 0) {
+            isample++;
+            continue;
+          }
+          if (position.mColumn == 105) {
+            std::bitset<6> patchBits(adc & 0x3F);
+            std::bitset<4> headerbits((adc >> 6) & 0xF);
+            for (auto localindex = 0; localindex < patchBits.size(); localindex++) {
+              if (patchBits.test(localindex)) {
+                auto globalindex = (position.mColumn - 96) * 10 + localindex;
+                LOG(debug) << "Found patch with index " << globalindex << " in sample " << isample;
+                // std::cout << "Found patch with index " << globalindex << " in sample " << isample << " (" << (bunch.getStartTime() - isample) << ")" << std::endl;
+                try {
+                  trudata.setPatch(globalindex, bunch.getStartTime() - isample);
+                } catch (TRUDataHandler::PatchIndexException& e) {
+                  handlePatchError(e, position.mFeeID, tru);
+                }
+              }
+            }
+            if (headerbits.test(2)) {
+              LOG(debug) << "TRU " << tru << ": Found TRU fired (" << tru << ") in sample " << isample;
+              // std::cout << "TRU " << tru << ": Found TRU fired (" << tru << ") in sample " << isample << " (" << (bunch.getStartTime() - isample) << ")" << std::endl;
+              trudata.setFired(true);
+              trudata.setL0time(bunch.getStartTime() - isample);
+            }
+          } else {
+            std::bitset<10> patchBits(adc & 0x3FF);
+            for (auto localindex = 0; localindex < patchBits.size(); localindex++) {
+              if (patchBits.test(localindex)) {
+                auto globalindex = (position.mColumn - 96) * 10 + localindex;
+                LOG(debug) << "TRU " << tru << ": Found patch with index " << globalindex << " in sample " << isample;
+                // std::cout << "TRU " << tru << ": Found patch with index " << globalindex << " in sample " << isample << " (" << (bunch.getStartTime() - isample) << ")" << std::endl;
+                try {
+                  trudata.setPatch(globalindex, bunch.getStartTime() - isample);
+                } catch (TRUDataHandler::PatchIndexException& e) {
+                  handlePatchError(e, position.mFeeID, tru);
+                }
+              }
+            }
+          }
+          isample++;
+        }
+      }
+    } else {
+      try {
+        auto absFastOR = mTriggerMapping->getAbsFastORIndexFromIndexInTRU(tru, position.mColumn);
+        for (auto& bunch : currentchannel.getBunches()) {
+          // FastOR data reversed internally (positive in time direction)
+          // -> Start time marks the first timebin, consequently it must be also reversed.
+          // std::cout << "Adding non-reversed FastOR time series for FastOR " << absFastOR << " (TRU " << tru << ", index " << static_cast(position.mColumn) << ") with start time " << static_cast(bunch.getStartTime()) << " (reversed " << bunch.getStartTime() + 1 - bunch.getADC().size() << "): ";
+          // for (auto adc : bunch.getADC()) {
+          //  std::cout << adc << ", ";
+          //}
+          // std::cout << std::endl;
+
+          try {
+            currentEvent.setFastOR(absFastOR, bunch.getStartTime(), bunch.getADC());
+          } catch (FastOrStartTimeInvalidException& e) {
+            handleFastORStartTimeErrors(e, position.mFeeID, tru);
+          }
+        }
+      } catch (FastORIndexException& e) {
+        handleFastORErrors(e, position.mFeeID, tru);
+      }
+    }
+  } catch (TRUIndexException& e) {
+    handleTRUIndexError(e, position.mFeeID, currentchannel.getHardwareAddress());
+  }
+}
+
+std::tuple RawToCellConverterSpec::buildL0Patches(const EventContainer& currentevent) const
+{
+  LOG(debug) << "Reconstructing patches for Orbit " << currentevent.getInteractionRecord().orbit << ", BC " << currentevent.getInteractionRecord().bc;
+  TRUContainer eventTRUs;
+  PatchContainer eventPatches;
+  auto& fastOrs = currentevent.getTimeSeriesContainer();
+  std::set foundFastOrs;
+  for (auto fastor : fastOrs) {
+    foundFastOrs.insert(fastor.first);
+  }
+  for (std::size_t itru = 0; itru < TriggerMappingV2::ALLTRUS; itru++) {
+    auto& currenttru = currentevent.readTRUData(itru);
+    if (!currenttru.hasAnyPatch()) {
+      continue;
+    }
+    auto l0time = currenttru.getL0time();
+    LOG(debug) << "Found patches in TRU " << itru << ", fired:  " << (currenttru.isFired() ? "yes" : "no") << ", L0 time " << static_cast(l0time);
+    uint8_t npatches = 0;
+    for (auto ipatch = 0; ipatch < o2::emcal::TriggerMappingV2::PATCHESINTRU; ipatch++) {
+      if (currenttru.hasPatch(ipatch)) {
+        auto patchtime = currenttru.getPatchTime(ipatch);
+        LOG(debug) << "Found patch " << ipatch << " in TRU " << itru << " with time " << static_cast(patchtime);
+        auto fastorStart = mTriggerMapping->getAbsFastORIndexFromIndexInTRU(itru, ipatch);
+        auto fastORs = mTriggerMapping->getFastORIndexFromL0Index(itru, ipatch, 4);
+        std::array fastors;
+        std::fill(fastors.begin(), fastors.end(), nullptr);
+        int indexFastorInTRU = 0;
+        for (auto fastor : fastORs) {
+          auto [truID, fastorTRU] = mTriggerMapping->getTRUFromAbsFastORIndex(fastor);
+          // std::cout << "Patch has abs FastOR " << fastor << " -> " << fastorTRU << " (in TRU)" << std::endl;
+          auto timeseriesFound = fastOrs.find(fastor);
+          if (timeseriesFound != fastOrs.end()) {
+            LOG(debug) << "Adding FastOR (" << indexFastorInTRU << ") with index " << fastor << " to patch";
+            fastors[indexFastorInTRU] = &(timeseriesFound->second);
+            indexFastorInTRU++;
+          }
+        }
+        auto [patchADC, recpatchtime] = reconstructTriggerPatch(fastors);
+        // Correct for bit shift 12->10 bits due to ALTRO format
+        patchADC = patchADC << 2;
+        LOG(debug) << "Reconstructed patch at index " << ipatch << " with peak time " << static_cast(recpatchtime) << " (time sample " << static_cast(patchtime) << ") and energy " << patchADC;
+        eventPatches.push_back({static_cast(itru), static_cast(ipatch), patchtime, patchADC});
+      }
+    }
+    eventTRUs.push_back({static_cast(itru), static_cast(l0time), currenttru.isFired(), npatches});
+  }
+  return std::make_tuple(eventTRUs, eventPatches);
+}
+
+std::vector RawToCellConverterSpec::buildL0Timesums(const o2::emcal::EventContainer& currentevent, uint8_t l0time) const
+{
+  std::vector timesums;
+  for (const auto& [fastorID, timeseries] : currentevent.getTimeSeriesContainer()) {
+    timesums.push_back({fastorID, static_cast(timeseries.calculateL1TimeSum(l0time) << 2)});
+  }
+  return timesums;
+}
+
+std::tuple RawToCellConverterSpec::reconstructTriggerPatch(const gsl::span fastors) const
+{
+  constexpr size_t INTEGRATE_SAMPLES = 4,
+                   MAX_SAMPLES = 12;
+  double maxpatchenergy = 0;
+  uint8_t foundtime = 0;
+  for (size_t itime = 0; itime < MAX_SAMPLES - INTEGRATE_SAMPLES; ++itime) {
+    double currenttimesum = 0;
+    for (size_t isample = 0; isample < INTEGRATE_SAMPLES; ++isample) {
+      for (auto ifastor = 0; ifastor < fastors.size(); ++ifastor) {
+        if (fastors[ifastor]) {
+          currenttimesum += fastors[ifastor]->getADCs()[itime + isample];
+        }
+      }
+    }
+    if (currenttimesum > maxpatchenergy) {
+      maxpatchenergy = currenttimesum;
+      foundtime = itime;
+    }
+  }
+
+  return std::make_tuple(maxpatchenergy, foundtime);
+}
+
+int RawToCellConverterSpec::bookEventCells(const gsl::span& cells, bool isLELDMON)
+{
+  double noiseThresholLGnoHG = RecoParam::Instance().getNoiseThresholdLGnoHG();
+  int ncellsSelected = 0;
+  for (const auto& cell : cells) {
+    if (cell.mIsLGnoHG) {
+      // Treat error only in case the LG is above the noise threshold
+      // no HG cell found, we can assume the cell amplitude is the LG amplitude
+      int ampLG = cell.mCellData.getAmplitude() / (o2::emcal::constants::EMCAL_ADCENERGY * o2::emcal::constants::EMCAL_HGLGFACTOR);
+      // use cut at 3 sigma where sigma for the LG digitizer is 0.4 ADC counts (EMCAL-502)
+      if (ampLG > noiseThresholLGnoHG) {
+        handleGainError(reconstructionerrors::GainError_t::LGNOHG, cell.mDDLID, cell.mHWAddressLG);
+      }
+      continue;
+    }
+    if (cell.mHGOutOfRange) {
+      handleGainError(reconstructionerrors::GainError_t::HGNOLG, cell.mDDLID, cell.mHWAddressHG);
+      continue;
+    }
+    ncellsSelected++;
+    mOutputCells.push_back(cell.mCellData);
+    if (isLELDMON) {
+      // LEDMON was handled internally in the reco container as HG/LG cell for gain type merging - reflag as LEDMON
+      mOutputCells.back().setLEDMon();
+    }
+  }
+  return ncellsSelected;
+}
+
+int RawToCellConverterSpec::getCellAbsID(int supermoduleID, int column, int row)
+{
+  auto [phishift, etashift] = mGeometry->ShiftOnlineToOfflineCellIndexes(supermoduleID, row, column);
+  int cellID = mGeometry->GetAbsCellIdFromCellIndexes(supermoduleID, phishift, etashift);
+  if (cellID > 17664 || cellID < 0) {
+    throw ModuleIndexException(cellID, column, row, etashift, phishift);
+  }
+  return cellID;
+}
+
+int RawToCellConverterSpec::geLEDMONAbsID(int supermoduleID, int moduleID)
+{
+  if (moduleID >= o2::emcal::EMCAL_LEDREFS || moduleID < 0) {
+    throw ModuleIndexException(moduleID);
+  }
+  return supermoduleID * o2::emcal::EMCAL_LEDREFS + moduleID;
+}
+
+void RawToCellConverterSpec::handleAddressError(const Mapper::AddressNotFoundException& error, int feeID, int hwaddress)
+{
+  if (mNumErrorMessages < mMaxErrorMessages) {
+    LOG(warning) << "Mapping error DDL " << feeID << ": " << error.what();
+    mNumErrorMessages++;
+    if (mNumErrorMessages == mMaxErrorMessages) {
+      LOG(warning) << "Max. amount of error messages (" << mMaxErrorMessages << " reached, further messages will be suppressed";
+    }
+  } else {
+    mErrorMessagesSuppressed++;
+  }
+  if (mCreateRawDataErrors) {
+    ErrorTypeFEE mappingError{feeID, ErrorTypeFEE::ErrorSource_t::ALTRO_ERROR, AltroDecoderError::errorTypeToInt(AltroDecoderError::ErrorType_t::ALTRO_MAPPING_ERROR), -1, hwaddress};
+    mOutputDecoderErrors.push_back(mappingError);
+  }
+}
+
+void RawToCellConverterSpec::handleAltroError(const o2::emcal::AltroDecoderError& altroerror, int ddlID)
+{
+  if (mNumErrorMessages < mMaxErrorMessages) {
+    std::string errormessage;
+    using AltroErrType = AltroDecoderError::ErrorType_t;
+    switch (altroerror.getErrorType()) {
+      case AltroErrType::RCU_TRAILER_ERROR:
+        errormessage = " RCU Trailer Error ";
+        break;
+      case AltroErrType::RCU_VERSION_ERROR:
+        errormessage = " RCU Version Error ";
+        break;
+      case AltroErrType::RCU_TRAILER_SIZE_ERROR:
+        errormessage = " RCU Trailer Size Error ";
+        break;
+      case AltroErrType::ALTRO_BUNCH_HEADER_ERROR:
+        errormessage = " ALTRO Bunch Header Error ";
+        break;
+      case AltroErrType::ALTRO_BUNCH_LENGTH_ERROR:
+        errormessage = " ALTRO Bunch Length Error ";
+        break;
+      case AltroErrType::ALTRO_PAYLOAD_ERROR:
+        errormessage = " ALTRO Payload Error ";
+        break;
+      case AltroErrType::ALTRO_MAPPING_ERROR:
+        errormessage = " ALTRO Mapping Error ";
+        break;
+      case AltroErrType::CHANNEL_ERROR:
+        errormessage = " Channel Error ";
+        break;
+      default:
+        break;
+    };
+    LOG(warning) << " EMCAL raw task: " << errormessage << " in DDL " << ddlID;
+    mNumErrorMessages++;
+    if (mNumErrorMessages == mMaxErrorMessages) {
+      LOG(warning) << "Max. amount of error messages (" << mMaxErrorMessages << " reached, further messages will be suppressed";
+    }
+  } else {
+    mErrorMessagesSuppressed++;
+  }
+  if (mCreateRawDataErrors) {
+    // fill histograms  with error types
+    ErrorTypeFEE errornum(ddlID, ErrorTypeFEE::ErrorSource_t::ALTRO_ERROR, AltroDecoderError::errorTypeToInt(altroerror.getErrorType()), -1, -1);
+    mOutputDecoderErrors.push_back(errornum);
+  }
+}
+
+void RawToCellConverterSpec::handleMinorAltroError(const o2::emcal::MinorAltroDecodingError& minorerror, int ddlID)
+{
+  if (mNumErrorMessages < mMaxErrorMessages) {
+    LOG(warning) << " EMCAL raw task - Minor error in DDL " << ddlID << ": " << minorerror.what();
+    mNumErrorMessages++;
+    if (mNumErrorMessages == mMaxErrorMessages) {
+      LOG(warning) << "Max. amount of error messages (" << mMaxErrorMessages << " reached, further messages will be suppressed";
+    }
+  } else {
+    mErrorMessagesSuppressed++;
+  }
+  if (mCreateRawDataErrors) {
+    int fecID = -1, hwaddress = -1;
+    try {
+      hwaddress = Channel::getHardwareAddressFromChannelHeader(minorerror.getChannelHeader());
+      fecID = mMapper->getFEEForChannelInDDL(ddlID, Channel::getFecIndexFromHwAddress(hwaddress), Channel::getBranchIndexFromHwAddress(hwaddress));
+    } catch (Mapper::AddressNotFoundException& e) {
+      // Unfortunately corrupted FEC IDs will not have useful information, so we need to initalize with -1
+    } catch (MappingHandler::DDLInvalid& e) {
+      // Unfortunately corrupted FEC IDs will not have useful information, so we need to initalize with -1
+    }
+    ErrorTypeFEE errornum(ddlID, ErrorTypeFEE::ErrorSource_t::MINOR_ALTRO_ERROR, MinorAltroDecodingError::errorTypeToInt(minorerror.getErrorType()), fecID, hwaddress);
+    mOutputDecoderErrors.push_back(errornum);
+  }
+}
+
+void RawToCellConverterSpec::handleDDLError(const MappingHandler::DDLInvalid& error, int feeID)
+{
+  if (mNumErrorMessages < mMaxErrorMessages) {
+    LOG(error) << "Failed obtaining mapping for DDL " << error.getDDDL();
+    mNumErrorMessages++;
+    if (mNumErrorMessages == mMaxErrorMessages) {
+      LOG(error) << "Max. amount of error messages (" << mMaxErrorMessages << " reached, further messages will be suppressed";
+    }
+  }
+  if (mCreateRawDataErrors) {
+    mOutputDecoderErrors.emplace_back(feeID, ErrorTypeFEE::ErrorSource_t::ALTRO_ERROR, AltroDecoderError::errorTypeToInt(AltroDecoderError::ErrorType_t::ALTRO_MAPPING_ERROR), -1, -1);
+  }
+}
+
+void RawToCellConverterSpec::handleFitError(const o2::emcal::CaloRawFitter::RawFitterError_t& fiterror, int ddlID, int cellID, int hwaddress)
+{
+  if (fiterror != CaloRawFitter::RawFitterError_t::BUNCH_NOT_OK) {
+    // Display
+    if (mNumErrorMessages < mMaxErrorMessages) {
+      LOG(warning) << "Failure in raw fitting: " << CaloRawFitter::createErrorMessage(fiterror);
+      mNumErrorMessages++;
+      if (mNumErrorMessages == mMaxErrorMessages) {
+        LOG(warning) << "Max. amount of error messages (" << mMaxErrorMessages << " reached, further messages will be suppressed";
+      }
+    } else {
+      mErrorMessagesSuppressed++;
+    }
+    // Exclude BUNCH_NOT_OK also from raw error objects
+    if (mCreateRawDataErrors) {
+      mOutputDecoderErrors.emplace_back(ddlID, ErrorTypeFEE::ErrorSource_t::FIT_ERROR, CaloRawFitter::getErrorNumber(fiterror), cellID, hwaddress);
+    }
+  } else {
+    LOG(debug2) << "Failure in raw fitting: " << CaloRawFitter::createErrorMessage(fiterror);
+  }
+}
+
+void RawToCellConverterSpec::handleGainError(const o2::emcal::reconstructionerrors::GainError_t& errortype, int ddlID, int hwaddress)
+{
+  int fecID = mMapper->getFEEForChannelInDDL(ddlID, Channel::getFecIndexFromHwAddress(hwaddress), Channel::getBranchIndexFromHwAddress(hwaddress));
+  if (mNumErrorMessages < mMaxErrorMessages) {
+    switch (errortype) {
+      case reconstructionerrors::GainError_t::HGNOLG:
+        LOG(warning) << "FEC " << fecID << ": 0x" << std::hex << hwaddress << std::dec << " (DDL " << ddlID << ") has only high-gain out-of-range";
+        break;
+      case reconstructionerrors::GainError_t::LGNOHG:
+        LOG(warning) << "FEC " << fecID << ": 0x" << std::hex << hwaddress << std::dec << " (DDL " << ddlID << ") has low gain but no high-gain";
+        break;
+      default:
+        LOG(warning) << "FEC " << fecID << ": 0x" << std::hex << hwaddress << std::dec << " (DDL " << ddlID << ") falsely flagged as gain error";
+    };
+    if (mNumErrorMessages == mMaxErrorMessages) {
+      LOG(warning) << "Max. amount of error messages (" << mMaxErrorMessages << " reached, further messages will be suppressed";
+    }
+  } else {
+    mErrorMessagesSuppressed++;
+  }
+  if (mCreateRawDataErrors) {
+    mOutputDecoderErrors.emplace_back(ddlID, ErrorTypeFEE::GAIN_ERROR, reconstructionerrors::getErrorCodeFromGainError(errortype), fecID, hwaddress);
+  }
+}
+
+void RawToCellConverterSpec::handleGeometryError(const ModuleIndexException& error, int feeID, int cellID, int hwaddress, ChannelType_t chantype)
+{
+  if (mNumErrorMessages < mMaxErrorMessages) {
+    std::string celltypename;
+    switch (chantype) {
+      case o2::emcal::ChannelType_t::HIGH_GAIN:
+        celltypename = "high gain";
+        break;
+      case o2::emcal::ChannelType_t::LOW_GAIN:
+        celltypename = "low-gain";
+        break;
+      case o2::emcal::ChannelType_t::TRU:
+        celltypename = "TRU";
+        break;
+      case o2::emcal::ChannelType_t::LEDMON:
+        celltypename = "LEDMON";
+        break;
+    };
+    int supermoduleID = feeID / 2;
+    switch (error.getModuleType()) {
+      case ModuleIndexException::ModuleType_t::CELL_MODULE:
+        LOG(warning) << "Sending invalid or negative cell ID " << error.getIndex() << " (SM " << supermoduleID << ", row " << error.getRow() << " - shift " << error.getRowShifted() << ", col " << error.getColumn() << " - shift " << error.getColumnShifted() << ") of type " << celltypename;
+        break;
+      case ModuleIndexException::ModuleType_t::LEDMON_MODULE:
+        LOG(warning) << "Sending invalid or negative LEDMON module ID " << error.getIndex() << "( SM" << supermoduleID << ")";
+        break;
+    };
+    mNumErrorMessages++;
+    if (mNumErrorMessages == mMaxErrorMessages) {
+      LOG(warning) << "Max. amount of error messages (" << mMaxErrorMessages << " reached, further messages will be suppressed";
+    }
+  } else {
+    mErrorMessagesSuppressed++;
+  }
+  if (mCreateRawDataErrors) {
+    mOutputDecoderErrors.emplace_back(feeID, ErrorTypeFEE::ErrorSource_t::GEOMETRY_ERROR, reconstructionerrors::getErrorCodeFromGeometryError(cellID < 0 ? reconstructionerrors::GeometryError_t::CELL_INDEX_NEGATIVE : reconstructionerrors::GeometryError_t::CELL_RANGE_EXCEED), cellID, hwaddress); // 0 -> Cell ID out of range
+  }
+}
+
+void RawToCellConverterSpec::handlePageError(const RawDecodingError& e)
+{
+  if (mCreateRawDataErrors) {
+    mOutputDecoderErrors.emplace_back(e.getFECID(), ErrorTypeFEE::ErrorSource_t::PAGE_ERROR, RawDecodingError::ErrorTypeToInt(e.getErrorType()), -1, -1);
+  }
+  if (mNumErrorMessages < mMaxErrorMessages) {
+    LOG(warning) << " Page decoding: " << e.what() << " in FEE ID " << e.getFECID();
+    mNumErrorMessages++;
+    if (mNumErrorMessages == mMaxErrorMessages) {
+      LOG(warning) << "Max. amount of error messages (" << mMaxErrorMessages << " reached, further messages will be suppressed";
+    }
+  } else {
+    mErrorMessagesSuppressed++;
+  }
+}
+
+void RawToCellConverterSpec::handleMinorPageError(const RawReaderMemory::MinorError& e)
+{
+  if (mCreateRawDataErrors) {
+    mOutputDecoderErrors.emplace_back(e.getFEEID(), ErrorTypeFEE::ErrorSource_t::PAGE_ERROR, RawDecodingError::ErrorTypeToInt(e.getErrorType()), -1, -1);
+  }
+  if (mNumErrorMessages < mMaxErrorMessages) {
+    LOG(warning) << " Page decoding: " << RawDecodingError::getErrorCodeDescription(e.getErrorType()) << " in FEE ID " << e.getFEEID();
+    mNumErrorMessages++;
+    if (mNumErrorMessages == mMaxErrorMessages) {
+      LOG(warning) << "Max. amount of error messages (" << mMaxErrorMessages << " reached, further messages will be suppressed";
+    }
+  } else {
+    mErrorMessagesSuppressed++;
+  }
+}
+
+void RawToCellConverterSpec::handleFastORErrors(const FastORIndexException& e, unsigned int linkID, unsigned int indexTRU)
+{
+  if (mCreateRawDataErrors) {
+    mOutputDecoderErrors.emplace_back(linkID, ErrorTypeFEE::ErrorSource_t::TRU_ERROR, reconstructionerrors::getErrorCodeFromTRUDecodingError(reconstructionerrors::TRUDecodingError_t::TRU_INDEX_INVALID), indexTRU, -1);
+  }
+  if (mNumErrorMessages < mMaxErrorMessages) {
+    LOG(warning) << " TRU decoding: " << e.what() << " in FEE ID " << linkID << ", TRU " << indexTRU;
+    mNumErrorMessages++;
+    if (mNumErrorMessages == mMaxErrorMessages) {
+      LOG(warning) << "Max. amount of error messages (" << mMaxErrorMessages << " reached, further messages will be suppressed";
+    }
+  } else {
+    mErrorMessagesSuppressed++;
+  }
+}
+
+void RawToCellConverterSpec::handleFastORStartTimeErrors(const FastOrStartTimeInvalidException& e, unsigned int linkID, unsigned int indexTRU)
+{
+  if (mCreateRawDataErrors) {
+    mOutputDecoderErrors.emplace_back(linkID, ErrorTypeFEE::ErrorSource_t::TRU_ERROR, reconstructionerrors::getErrorCodeFromTRUDecodingError(reconstructionerrors::TRUDecodingError_t::FASTOR_STARTTIME_INVALID), indexTRU, -1);
+  }
+  if (mNumErrorMessages < mMaxErrorMessages) {
+    LOG(warning) << " TRU decoding: " << e.what() << " in FEE ID " << linkID << ", TRU " << indexTRU;
+    mNumErrorMessages++;
+    if (mNumErrorMessages == mMaxErrorMessages) {
+      LOG(warning) << "Max. amount of error messages (" << mMaxErrorMessages << " reached, further messages will be suppressed";
+    }
+  } else {
+    mErrorMessagesSuppressed++;
+  }
+}
+
+void RawToCellConverterSpec::handlePatchError(const TRUDataHandler::PatchIndexException& e, unsigned int linkID, unsigned int indexTRU)
+{
+  if (mCreateRawDataErrors) {
+    mOutputDecoderErrors.emplace_back(linkID, ErrorTypeFEE::ErrorSource_t::TRU_ERROR, reconstructionerrors::getErrorCodeFromTRUDecodingError(reconstructionerrors::TRUDecodingError_t::PATCH_INDEX_INVALID), indexTRU, -1);
+  }
+  if (mNumErrorMessages < mMaxErrorMessages) {
+    LOG(warning) << " TRU decoding: " << e.what() << " in FEE ID " << linkID << ", TRU " << indexTRU;
+    mNumErrorMessages++;
+    if (mNumErrorMessages == mMaxErrorMessages) {
+      LOG(warning) << "Max. amount of error messages (" << mMaxErrorMessages << " reached, further messages will be suppressed";
+    }
+  } else {
+    mErrorMessagesSuppressed++;
+  }
+}
+
+void RawToCellConverterSpec::handleTRUIndexError(const TRUIndexException& e, unsigned int linkID, unsigned int hwaddress)
+{
+  if (mCreateRawDataErrors) {
+    mOutputDecoderErrors.emplace_back(linkID, ErrorTypeFEE::ErrorSource_t::TRU_ERROR, reconstructionerrors::getErrorCodeFromTRUDecodingError(reconstructionerrors::TRUDecodingError_t::PATCH_INDEX_INVALID), e.getTRUIndex(), hwaddress);
+  }
+  if (mNumErrorMessages < mMaxErrorMessages) {
+    LOG(warning) << " TRU decoding: " << e.what() << " in FEE ID " << linkID << ", TRU " << e.getTRUIndex() << "(hardware address: " << hwaddress << ")";
+    mNumErrorMessages++;
+    if (mNumErrorMessages == mMaxErrorMessages) {
+      LOG(warning) << "Max. amount of error messages (" << mMaxErrorMessages << " reached, further messages will be suppressed";
+    }
+  } else {
+    mErrorMessagesSuppressed++;
+  }
+}
+
+void RawToCellConverterSpec::sendData(framework::ProcessingContext& ctx) const
 {
   constexpr auto originEMC = o2::header::gDataOriginEMC;
-  ctx.outputs().snapshot(framework::Output{originEMC, "CELLS", mSubspecification, framework::Lifetime::Timeframe}, cells);
-  ctx.outputs().snapshot(framework::Output{originEMC, "CELLSTRGR", mSubspecification, framework::Lifetime::Timeframe}, triggers);
+  ctx.outputs().snapshot(framework::Output{originEMC, "CELLS", mSubspecification}, mOutputCells);
+  ctx.outputs().snapshot(framework::Output{originEMC, "CELLSTRGR", mSubspecification}, mOutputTriggerRecords);
   if (mCreateRawDataErrors) {
-    LOG(debug) << "Sending " << decodingErrors.size() << " decoding errors";
-    ctx.outputs().snapshot(framework::Output{originEMC, "DECODERERR", mSubspecification, framework::Lifetime::Timeframe}, decodingErrors);
+    LOG(debug) << "Sending " << mOutputDecoderErrors.size() << " decoding errors";
+    ctx.outputs().snapshot(framework::Output{originEMC, "DECODERERR", mSubspecification}, mOutputDecoderErrors);
+  }
+  if (mDoTriggerReconstruction) {
+    ctx.outputs().snapshot(framework::Output{originEMC, "TRUS", mSubspecification}, mOutputTRUs);
+    ctx.outputs().snapshot(framework::Output{originEMC, "TRUSTRGR", mSubspecification}, mOutputTRUTriggerRecords);
+    ctx.outputs().snapshot(framework::Output{originEMC, "PATCHES", mSubspecification}, mOutputPatches);
+    ctx.outputs().snapshot(framework::Output{originEMC, "PATCHESTRGR", mSubspecification}, mOutputPatchTriggerRecords);
+    ctx.outputs().snapshot(framework::Output{originEMC, "FASTORS", mSubspecification}, mOutputTimesums);
+    ctx.outputs().snapshot(framework::Output{originEMC, "FASTORSTRGR", mSubspecification}, mOutputTimesumTriggerRecords);
   }
 }
 
-o2::framework::DataProcessorSpec o2::emcal::reco_workflow::getRawToCellConverterSpec(bool askDISTSTF, bool disableDecodingErrors, int subspecification)
+RawToCellConverterSpec::ModuleIndexException::ModuleIndexException(int moduleIndex, int column, int row, int shiftedColumn, int shiftedRow) : mModuleType(ModuleType_t::CELL_MODULE),
+                                                                                                                                              mIndex(moduleIndex),
+                                                                                                                                              mColumn(column),
+                                                                                                                                              mRow(row),
+                                                                                                                                              mColumnShifted(shiftedColumn),
+                                                                                                                                              mRowShifted(shiftedRow) {}
+
+RawToCellConverterSpec::ModuleIndexException::ModuleIndexException(int moduleIndex) : mModuleType(ModuleType_t::LEDMON_MODULE), mIndex(moduleIndex) {}
+
+o2::framework::DataProcessorSpec o2::emcal::reco_workflow::getRawToCellConverterSpec(bool askDISTSTF, bool disableDecodingErrors, bool disableTriggerReconstruction, int subspecification)
 {
   constexpr auto originEMC = o2::header::gDataOriginEMC;
   std::vector outputs;
@@ -637,20 +1033,35 @@ o2::framework::DataProcessorSpec o2::emcal::reco_workflow::getRawToCellConverter
   if (!disableDecodingErrors) {
     outputs.emplace_back(originEMC, "DECODERERR", subspecification, o2::framework::Lifetime::Timeframe);
   }
+  if (!disableTriggerReconstruction) {
+    outputs.emplace_back(originEMC, "TRUS", subspecification, o2::framework::Lifetime::Timeframe);
+    outputs.emplace_back(originEMC, "TRUSTRGR", subspecification, o2::framework::Lifetime::Timeframe);
+    outputs.emplace_back(originEMC, "PATCHES", subspecification, o2::framework::Lifetime::Timeframe);
+    outputs.emplace_back(originEMC, "PATCHESTRGR", subspecification, o2::framework::Lifetime::Timeframe);
+    outputs.emplace_back(originEMC, "FASTORS", subspecification, o2::framework::Lifetime::Timeframe);
+    outputs.emplace_back(originEMC, "FASTORSTRGR", subspecification, o2::framework::Lifetime::Timeframe);
+  }
 
-  std::vector inputs{{"stf", o2::framework::ConcreteDataTypeMatcher{originEMC, o2::header::gDataDescriptionRawData}, o2::framework::Lifetime::Optional}};
+  std::vector inputs{{"stf", o2::framework::ConcreteDataTypeMatcher{originEMC, o2::header::gDataDescriptionRawData}, o2::framework::Lifetime::Timeframe}};
   if (askDISTSTF) {
     inputs.emplace_back("stdDist", "FLP", "DISTSUBTIMEFRAME", 0, o2::framework::Lifetime::Timeframe);
   }
+  // CCDB objects
+  auto calibhandler = std::make_shared();
+  calibhandler->enableRecoParams(true);
+  calibhandler->enableFEEDCS(true);
+  calibhandler->defineInputSpecs(inputs);
 
-  return o2::framework::DataProcessorSpec{"EMCALRawToCellConverterSpec",
-                                          inputs,
-                                          outputs,
-                                          o2::framework::adaptFromTask(subspecification, !disableDecodingErrors),
-                                          o2::framework::Options{
-                                            {"fitmethod", o2::framework::VariantType::String, "gamma2", {"Fit method (standard or gamma2)"}},
-                                            {"maxmessage", o2::framework::VariantType::Int, 100, {"Max. amout of error messages to be displayed"}},
-                                            {"printtrailer", o2::framework::VariantType::Bool, false, {"Print RCU trailer (for debugging)"}},
-                                            {"no-mergeHGLG", o2::framework::VariantType::Bool, false, {"Do not merge HG and LG channels for same tower"}},
-                                            {"no-evalpedestal", o2::framework::VariantType::Bool, false, {"Disable pedestal evaluation"}}}};
+  return o2::framework::DataProcessorSpec{
+    "EMCALRawToCellConverterSpec",
+    inputs,
+    outputs,
+    o2::framework::adaptFromTask(subspecification, !disableDecodingErrors, !disableTriggerReconstruction, calibhandler),
+    o2::framework::Options{
+      {"fitmethod", o2::framework::VariantType::String, "gamma2", {"Fit method (standard or gamma2)"}},
+      {"maxmessage", o2::framework::VariantType::Int, 100, {"Max. amout of error messages to be displayed"}},
+      {"printtrailer", o2::framework::VariantType::Bool, false, {"Print RCU trailer (for debugging)"}},
+      {"no-mergeHGLG", o2::framework::VariantType::Bool, false, {"Do not merge HG and LG channels for same tower"}},
+      {"no-checkactivelinks", o2::framework::VariantType::Bool, false, {"Do not check for active links per BC"}},
+      {"no-evalpedestal", o2::framework::VariantType::Bool, false, {"Disable pedestal evaluation"}}}};
 }
diff --git a/Detectors/EMCAL/workflow/src/RecoWorkflow.cxx b/Detectors/EMCAL/workflow/src/RecoWorkflow.cxx
index 84684e63b8df6..28e0deb3ae0b3 100644
--- a/Detectors/EMCAL/workflow/src/RecoWorkflow.cxx
+++ b/Detectors/EMCAL/workflow/src/RecoWorkflow.cxx
@@ -13,7 +13,7 @@
 #include 
 #include 
 
-#include "FairLogger.h"
+#include 
 
 #include "Algorithm/RangeTokenizer.h"
 #include "DPLUtils/MakeRootTreeWriterSpec.h"
@@ -47,12 +47,14 @@ namespace reco_workflow
 o2::framework::WorkflowSpec getWorkflow(bool propagateMC,
                                         bool askDISTSTF,
                                         bool enableDigitsPrinter,
-                                        int subspecification,
+                                        int subspecificationIn,
+                                        int subspecificationOut,
                                         std::string const& cfgInput,
                                         std::string const& cfgOutput,
                                         bool disableRootInput,
                                         bool disableRootOutput,
-                                        bool disableDecodingErrors)
+                                        bool disableDecodingErrors,
+                                        bool disableTriggerReconstruction)
 {
 
   const std::unordered_map InputMap{
@@ -132,7 +134,7 @@ o2::framework::WorkflowSpec getWorkflow(bool propagateMC,
                                                                        o2::framework::OutputSpec{"EMC", "DIGITS"},
                                                                        o2::framework::OutputSpec{"EMC", "DIGITSTRGR"},
                                                                        o2::framework::OutputSpec{"EMC", "DIGITSMCTR"}},
-                                                                     propagateMC));
+                                                                     0, propagateMC));
     }
 
     if (enableDigitsPrinter) {
@@ -155,7 +157,7 @@ o2::framework::WorkflowSpec getWorkflow(bool propagateMC,
                                                                       o2::framework::OutputSpec{"EMC", "CELLS"},
                                                                       o2::framework::OutputSpec{"EMC", "CELLSTRGR"},
                                                                       o2::framework::OutputSpec{"EMC", "CELLSMCTR"}},
-                                                                    propagateMC));
+                                                                    0, propagateMC));
     }
     if (enableDigitsPrinter) {
       try {
@@ -169,10 +171,10 @@ o2::framework::WorkflowSpec getWorkflow(bool propagateMC,
   if (isEnabled(OutputType::Cells)) {
     // add converter for cells
     if (inputType == InputType::Digits) {
-      specs.emplace_back(o2::emcal::reco_workflow::getCellConverterSpec(propagateMC));
+      specs.emplace_back(o2::emcal::reco_workflow::getCellConverterSpec(propagateMC, subspecificationIn, subspecificationOut));
     } else if (inputType == InputType::Raw) {
       // raw data will come from upstream
-      specs.emplace_back(o2::emcal::reco_workflow::getRawToCellConverterSpec(askDISTSTF, disableDecodingErrors, subspecification));
+      specs.emplace_back(o2::emcal::reco_workflow::getRawToCellConverterSpec(askDISTSTF, disableDecodingErrors, disableTriggerReconstruction, subspecificationOut));
     }
   }
 
diff --git a/Detectors/EMCAL/workflow/src/StandaloneAODProducerSpec.cxx b/Detectors/EMCAL/workflow/src/StandaloneAODProducerSpec.cxx
index cd2b206257984..227fc373bf20c 100644
--- a/Detectors/EMCAL/workflow/src/StandaloneAODProducerSpec.cxx
+++ b/Detectors/EMCAL/workflow/src/StandaloneAODProducerSpec.cxx
@@ -17,7 +17,6 @@
 #include "Framework/InputRecordWalker.h"
 #include "Framework/Logger.h"
 #include "Framework/TableBuilder.h"
-#include "Framework/TableTreeHelpers.h"
 #include "MathUtils/Utils.h"
 
 using namespace o2::framework;
@@ -64,21 +63,21 @@ void StandaloneAODProducerSpec::run(ProcessingContext& pc)
   LOG(info) << "FOUND " << cellsIn.size() << " EMC cells in CTF";
   LOG(info) << "FOUND " << triggersIn.size() << " EMC tiggers in CTF";
 
-  auto& bcBuilder = pc.outputs().make(Output{"AOD", "BC"});
-  auto& collisionsBuilder = pc.outputs().make(Output{"AOD", "COLLISION"});
-  auto& caloCellsBuilder = pc.outputs().make(Output{"AOD", "CALO"});
-  auto& caloCellsTRGTableBuilder = pc.outputs().make(Output{"AOD", "CALOTRIGGER"});
+  auto bcBuilder = pc.outputs().make(Output{"AOD", "BC"});
+  auto collisionsBuilder = pc.outputs().make(Output{"AOD", "COLLISION"});
+  auto caloCellsBuilder = pc.outputs().make(Output{"AOD", "CALO"});
+  auto caloCellsTRGTableBuilder = pc.outputs().make(Output{"AOD", "CALOTRIGGER"});
 
-  auto bcCursor = bcBuilder.cursor();
-  auto collisionsCursor = collisionsBuilder.cursor();
-  auto caloCellsCursor = caloCellsBuilder.cursor();
-  auto caloCellsTRGTableCursor = caloCellsTRGTableBuilder.cursor();
+  auto bcCursor = bcBuilder->cursor();
+  auto collisionsCursor = collisionsBuilder->cursor();
+  auto caloCellsCursor = caloCellsBuilder->cursor();
+  auto caloCellsTRGTableCursor = caloCellsTRGTableBuilder->cursor();
 
   // build event for easier handling
   mCaloEventHandler->reset();
   mCaloEventHandler->setCellData(cellsIn, triggersIn);
 
-  uint64_t triggerMask = 1;
+  uint64_t triggerMask = 1, inputMask = 1;
   // loop over events
   for (int iev = 0; iev < mCaloEventHandler->getNumberOfEvents(); iev++) {
     o2::emcal::EventData inputEvent = mCaloEventHandler->buildEvent(iev);
@@ -91,7 +90,8 @@ void StandaloneAODProducerSpec::run(ProcessingContext& pc)
     bcCursor(0,
              runNumber,
              bcID,
-             triggerMask);
+             triggerMask,
+             inputMask);
     auto indexBC = iev;
 
     for (auto& cell : cellsInEvent) {
@@ -104,7 +104,7 @@ void StandaloneAODProducerSpec::run(ProcessingContext& pc)
                       o2::math_utils::detail::truncateFloatFraction(cell.getTimeStamp(), mCaloTime),
                       cell.getType(),
                       1); // hard coded for emcal (-1 would be undefined, 0 phos)
-    }                     // end of cell loop
+    } // end of cell loop
 
     // filled only with BCID, rest dummy for no2
     caloCellsTRGTableCursor(0,
@@ -135,7 +135,8 @@ void StandaloneAODProducerSpec::run(ProcessingContext& pc)
   } // end of event loop
   // std::cout << "Finished cell loop" << std::endl;
 
-  pc.outputs().snapshot(Output{"TFN", "TFNumber", 0, Lifetime::Timeframe}, tfNumber);
+  pc.outputs().snapshot(Output{"TFN", "TFNumber", 0}, tfNumber);
+  pc.outputs().snapshot(Output{"TFF", "TFFilename", 0}, "");
 
   mTimer.Stop();
 }
@@ -154,6 +155,7 @@ DataProcessorSpec getStandaloneAODProducerSpec()
   outputs.emplace_back(OutputLabel{"O2caloCell"}, "AOD", "CALO", 0, Lifetime::Timeframe);
   outputs.emplace_back(OutputLabel{"O2caloCellTRGR"}, "AOD", "CALOTRIGGER", 0, Lifetime::Timeframe);
   outputs.emplace_back(OutputSpec{"TFN", "TFNumber"});
+  outputs.emplace_back(OutputSpec{"TFF", "TFFilename"});
 
   return DataProcessorSpec{
     "standalone-aod-producer-workflow",
diff --git a/Detectors/EMCAL/workflow/src/cell-reader-workflow.cxx b/Detectors/EMCAL/workflow/src/cell-reader-workflow.cxx
index a290f76c2d873..52cae11a6bbad 100644
--- a/Detectors/EMCAL/workflow/src/cell-reader-workflow.cxx
+++ b/Detectors/EMCAL/workflow/src/cell-reader-workflow.cxx
@@ -31,6 +31,7 @@ void customize(std::vector& policies)
 void customize(std::vector& workflowOptions)
 {
   std::vector options{{"disable-mc", VariantType::Bool, false, {"Do not propagate MC labels"}},
+                                       {"subspec", VariantType::UInt32, 0U, {"Subspecification for cell output"}},
                                        {"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}}};
   o2::raw::HBFUtilsInitializer::addConfigOption(options);
   workflowOptions.insert(workflowOptions.end(), options.begin(), options.end());
@@ -41,7 +42,9 @@ void customize(std::vector& workflowOptions)
 WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
 {
   bool disableMC = cfgc.options().get("disable-mc");
+  auto subspec = cfgc.options().get("subspec");
   o2::conf::ConfigurableParam::updateFromString(cfgc.options().get("configKeyValues"));
+  LOG(info) << "Cell reader: publishing on subspec " << subspec << std::endl;
 
   WorkflowSpec specs;
   specs.emplace_back(o2::emcal::getPublisherSpec>(PublisherConf{
@@ -51,10 +54,10 @@ WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
                                                                                  {"cellbranch", "EMCALCell", "Cell branch"},
                                                                                  {"celltriggerbranch", "EMCALCellTRGR", "Trigger record branch"},
                                                                                  {"mcbranch", "EMCALCellMCTruth", "MC label branch"},
-                                                                                 o2::framework::OutputSpec{"EMC", "CELLS"},
-                                                                                 o2::framework::OutputSpec{"EMC", "CELLSTRGR"},
-                                                                                 o2::framework::OutputSpec{"EMC", "CELLSMCTR"}},
-                                                                               !disableMC));
+                                                                                 o2::framework::OutputSpec{{"cells"}, "EMC", "CELLS", subspec, o2::framework::Lifetime::Timeframe},
+                                                                                 o2::framework::OutputSpec{{"triggerrecords"}, "EMC", "CELLSTRGR", subspec, o2::framework::Lifetime::Timeframe},
+                                                                                 o2::framework::OutputSpec{{"mclabels"}, "EMC", "CELLSMCTR", subspec, o2::framework::Lifetime::Timeframe}},
+                                                                               subspec, !disableMC));
 
   // configure dpl timer to inject correct firstTForbit: start from the 1st orbit of TF containing 1st sampled orbit
   o2::raw::HBFUtilsInitializer hbfIni(cfgc, specs);
diff --git a/Detectors/EMCAL/workflow/src/cell-recalibrator-workflow.cxx b/Detectors/EMCAL/workflow/src/cell-recalibrator-workflow.cxx
new file mode 100644
index 0000000000000..bf4538cfc7e1d
--- /dev/null
+++ b/Detectors/EMCAL/workflow/src/cell-recalibrator-workflow.cxx
@@ -0,0 +1,81 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include 
+#include 
+#include "Framework/Variant.h"
+#include "Framework/ConfigParamSpec.h"
+#include "Framework/CallbacksPolicy.h"
+#include "DetectorsRaw/HBFUtilsInitializer.h"
+#include "EMCALWorkflow/CellRecalibratorSpec.h"
+#include "CommonUtils/ConfigurableParam.h"
+
+using namespace o2::framework;
+using namespace o2::emcal;
+
+void customize(std::vector& policies)
+{
+  o2::raw::HBFUtilsInitializer::addNewTimeSliceCallback(policies);
+}
+
+// we need to add workflow options before including Framework/runDataProcessing
+void customize(std::vector& workflowOptions)
+{
+  std::vector options{{"input-subspec", VariantType::UInt32, 1U, {"Subspecification for input objects"}},
+                                       {"output-subspec", VariantType::UInt32, 0U, {"Subspecification for output objects"}},
+                                       {"isMC", VariantType::Bool, false, {"Monte-Carlo mode"}},
+                                       {"no-badchannelcalib", VariantType::Bool, false, {"Disable bad channel calibration"}},
+                                       {"no-timecalib", VariantType::Bool, false, {"Disable time calibration"}},
+                                       {"no-gaincalib", VariantType::Bool, false, {"Disable gain calibration"}},
+                                       {"drop-led", VariantType::Bool, false, {"Drop LED events"}},
+                                       {"redirect-led", VariantType::Bool, false, {"Redirect LED events"}},
+                                       {"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}}};
+  o2::raw::HBFUtilsInitializer::addConfigOption(options);
+  workflowOptions.insert(workflowOptions.end(), options.begin(), options.end());
+}
+
+#include "Framework/runDataProcessing.h"
+
+WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
+{
+  // Calibration types
+  auto disableBadchannels = cfgc.options().get("no-badchannelcalib"),
+       disableTime = cfgc.options().get("no-timecalib"),
+       disableEnergy = cfgc.options().get("no-gaincalib");
+
+  // subpsecs for input and output
+  auto inputsubspec = cfgc.options().get("input-subspec"),
+       outputsubspec = cfgc.options().get("output-subspec");
+
+  // LED event handling
+  auto dropled = cfgc.options().get("drop-led"),
+       redirectled = cfgc.options().get("redirect-led");
+
+  if (dropled && redirectled) {
+    LOG(fatal) << "Ambiguous handling of LED events";
+  }
+  uint32_t ledsetting = 0;
+  if (dropled) {
+    ledsetting = 1;
+  }
+  if (redirectled) {
+    ledsetting = 2;
+  }
+
+  o2::conf::ConfigurableParam::updateFromString(cfgc.options().get("configKeyValues"));
+
+  WorkflowSpec specs;
+  specs.emplace_back(o2::emcal::getCellRecalibratorSpec(inputsubspec, outputsubspec, ledsetting, !disableBadchannels, !disableTime, !disableEnergy, cfgc.options().get("isMC")));
+
+  // configure dpl timer to inject correct firstTForbit: start from the 1st orbit of TF containing 1st sampled orbit
+  o2::raw::HBFUtilsInitializer hbfIni(cfgc, specs);
+  return specs;
+}
\ No newline at end of file
diff --git a/Detectors/EMCAL/workflow/src/cell-writer-workflow.cxx b/Detectors/EMCAL/workflow/src/cell-writer-workflow.cxx
index 234d32ffeccee..d3a7140f1abcc 100644
--- a/Detectors/EMCAL/workflow/src/cell-writer-workflow.cxx
+++ b/Detectors/EMCAL/workflow/src/cell-writer-workflow.cxx
@@ -27,7 +27,9 @@ using namespace o2::emcal;
 // we need to add workflow options before including Framework/runDataProcessing
 void customize(std::vector& workflowOptions)
 {
-  std::vector options{{"disable-mc", VariantType::Bool, false, {"Do not propagate MC labels"}},
+  std::vector options{{"cell-writer-name", VariantType::String, "emcal-cells-writer", {"Workflow name"}},
+                                       {"disable-mc", VariantType::Bool, false, {"Do not propagate MC labels"}},
+                                       {"subspec", VariantType::UInt32, 0U, {"Input subspecification"}},
                                        {"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}}};
   workflowOptions.insert(workflowOptions.end(), options.begin(), options.end());
 }
@@ -43,12 +45,14 @@ void customize(std::vector& policies)
 WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
 {
   bool disableMC = cfgc.options().get("disable-mc");
+  auto subspec = cfgc.options().get("subspec");
+  auto workflowname = cfgc.options().get("cell-writer-name");
   o2::conf::ConfigurableParam::updateFromString(cfgc.options().get("configKeyValues"));
 
   WorkflowSpec specs;
-  specs.emplace_back(MakeRootTreeWriterSpec("emcal-cells-writer", "emccells.root", "o2sim",
-                                            MakeRootTreeWriterSpec::BranchDefinition>{InputSpec{"data", "EMC", "CELLS", 0}, "EMCALCell", "cell-branch-name"},
-                                            MakeRootTreeWriterSpec::BranchDefinition>{InputSpec{"trigger", "EMC", "CELLSTRGR", 0}, "EMCALCellTRGR", "celltrigger-branch-name"},
-                                            MakeRootTreeWriterSpec::BranchDefinition>{InputSpec{"mc", "EMC", "CELLSMCTR", 0}, "EMCALCellMCTruth", "cellmc-branch-name", disableMC ? 0 : 1})());
+  specs.emplace_back(MakeRootTreeWriterSpec(workflowname.data(), "emccells.root", "o2sim",
+                                            MakeRootTreeWriterSpec::BranchDefinition>{InputSpec{"data", "EMC", "CELLS", subspec}, "EMCALCell", "cell-branch-name"},
+                                            MakeRootTreeWriterSpec::BranchDefinition>{InputSpec{"trigger", "EMC", "CELLSTRGR", subspec}, "EMCALCellTRGR", "celltrigger-branch-name"},
+                                            MakeRootTreeWriterSpec::BranchDefinition>{InputSpec{"mc", "EMC", "CELLSMCTR", subspec}, "EMCALCellMCTruth", "cellmc-branch-name", disableMC ? 0 : 1})());
   return specs;
 }
diff --git a/Detectors/EMCAL/workflow/src/emc-channel-data-producer.cxx b/Detectors/EMCAL/workflow/src/emc-channel-data-producer.cxx
index 122fa80117796..75e2cdbd9fce3 100644
--- a/Detectors/EMCAL/workflow/src/emc-channel-data-producer.cxx
+++ b/Detectors/EMCAL/workflow/src/emc-channel-data-producer.cxx
@@ -11,7 +11,6 @@
 
 /// \file  emc-channel-data-producer.cxx
 /// \brief This task generates an emcal event with a number of cells. It is designed to produce data for the testing of the bad channel and time calibration of the emcal
-///
 /// \author  Joshua Koenig 
 
 #include "DataFormatsEMCAL/Cell.h"
@@ -23,6 +22,7 @@
 
 #include "Framework/DataRefUtils.h"
 #include "Framework/ControlService.h"
+#include "Framework/RawDeviceService.h"
 #include "Algorithm/RangeTokenizer.h"
 #include 
 
@@ -34,7 +34,7 @@
 
 using namespace o2::framework;
 
-DataProcessorSpec generateData(const std::string nameRootFile, const std::string nameInputHist, const bool isTimeCalib, const int nCellsPerEvent);
+DataProcessorSpec generateData(const std::string nameRootFile, const std::string nameInputHist, const std::string nameInputHistAdd, const bool isTimeCalib, const int nCellsPerEvent, const int nEventsMax);
 
 // we need to add workflow options before including Framework/runDataProcessing
 void customize(std::vector& workflowOptions)
@@ -42,7 +42,9 @@ void customize(std::vector& workflowOptions)
   std::vector options{
     {"inputRootFile", VariantType::String, "", {"input root file from which data is taken, if empty, random data will be produced"}},
     {"nameInputHist", VariantType::String, "", {"name of the 2d histogram inside the root file used for the data generation"}},
+    {"nameInputHistAdd", VariantType::String, "", {"Same as nameInputHist but can be added in addition if the time distribution is also needed for the bad channel calibration"}},
     {"isInputTimeCalib", VariantType::Bool, false, {"input is produced for time clibration or bad channel calibration. Information is needed if inputRootFiel is specified as it has ifferent content for bad channel calib and time calib"}},
+    {"nEvents", VariantType::Int, 10000, {"number of events that will be processed"}},
     {"nCellsPerEvent", VariantType::Int, 5, {"number of cells per emcal triggered event"}}};
 
   std::swap(workflowOptions, options);
@@ -55,21 +57,26 @@ WorkflowSpec defineDataProcessing(ConfigContext const& config)
 
   const std::string nameRootFile = config.options().get("inputRootFile");
   const std::string nameInputHist = config.options().get("nameInputHist");
+  const std::string nameInputHistAdd = config.options().get("nameInputHistAdd");
   const bool isTimeCalib = config.options().get("isInputTimeCalib");
+  int nEventsMax = config.options().get("nEvents");
   const int nCellsPerEvent = config.options().get("nCellsPerEvent");
 
   WorkflowSpec workflow;
-  workflow.emplace_back(generateData(nameRootFile, nameInputHist, isTimeCalib, nCellsPerEvent));
+  workflow.emplace_back(generateData(nameRootFile, nameInputHist, nameInputHistAdd, isTimeCalib, nCellsPerEvent, nEventsMax));
 
   return workflow;
 }
 
-DataProcessorSpec generateData(const std::string nameRootFile, const std::string nameInputHist, const bool isTimeCalib, const int nCellsPerEvent)
+DataProcessorSpec generateData(const std::string nameRootFile, const std::string nameInputHist, const std::string nameInputHistAdd, const bool isTimeCalib, const int nCellsPerEvent, const int nEventsMax)
 {
   std::vector outputSpecs;
   outputSpecs.emplace_back(o2::header::gDataOriginEMC, "CELLS", 0, o2::framework::Lifetime::Timeframe);
   outputSpecs.emplace_back(o2::header::gDataOriginEMC, "CELLSTRGR", 0, o2::framework::Lifetime::Timeframe);
 
+  // bool if realistic time distribution should be used in bad channel calib
+  bool useBadCellTimeInfo = false;
+
   // initialize random number generators to generate cell time, cell energy and cellID
   std::default_random_engine generator;
   std::uniform_real_distribution<> disCellID(0, 17663);
@@ -78,16 +85,30 @@ DataProcessorSpec generateData(const std::string nameRootFile, const std::string
 
   // load 2d root file in case the paths are specified
   TH2F* h2d = nullptr;
+  std::array arrCellTimeHists;
   if (nameRootFile.find(".root") != std::string::npos) {
+
     TFile* f = TFile::Open(nameRootFile.c_str());
     if (!f) {
       LOG(error) << "root file does not exist " << nameRootFile;
     }
     h2d = (TH2F*)f->Get(nameInputHist.c_str());
-    h2d->SetDirectory(nullptr);
     if (!h2d) {
       LOG(error) << "histogram does not exist " << nameInputHist;
     }
+    h2d->SetDirectory(nullptr);
+    if (!nameInputHistAdd.empty()) {
+      TH2F* h2dAdd = (TH2F*)f->Get(nameInputHistAdd.c_str());
+      if (!h2dAdd) {
+        LOG(error) << "histogram does not exist " << nameInputHistAdd;
+      }
+      h2dAdd->SetDirectory(nullptr);
+      useBadCellTimeInfo = true;
+      for (int i = 0; i < 17664; ++i) {
+        arrCellTimeHists[i] = (TH1F*)h2dAdd->ProjectionX(Form("proj_cell_time_%i", i), i + 1, i + 1);
+        arrCellTimeHists[i]->SetDirectory(nullptr);
+      }
+    }
   }
 
   return DataProcessorSpec{
@@ -96,21 +117,37 @@ DataProcessorSpec generateData(const std::string nameRootFile, const std::string
     outputSpecs,
     AlgorithmSpec{
       [=](ProcessingContext& ctx) mutable {
+        // set up event counter. Return as soon as maximum number of events is reached
+        static int nEvent = 0;
+        if (nEvent >= nEventsMax) {
+          LOG(info) << "reached maximum amount of events requested " << nEvent << ". returning now";
+          ctx.services().get().endOfStream();
+        }
+        LOG(debug) << "process event " << nEvent;
+        nEvent++;
+
+        // loop over cells
+        // ToDo: Make more realistic assumption that we dont always have the same amount of cells per event
         o2::pmr::vector CellOutput;
         for (int i = 0; i < nCellsPerEvent; ++i) {
           double cellID = 0;
           double cellE = 0;
           double cellTime = 0;
-          if (h2d) {
+          if (h2d) { // get values from histogram
             // case for time calibration
             if (isTimeCalib) {
               h2d->GetRandom2(cellTime, cellID);
               cellE = disEnergy(generator);
+              // bad channel calibration
             } else {
               h2d->GetRandom2(cellE, cellID);
-              cellTime = disTime(generator);
+              if (useBadCellTimeInfo) {
+                cellTime = arrCellTimeHists[cellID]->GetRandom();
+              } else {
+                cellTime = disTime(generator);
+              }
             }
-          } else {
+          } else { // get values from random distributions
             cellID = disCellID(generator);
             cellE = disEnergy(generator);
             cellTime = disTime(generator);
@@ -118,11 +155,12 @@ DataProcessorSpec generateData(const std::string nameRootFile, const std::string
           // for now only consider low gain cells. Maybe implement high gain cells
           CellOutput.emplace_back(static_cast(cellID), cellE, cellTime, o2::emcal::ChannelType_t::LOW_GAIN);
         }
+        // send output
         LOG(debug) << "sending " << CellOutput.size() << "cells";
         o2::pmr::vector TriggerOutput;
         TriggerOutput.emplace_back(0, 0, 0, CellOutput.size());
 
-        ctx.outputs().adoptContainer(Output{o2::header::gDataOriginEMC, "CELLS", 0, Lifetime::Timeframe}, std::move(CellOutput));
-        ctx.outputs().adoptContainer(Output{o2::header::gDataOriginEMC, "CELLSTRGR", 0, Lifetime::Timeframe}, std::move(TriggerOutput));
+        ctx.outputs().adoptContainer(Output{o2::header::gDataOriginEMC, "CELLS", 0}, std::move(CellOutput));
+        ctx.outputs().adoptContainer(Output{o2::header::gDataOriginEMC, "CELLSTRGR", 0}, std::move(TriggerOutput));
       }}};
 }
diff --git a/Detectors/EMCAL/workflow/src/emc-offline-calib-workflow.cxx b/Detectors/EMCAL/workflow/src/emc-offline-calib-workflow.cxx
new file mode 100644
index 0000000000000..9ea41f2ee342b
--- /dev/null
+++ b/Detectors/EMCAL/workflow/src/emc-offline-calib-workflow.cxx
@@ -0,0 +1,54 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "EMCALWorkflow/OfflineCalibSpec.h"
+#include "CommonUtils/ConfigurableParam.h"
+#include "Framework/ConfigParamSpec.h"
+
+using namespace o2::framework;
+
+// ------------------------------------------------------------------
+
+// we need to add workflow options before including Framework/runDataProcessing
+void customize(std::vector& workflowOptions)
+{
+  // option allowing to set parameters
+  std::vector options{{"makeCellIDTimeEnergy", VariantType::Bool, false, {"list whether or not to make the cell ID, time, energy THnSparse"}},
+                                       {"no-rejectCalibTrigg", VariantType::Bool, false, {"if set to true, all events, including calibration triggered events, will be accepted"}},
+                                       {"input-subspec", VariantType::UInt32, 0U, {"Subspecification for input objects"}},
+                                       {"applyGainCalib", VariantType::Bool, false, {"Apply the gain calibration parameters for the bad channel calibration"}},
+                                       {"rejectL0Trigger", VariantType::Bool, false, {"Reject all emcal triggers except the minimum bias trigger"}},
+                                       {"ctpconfig-run-independent", VariantType::Bool, false, {"Use CTP config w/o runNumber tag"}},
+                                       {"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}}};
+  workflowOptions.insert(workflowOptions.end(), options.begin(), options.end());
+}
+
+// ------------------------------------------------------------------
+
+#include "Framework/runDataProcessing.h"
+
+WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
+{
+  WorkflowSpec wf;
+  // Update the (declared) parameters if changed from the command line
+  bool makeCellIDTimeEnergy = cfgc.options().get("makeCellIDTimeEnergy");
+  bool rejectCalibTrigg = !cfgc.options().get("no-rejectCalibTrigg");
+  bool doApplyGainCalib = cfgc.options().get("applyGainCalib");
+  bool doRejectL0Trigger = cfgc.options().get("rejectL0Trigger");
+  bool ctpcfgperrun = !cfgc.options().get("ctpconfig-run-independent");
+
+  // subpsecs for input
+  auto inputsubspec = cfgc.options().get("input-subspec");
+
+  o2::conf::ConfigurableParam::updateFromString(cfgc.options().get("configKeyValues"));
+  wf.emplace_back(o2::emcal::getEmcalOfflineCalibSpec(makeCellIDTimeEnergy, rejectCalibTrigg, doRejectL0Trigger, inputsubspec, doApplyGainCalib, ctpcfgperrun));
+  return wf;
+}
diff --git a/Detectors/EMCAL/workflow/src/emc-reco-workflow.cxx b/Detectors/EMCAL/workflow/src/emc-reco-workflow.cxx
index d9a3cf5a4fdd1..4e31a26ee4b5c 100644
--- a/Detectors/EMCAL/workflow/src/emc-reco-workflow.cxx
+++ b/Detectors/EMCAL/workflow/src/emc-reco-workflow.cxx
@@ -53,8 +53,10 @@ void customize(std::vector& workflowOptions)
     {"configKeyValues", o2::framework::VariantType::String, "", {"Semicolon separated key=value strings"}},
     {"disable-mc", o2::framework::VariantType::Bool, false, {"disable sending of MC information"}},
     {"disable-decoding-errors", o2::framework::VariantType::Bool, false, {"disable propagating decoding errors"}},
+    {"disable-trigger-reconstruction", o2::framework::VariantType::Bool, false, {"disable reconstruction of trigger data"}},
     {"ignore-dist-stf", o2::framework::VariantType::Bool, false, {"do not subscribe to FLP/DISTSUBTIMEFRAME/0 message (no lost TF recovery)"}},
-    {"subspecification", o2::framework::VariantType::Int, 0, {"Subspecification in case the workflow runs in parallel on multiple nodes (i.e. different FLPs)"}}};
+    {"subspecificationIn", o2::framework::VariantType::Int, 0, {"Subspecification for input in case the workflow runs in parallel on multiple nodes (i.e. different FLPs)"}},
+    {"subspecificationOut", o2::framework::VariantType::Int, 0, {"Subspecification for output in case the workflow runs in parallel on multiple nodes (i.e. different FLPs)"}}};
   o2::raw::HBFUtilsInitializer::addConfigOption(options);
   std::swap(workflowOptions, options);
 }
@@ -79,12 +81,14 @@ o2::framework::WorkflowSpec defineDataProcessing(o2::framework::ConfigContext co
   auto wf = o2::emcal::reco_workflow::getWorkflow(!cfgc.options().get("disable-mc"),
                                                   !cfgc.options().get("ignore-dist-stf"),
                                                   cfgc.options().get("enable-digits-printer"),
-                                                  cfgc.options().get("subspecification"),
+                                                  cfgc.options().get("subspecificationIn"),
+                                                  cfgc.options().get("subspecificationOut"),
                                                   cfgc.options().get("input-type"),
                                                   cfgc.options().get("output-type"),
                                                   cfgc.options().get("disable-root-input"),
                                                   cfgc.options().get("disable-root-output"),
-                                                  cfgc.options().get("disable-decoding-errors"));
+                                                  cfgc.options().get("disable-decoding-errors"),
+                                                  cfgc.options().get("disable-trigger-reconstruction"));
 
   // configure dpl timer to inject correct firstTForbit: start from the 1st orbit of TF containing 1st sampled orbit
   o2::raw::HBFUtilsInitializer hbfIni(cfgc, wf);
diff --git a/Detectors/EMCAL/workflow/src/entropy-encoder-workflow.cxx b/Detectors/EMCAL/workflow/src/entropy-encoder-workflow.cxx
index 2fe9aefcb5ed2..d5264aabc0566 100644
--- a/Detectors/EMCAL/workflow/src/entropy-encoder-workflow.cxx
+++ b/Detectors/EMCAL/workflow/src/entropy-encoder-workflow.cxx
@@ -21,7 +21,10 @@ using namespace o2::framework;
 void customize(std::vector& workflowOptions)
 {
   // option allowing to set parameters
-  std::vector options{ConfigParamSpec{"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}}};
+  std::vector options{
+    ConfigParamSpec{"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}},
+    ConfigParamSpec{"ctf-dict", VariantType::String, "none", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}},
+    ConfigParamSpec{"select-ir-frames", VariantType::Bool, false, {"Subscribe and filter according to external IR Frames"}}};
 
   std::swap(workflowOptions, options);
 }
@@ -35,6 +38,6 @@ WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
   WorkflowSpec wf;
   // Update the (declared) parameters if changed from the command line
   o2::conf::ConfigurableParam::updateFromString(cfgc.options().get("configKeyValues"));
-  wf.emplace_back(o2::emcal::getEntropyEncoderSpec());
+  wf.emplace_back(o2::emcal::getEntropyEncoderSpec(cfgc.options().get("select-ir-frames"), cfgc.options().get("ctf-dict")));
   return wf;
 }
diff --git a/Detectors/FIT/FDD/CMakeLists.txt b/Detectors/FIT/FDD/CMakeLists.txt
index 940cdd85b9441..a559e8d9f82f0 100644
--- a/Detectors/FIT/FDD/CMakeLists.txt
+++ b/Detectors/FIT/FDD/CMakeLists.txt
@@ -10,6 +10,10 @@
 # or submit itself to any jurisdiction.
 
 add_subdirectory(base)
+add_subdirectory(dcsmonitoring)
+if(BUILD_TESTING)
+    add_subdirectory(dcsmonitoring/macros)
+endif()
 add_subdirectory(raw)
 add_subdirectory(simulation)
 add_subdirectory(reconstruction)
diff --git a/Detectors/FIT/FDD/base/include/FDDBase/Constants.h b/Detectors/FIT/FDD/base/include/FDDBase/Constants.h
index d15da52232af3..f3c315457c46d 100644
--- a/Detectors/FIT/FDD/base/include/FDDBase/Constants.h
+++ b/Detectors/FIT/FDD/base/include/FDDBase/Constants.h
@@ -37,21 +37,6 @@ constexpr float PMNbOfSecElec = 6.0;          // Number of secondary electrons e
 
 constexpr int NTimeBinsPerBC = 256; // number of samples per BC
 
-// Detector TOF correction in ns
-constexpr float FDAdist = 1696.67;
-constexpr float FDCdist = 1954.4;
-constexpr float LayerWidth = 1.27;
-
-constexpr float getTOFCorrection(int det)
-{
-  constexpr float TOFCorr[4] = {
-    (FDCdist + LayerWidth) / o2::constants::physics::LightSpeedCm2NS,
-    (FDCdist - LayerWidth) / o2::constants::physics::LightSpeedCm2NS,
-    (FDAdist - LayerWidth) / o2::constants::physics::LightSpeedCm2NS,
-    (FDAdist + LayerWidth) / o2::constants::physics::LightSpeedCm2NS};
-  return TOFCorr[det];
-}
-
 } // namespace fdd
 } // namespace o2
 
diff --git a/Detectors/FIT/FDD/base/src/Geometry.cxx b/Detectors/FIT/FDD/base/src/Geometry.cxx
index 441a30fdda44f..149c086f4fc81 100644
--- a/Detectors/FIT/FDD/base/src/Geometry.cxx
+++ b/Detectors/FIT/FDD/base/src/Geometry.cxx
@@ -152,7 +152,7 @@ void Geometry::buildGeometry()
   if (!vCaveRB24) {
     LOG(fatal) << "Could not find the top volume for A-side";
   }
-  const Float_t kPosFDA = 1696.67 - 1313.347; // z-center of assembly (cm)
+  const Float_t kPosFDA = 1696.67 - 1313.347 - 75.; // z-center of assembly (cm)
   vCaveRB24->AddNode(vFDAarray, 1, new TGeoTranslation(0., 0., kPosFDA - kFDACelldz / 2. - 0.1));
   vCaveRB24->AddNode(vFDAarray, 2, new TGeoTranslation(0., 0., kPosFDA + kFDACelldz / 2. + 0.1));
 
diff --git a/Detectors/FIT/FDD/dcsmonitoring/CMakeLists.txt b/Detectors/FIT/FDD/dcsmonitoring/CMakeLists.txt
new file mode 100644
index 0000000000000..a13227dc90605
--- /dev/null
+++ b/Detectors/FIT/FDD/dcsmonitoring/CMakeLists.txt
@@ -0,0 +1,33 @@
+# Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+# See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+# All rights not expressly granted are reserved.
+#
+# This software is distributed under the terms of the GNU General Public
+# License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+#
+# In applying this license CERN does not waive the privileges and immunities
+# granted to it by virtue of its status as an Intergovernmental Organization
+# or submit itself to any jurisdiction.
+
+o2_add_library(FDDDCSMonitoring
+               SOURCES src/FDDDCSDataProcessor.cxx
+               PUBLIC_LINK_LIBRARIES O2::FITDCSMonitoring)
+
+o2_add_executable(dcs-sim-workflow
+                  COMPONENT_NAME fdd
+                  SOURCES workflow/fdd-dcs-sim-workflow.cxx
+                  PUBLIC_LINK_LIBRARIES O2::DCStestWorkflow)
+
+o2_add_executable(dcs-workflow
+                  COMPONENT_NAME fdd
+                  SOURCES workflow/fdd-dcs-data-workflow.cxx
+                  PUBLIC_LINK_LIBRARIES O2::DetectorsDCS
+                                        O2::FDDDCSMonitoring
+                                        O2::Framework)
+
+o2_add_executable(dcs-config-workflow
+                  COMPONENT_NAME fdd
+                  SOURCES workflow/fdd-dcs-config-workflow.cxx
+                  PUBLIC_LINK_LIBRARIES O2::DetectorsCalibration
+                                        O2::FITDCSMonitoring
+                                        O2::Framework)
diff --git a/Detectors/FIT/FDD/dcsmonitoring/include/FDDDCSMonitoring/FDDDCSDataProcessor.h b/Detectors/FIT/FDD/dcsmonitoring/include/FDDDCSMonitoring/FDDDCSDataProcessor.h
new file mode 100644
index 0000000000000..959640af12c86
--- /dev/null
+++ b/Detectors/FIT/FDD/dcsmonitoring/include/FDDDCSMonitoring/FDDDCSDataProcessor.h
@@ -0,0 +1,44 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// @file  FDDDCSDataProcessor.h
+/// @brief Task for processing FDD DCS data
+///
+/// \author Andreas Molander , University of Jyvaskyla, Finland
+
+#ifndef O2_FDD_DATAPROCESSOR_H
+#define O2_FDD_DATAPROCESSOR_H
+
+#include "DetectorsDCS/DataPointIdentifier.h"
+#include "FITDCSMonitoring/FITDCSDataProcessor.h"
+
+#include 
+#include 
+
+namespace o2
+{
+namespace fdd
+{
+
+class FDDDCSDataProcessor : public o2::fit::FITDCSDataProcessor
+{
+ public:
+  FDDDCSDataProcessor(const std::string detectorName, const o2::header::DataDescription& dataDescription)
+    : o2::fit::FITDCSDataProcessor(detectorName, dataDescription) {}
+
+ protected:
+  std::vector getHardCodedDPIDs() override;
+}; // end class
+
+} // namespace fdd
+} // namespace o2
+
+#endif // O2_FDD_DATAPROCESSOR_H
diff --git a/Detectors/FIT/FDD/dcsmonitoring/macros/CMakeLists.txt b/Detectors/FIT/FDD/dcsmonitoring/macros/CMakeLists.txt
new file mode 100644
index 0000000000000..63571c1001d3f
--- /dev/null
+++ b/Detectors/FIT/FDD/dcsmonitoring/macros/CMakeLists.txt
@@ -0,0 +1,15 @@
+# Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+# See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+# All rights not expressly granted are reserved.
+#
+# This software is distributed under the terms of the GNU General Public
+# License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+#
+# In applying this license CERN does not waive the privileges and immunities
+# granted to it by virtue of its status as an Intergovernmental Organization
+# or submit itself to any jurisdiction.
+
+o2_add_test_root_macro(makeFDDCCDBEntryForDCS.C
+                       PUBLIC_LINK_LIBRARIES O2::CCDB
+                                             O2::DetectorsDCS
+                       LABELS fdd)
diff --git a/Detectors/FIT/FDD/dcsmonitoring/macros/makeFDDCCDBEntryForDCS.C b/Detectors/FIT/FDD/dcsmonitoring/macros/makeFDDCCDBEntryForDCS.C
new file mode 100644
index 0000000000000..a3a170c92e175
--- /dev/null
+++ b/Detectors/FIT/FDD/dcsmonitoring/macros/makeFDDCCDBEntryForDCS.C
@@ -0,0 +1,98 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file makeFDDCCDBEntryForDCS.C
+/// \brief Macro for uploading a DCS data point definition object to CCDB
+///
+/// \author Andreas Molander , University of Jyvaskyla, Finland
+
+#include "CCDB/CcdbApi.h"
+#include "CCDB/CCDBTimeStampUtils.h"
+#include "DetectorsDCS/AliasExpander.h"
+#include "DetectorsDCS/DeliveryType.h"
+#include "DetectorsDCS/DataPointIdentifier.h"
+#include "TFile.h"
+
+#include 
+#include 
+#include 
+#include 
+
+using DPID = o2::dcs::DataPointIdentifier;
+
+int makeFDDCCDBEntryForDCS(const std::string ccdbUrl = "http://localhost:8080",
+                           const std::string fileName = "")
+{
+  std::unordered_map dpid2DataDesc;
+  std::vector aliasesHV = {"FDD/SIDE_A/HV_A9/[I,V]MON",
+                                        "FDD/SIDE_C/HV_C[9,32]/[I,V]MON",
+                                        "FDD/SIDE_C/LAYER0/PMT_0_[0..3]/[I,V]MON",
+                                        "FDD/SIDE_C/LAYER1/PMT_1_[0..3]/[I,V]MON",
+                                        "FDD/SIDE_A/LAYER2/PMT_2_[0..3]/[I,V]MON",
+                                        "FDD/SIDE_A/LAYER3/PMT_3_[0..3]/[I,V]MON"};
+  std::vector aliasesADC = {"FDD/PM/SIDE_A/PMT_A_9/ADC[0,1]_BASELINE",
+                                         "FDD/PM/SIDE_C/PMT_C_[9,32]/ADC[0,1]_BASELINE",
+                                         "FDD/PM/SIDE_C/LAYER0/PMT_0_[0..3]/ADC[0,1]_BASELINE",
+                                         "FDD/PM/SIDE_C/LAYER1/PMT_1_[0..3]/ADC[0,1]_BASELINE",
+                                         "FDD/PM/SIDE_A/LAYER2/PMT_2_[0..3]/ADC[0,1]_BASELINE",
+                                         "FDD/PM/SIDE_A/LAYER3/PMT_3_[0..3]/ADC[0,1]_BASELINE"};
+  std::vector aliasesRates = {"FDD/Trigger1_Central/CNT_RATE",
+                                           "FDD/Trigger2_SemiCentral/CNT_RATE",
+                                           "FDD/Trigger3_Vertex/CNT_RATE",
+                                           "FDD/Trigger4_OrC/CNT_RATE",
+                                           "FDD/Trigger5_OrA/CNT_RATE",
+                                           "FDD/Background/[0..9]/CNT_RATE",
+                                           "FDD/Background/[A,B,C,D,E,F,G,H]/CNT_RATE"};
+
+  std::vector expAliasesHV = o2::dcs::expandAliases(aliasesHV);
+  std::vector expAliasesADC = o2::dcs::expandAliases(aliasesADC);
+  std::vector expAliasesRates = o2::dcs::expandAliases(aliasesRates);
+
+  LOG(info) << "DCS DP IDs:";
+
+  DPID dpIdTmp;
+  for (size_t i = 0; i < expAliasesHV.size(); i++) {
+    DPID::FILL(dpIdTmp, expAliasesHV[i], o2::dcs::DeliveryType::DPVAL_DOUBLE);
+    dpid2DataDesc[dpIdTmp] = "FDDDATAPOINTS";
+    LOG(info) << dpIdTmp;
+  }
+  for (size_t i = 0; i < expAliasesADC.size(); i++) {
+    DPID::FILL(dpIdTmp, expAliasesADC[i], o2::dcs::DeliveryType::DPVAL_UINT);
+    dpid2DataDesc[dpIdTmp] = "FDDDATAPOINTS";
+    LOG(info) << dpIdTmp;
+  }
+  for (size_t i = 0; i < expAliasesRates.size(); i++) {
+    DPID::FILL(dpIdTmp, expAliasesRates[i], o2::dcs::DeliveryType::DPVAL_DOUBLE);
+    dpid2DataDesc[dpIdTmp] = "FDDDATAPOINTS";
+    LOG(info) << dpIdTmp;
+  }
+
+  LOG(info) << "Total number of DPs: " << dpid2DataDesc.size();
+
+  if (!ccdbUrl.empty()) {
+    const std::string ccdbPath = "FDD/Config/DCSDPconfig";
+    LOGP(info, "Storing DCS DP definition object on {}/{}", ccdbUrl, ccdbPath);
+    o2::ccdb::CcdbApi api;
+    api.init(ccdbUrl);
+    std::map metadata;
+    long ts = o2::ccdb::getCurrentTimestamp();
+    api.storeAsTFileAny(&dpid2DataDesc, ccdbPath, metadata, ts, 99999999999999);
+  }
+
+  if (!fileName.empty()) {
+    LOG(info) << "Storing DCS DP definitions locally in " << fileName;
+    TFile file(fileName.c_str(), "recreate");
+    file.WriteObjectAny(&dpid2DataDesc, "std::unordered_map", "DCSDPconfig");
+    file.Close();
+  }
+
+  return 0;
+}
diff --git a/Detectors/FIT/FDD/dcsmonitoring/src/FDDDCSDataProcessor.cxx b/Detectors/FIT/FDD/dcsmonitoring/src/FDDDCSDataProcessor.cxx
new file mode 100644
index 0000000000000..02da617263b3d
--- /dev/null
+++ b/Detectors/FIT/FDD/dcsmonitoring/src/FDDDCSDataProcessor.cxx
@@ -0,0 +1,60 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// @file  FDDDCSDataProcessor.cxx
+/// @brief Task for processing FDD DCS data
+///
+/// \author Andreas Molander , University of Jyvaskyla, Finland
+
+#include "FDDDCSMonitoring/FDDDCSDataProcessor.h"
+
+#include "DetectorsDCS/AliasExpander.h"
+#include "DetectorsDCS/DataPointIdentifier.h"
+
+#include 
+#include 
+
+std::vector o2::fdd::FDDDCSDataProcessor::getHardCodedDPIDs()
+{
+  std::vector vect;
+  std::vector aliasesHV = {"FDD/SIDE_A/HV_A9/[I,V]MON",
+                                        "FDD/SIDE_C/HV_C[9,32]/[I,V]MON",
+                                        "FDD/SIDE_C/LAYER0/PMT_0_[0..3]/[I,V]MON",
+                                        "FDD/SIDE_C/LAYER1/PMT_1_[0..3]/[I,V]MON",
+                                        "FDD/SIDE_A/LAYER2/PMT_2_[0..3]/[I,V]MON",
+                                        "FDD/SIDE_A/LAYER3/PMT_3_[0..3]/[I,V]MON"};
+  std::vector aliasesADC = {"FDD/PM/SIDE_A/PMT_A_9/ADC[0,1]_BASELINE",
+                                         "FDD/PM/SIDE_C/PMT_C_[9,32]/ADC[0,1]_BASELINE",
+                                         "FDD/PM/SIDE_C/LAYER0/PMT_0_[0..3]/ADC[0,1]_BASELINE",
+                                         "FDD/PM/SIDE_C/LAYER1/PMT_1_[0..3]/ADC[0,1]_BASELINE",
+                                         "FDD/PM/SIDE_A/LAYER2/PMT_2_[0..3]/ADC[0,1]_BASELINE",
+                                         "FDD/PM/SIDE_A/LAYER3/PMT_3_[0..3]/ADC[0,1]_BASELINE"};
+  std::vector aliasesRates = {"FDD/Trigger1_Central/CNT_RATE",
+                                           "FDD/Trigger2_SemiCentral/CNT_RATE",
+                                           "FDD/Trigger3_Vertex/CNT_RATE",
+                                           "FDD/Trigger4_OrC/CNT_RATE",
+                                           "FDD/Trigger5_OrA/CNT_RATE",
+                                           "FDD/Background/[0..9]/CNT_RATE",
+                                           "FDD/Background/[A,B,C,D,E,F,G,H]/CNT_RATE"};
+  std::vector expAliasesHV = o2::dcs::expandAliases(aliasesHV);
+  std::vector expAliasesADC = o2::dcs::expandAliases(aliasesADC);
+  std::vector expAliasesRates = o2::dcs::expandAliases(aliasesRates);
+  for (const auto& i : expAliasesHV) {
+    vect.emplace_back(i, o2::dcs::DPVAL_DOUBLE);
+  }
+  for (const auto& i : expAliasesADC) {
+    vect.emplace_back(i, o2::dcs::DPVAL_UINT);
+  }
+  for (const auto& i : expAliasesRates) {
+    vect.emplace_back(i, o2::dcs::DPVAL_DOUBLE);
+  }
+  return vect;
+}
diff --git a/Detectors/FIT/FDD/dcsmonitoring/workflow/FDDDCSConfigProcessorSpec.h b/Detectors/FIT/FDD/dcsmonitoring/workflow/FDDDCSConfigProcessorSpec.h
new file mode 100644
index 0000000000000..b2844fc0ac95a
--- /dev/null
+++ b/Detectors/FIT/FDD/dcsmonitoring/workflow/FDDDCSConfigProcessorSpec.h
@@ -0,0 +1,56 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file FDDDCSConfigProcessorSpec.cxx
+/// \brief FDD processor spec for DCS configurations
+///
+/// \author Andreas Molander , University of Jyvaskyla, Finland
+
+#ifndef O2_FDD_DCSCONFIGPROCESSOR_H
+#define O2_FDD_DCSCONFIGPROCESSOR_H
+
+#include "FITDCSMonitoring/FITDCSConfigProcessorSpec.h"
+#include "DetectorsCalibration/Utils.h"
+#include "Framework/WorkflowSpec.h"
+#include "Headers/DataHeader.h"
+
+#include 
+#include 
+
+namespace o2
+{
+
+namespace framework
+{
+
+DataProcessorSpec getFDDDCSConfigProcessorSpec()
+{
+  o2::header::DataDescription ddDChM = "FDD_DCHM";
+  std::vector outputs;
+  outputs.emplace_back(ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBPayload, ddDChM}, Lifetime::Sporadic);
+  outputs.emplace_back(ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBWrapper, ddDChM}, Lifetime::Sporadic);
+
+  return DataProcessorSpec{
+    "fdd-dcs-config-processor",
+    Inputs{{"inputConfig", o2::header::gDataOriginFDD, "DCS_CONFIG_FILE", Lifetime::Sporadic},
+           {"inputConfigFileName", o2::header::gDataOriginFDD, "DCS_CONFIG_NAME", Lifetime::Sporadic}},
+    outputs,
+    AlgorithmSpec{adaptFromTask("FDD", ddDChM)},
+    Options{{"use-verbose-mode", VariantType::Bool, false, {"Use verbose mode"}},
+            {"filename-dchm", VariantType::String, "FDD-deadchannels.txt", {"Dead channel map file name"}},
+            {"valid-days-dchm", VariantType::UInt32, 180u, {"Dead channel map validity in days"}},
+            {"no-validate", VariantType::Bool, false, {"Don't validate the CCDB uploads"}}}};
+}
+
+} // namespace framework
+} // namespace o2
+
+#endif // O2_FDD_DCSCONFIGPROCESSOR_H
diff --git a/Detectors/FIT/FDD/dcsmonitoring/workflow/FDDDCSDataProcessorSpec.h b/Detectors/FIT/FDD/dcsmonitoring/workflow/FDDDCSDataProcessorSpec.h
new file mode 100644
index 0000000000000..5f0a80da495b8
--- /dev/null
+++ b/Detectors/FIT/FDD/dcsmonitoring/workflow/FDDDCSDataProcessorSpec.h
@@ -0,0 +1,51 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// @file FDDDCSDataProcessorSpec.h
+/// @brief DataProcessorSpec for FDD DCS data
+///
+/// \author Andreas Molander , University of Jyvaskyla, Finland
+
+#ifndef O2_FDD_DATAPROCESSORSPEC_H
+#define O2_FDD_DATAPROCESSORSPEC_H
+
+#include "DetectorsCalibration/Utils.h"
+#include "FDDDCSMonitoring/FDDDCSDataProcessor.h"
+#include "Framework/Task.h"
+#include "Framework/WorkflowSpec.h"
+
+namespace o2
+{
+namespace framework
+{
+
+DataProcessorSpec getFDDDCSDataProcessorSpec()
+{
+  o2::header::DataDescription ddDCSDPs = "FDD_DCSDPs";
+  std::vector outputs;
+  outputs.emplace_back(ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBPayload, ddDCSDPs}, Lifetime::Sporadic);
+  outputs.emplace_back(ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBWrapper, ddDCSDPs}, Lifetime::Sporadic);
+
+  return DataProcessorSpec{
+    "fdd-dcs-data-processor",
+    Inputs{{"input", "DCS", "FDDDATAPOINTS"}},
+    outputs,
+    AlgorithmSpec{adaptFromTask("FDD", ddDCSDPs)},
+    Options{{"ccdb-path", VariantType::String, "http://localhost:8080", {"Path to CCDB"}},
+            {"use-ccdb-to-configure", VariantType::Bool, false, {"Use CCDB to configure"}},
+            {"use-verbose-mode", VariantType::Bool, false, {"Use verbose mode"}},
+            {"DPs-update-interval", VariantType::Int64, 600ll, {"Interval (in s) after which to update the DPs CCDB entry"}}}};
+}
+
+} // namespace framework
+} // namespace o2
+
+#endif // O2_FDD_DATAPROCESSORSPEC_H
diff --git a/Detectors/FIT/FDD/dcsmonitoring/workflow/fdd-dcs-config-workflow.cxx b/Detectors/FIT/FDD/dcsmonitoring/workflow/fdd-dcs-config-workflow.cxx
new file mode 100644
index 0000000000000..b6a18d7b45b66
--- /dev/null
+++ b/Detectors/FIT/FDD/dcsmonitoring/workflow/fdd-dcs-config-workflow.cxx
@@ -0,0 +1,35 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file fdd-dcs-config-workflow.cxx
+/// \brief Workflow for FDD DCS configuration processing
+///
+/// \author Andreas Molander , University of Jyvaskyla, Finland
+
+#include "Framework/DataProcessorSpec.h"
+#include "FDDDCSConfigProcessorSpec.h"
+
+using namespace o2::framework;
+
+// we need to add workflow options before including Framework/runDataProcessing.h
+void customize(std::vector& workflowOptions)
+{
+  // option allowing to set parameters
+}
+
+#include "Framework/runDataProcessing.h"
+
+WorkflowSpec defineDataProcessing(o2::framework::ConfigContext const& configcontext)
+{
+  WorkflowSpec specs;
+  specs.emplace_back(getFDDDCSConfigProcessorSpec());
+  return specs;
+}
diff --git a/Detectors/FIT/FDD/dcsmonitoring/workflow/fdd-dcs-data-workflow.cxx b/Detectors/FIT/FDD/dcsmonitoring/workflow/fdd-dcs-data-workflow.cxx
new file mode 100644
index 0000000000000..01a47e4cf196b
--- /dev/null
+++ b/Detectors/FIT/FDD/dcsmonitoring/workflow/fdd-dcs-data-workflow.cxx
@@ -0,0 +1,42 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file fdd-dcs-data-workflow.cxx
+/// \brief Workflow for FDD DCS data processing
+///
+/// \author Andreas Molander , University of Jyvaskyla, Finland
+
+#include "DetectorsDCS/DataPointIdentifier.h"
+#include "DetectorsDCS/DataPointValue.h"
+#include "FDDDCSDataProcessorSpec.h"
+#include "Framework/TypeTraits.h"
+
+#include 
+
+namespace o2::framework
+{
+template <>
+struct has_root_dictionary, void> : std::true_type {
+};
+} // namespace o2::framework
+
+#include "Framework/DataProcessorSpec.h"
+
+using namespace o2::framework;
+
+#include "Framework/runDataProcessing.h"
+
+WorkflowSpec defineDataProcessing(ConfigContext const& configcontext)
+{
+  WorkflowSpec specs;
+  specs.emplace_back(getFDDDCSDataProcessorSpec());
+  return specs;
+}
diff --git a/Detectors/FIT/FDD/dcsmonitoring/workflow/fdd-dcs-sim-workflow.cxx b/Detectors/FIT/FDD/dcsmonitoring/workflow/fdd-dcs-sim-workflow.cxx
new file mode 100644
index 0000000000000..bdf155a9c46b6
--- /dev/null
+++ b/Detectors/FIT/FDD/dcsmonitoring/workflow/fdd-dcs-sim-workflow.cxx
@@ -0,0 +1,46 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file fdd-dcs-sim-workflow.cxx
+/// \brief Simulate DCS data for FDD
+///
+/// \author Andreas Molander , University of Jyvaskyla, Finland
+
+#include "DCStestWorkflow/DCSRandomDataGeneratorSpec.h"
+#include "Framework/runDataProcessing.h"
+
+o2::framework::WorkflowSpec defineDataProcessing(o2::framework::ConfigContext const& configcontext)
+{
+  std::vector dphints;
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FDD/SIDE_A/HV_A9/[I,V]MON", 250, 350});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FDD/SIDE_C/HV_C[9,32]/[I,V]MON", 250, 350});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FDD/SIDE_C/LAYER0/PMT_0_[0..3]/[I,V]MON", 250, 350});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FDD/SIDE_C/LAYER1/PMT_1_[0..3]/[I,V]MON", 250, 350});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FDD/SIDE_A/LAYER2/PMT_2_[0..3]/[I,V]MON", 250, 350});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FDD/SIDE_A/LAYER3/PMT_3_[0..3]/[I,V]MON", 250, 350});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FDD/PM/SIDE_A/PMT_A_9/ADC[0,1]_BASELINE", 30, 150});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FDD/PM/SIDE_C/PMT_C_[9,32]/ADC[0,1]_BASELINE", 30, 150});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FDD/PM/SIDE_C/LAYER0/PMT_0_[0..3]/ADC[0,1]_BASELINE", 30, 150});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FDD/PM/SIDE_C/LAYER1/PMT_1_[0..3]/ADC[0,1]_BASELINE", 30, 150});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FDD/PM/SIDE_A/LAYER2/PMT_2_[0..3]/ADC[0,1]_BASELINE", 30, 150});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FDD/PM/SIDE_A/LAYER3/PMT_3_[0..3]/ADC[0,1]_BASELINE", 30, 150});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FDD/Trigger1_Central/CNT_RATE", 0, 5000000});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FDD/Trigger2_SemiCentral/CNT_RATE", 0, 5000000});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FDD/Trigger3_Vertex/CNT_RATE", 0, 5000000});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FDD/Trigger4_OrC/CNT_RATE", 0, 5000000});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FDD/Trigger5_OrA/CNT_RATE", 0, 5000000});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FDD/Background/[0..9]/CNT_RATE", 0, 50000});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FDD/Background/[A,B,C,D,E,F,G,H]/CNT_RATE", 0, 50000});
+
+  o2::framework::WorkflowSpec specs;
+  specs.emplace_back(o2::dcs::test::getDCSRandomDataGeneratorSpec(dphints, "FDD"));
+  return specs;
+}
diff --git a/Detectors/FIT/FDD/raw/include/FDDRaw/DataBlockFDD.h b/Detectors/FIT/FDD/raw/include/FDDRaw/DataBlockFDD.h
index a1499f3e2de25..71122cbeaf909 100644
--- a/Detectors/FIT/FDD/raw/include/FDDRaw/DataBlockFDD.h
+++ b/Detectors/FIT/FDD/raw/include/FDDRaw/DataBlockFDD.h
@@ -9,7 +9,7 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 //
-//file DataBlockFDD.h class  for RAW data format data blocks at FDD
+// file DataBlockFDD.h class  for RAW data format data blocks at FDD
 //
 // Artur.Furs
 // afurs@cern.ch
@@ -24,17 +24,17 @@ namespace o2
 {
 namespace fdd
 {
-//Raw event data for FDD
+// Raw event data for FDD
 using RawHeaderPM = o2::fdd::EventHeader;
 using RawDataPM = o2::fdd::EventData;
 using RawHeaderTCM = o2::fdd::EventHeader;
 using RawDataTCM = o2::fdd::TCMdata;
 using RawHeaderTCMext = o2::fdd::EventHeader;
 using RawDataTCMext = o2::fdd::TCMdataExtended;
-//Data block for FDD modules
-using DataBlockPM = o2::fit::DataBlockPM;
-using DataBlockTCM = o2::fit::DataBlockTCM;
-using DataBlockTCMext = o2::fit::DataBlockTCMext;
+// Data block for FDD modules, no padding - DataBlockConfig
+using DataBlockPM = o2::fit::DataBlockPM, RawHeaderPM, RawDataPM>;
+using DataBlockTCM = o2::fit::DataBlockTCM, RawHeaderTCM, RawDataTCM>;
+using DataBlockTCMext = o2::fit::DataBlockTCMext, RawHeaderTCMext, RawDataTCM, RawDataTCMext>;
 } // namespace fdd
 } // namespace o2
 #endif
diff --git a/Detectors/FIT/FDD/raw/include/FDDRaw/DigitBlockFDD.h b/Detectors/FIT/FDD/raw/include/FDDRaw/DigitBlockFDD.h
index ea86bfda3d406..1047413687dd9 100644
--- a/Detectors/FIT/FDD/raw/include/FDDRaw/DigitBlockFDD.h
+++ b/Detectors/FIT/FDD/raw/include/FDDRaw/DigitBlockFDD.h
@@ -9,7 +9,7 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 //
-//file DigitBlockFDD.h class  for proccessing RAW data into Digits
+// file DigitBlockFDD.h class  for proccessing RAW data into Digits
 //
 // Artur.Furs
 // afurs@cern.ch
@@ -25,10 +25,10 @@ namespace o2
 {
 namespace fdd
 {
-//Normal data taking mode
-using DigitBlockFDD = DigitBlockFIT;
-//TCM extended data taking mode
-using DigitBlockFDDext = DigitBlockFIText;
+// Normal data taking mode
+using DigitBlockFDD = o2::fit::DigitBlockFIT;
+// TCM extended data taking mode
+using DigitBlockFDDext = o2::fit::DigitBlockFIText;
 } // namespace fdd
 } // namespace o2
 #endif
diff --git a/Detectors/FIT/FDD/raw/include/FDDRaw/RawWriterFDD.h b/Detectors/FIT/FDD/raw/include/FDDRaw/RawWriterFDD.h
index 5cb267c40f63f..b04b347607b91 100644
--- a/Detectors/FIT/FDD/raw/include/FDDRaw/RawWriterFDD.h
+++ b/Detectors/FIT/FDD/raw/include/FDDRaw/RawWriterFDD.h
@@ -9,7 +9,7 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 //
-//file RawWriterFDD.h Raw writer class for FDD
+// file RawWriterFDD.h Raw writer class for FDD
 //
 // Artur.Furs
 // afurs@cern.ch
@@ -24,10 +24,12 @@ namespace o2
 {
 namespace fdd
 {
-//Normal TCM mode
+// Normal TCM mode
 using RawWriterFDD = o2::fit::RawWriterFIT;
-//Extended TCM mode
-//using RawWriterFDDext = o2::fit::RawWriterFIT;
+using RawWriterFDD_padded = o2::fit::RawWriterFIT;
+
+// Extended TCM mode
+// using RawWriterFDDext = o2::fit::RawWriterFIT;
 } // namespace fdd
 } // namespace o2
 
diff --git a/Detectors/FIT/FDD/raw/src/DataBlockFDD.cxx b/Detectors/FIT/FDD/raw/src/DataBlockFDD.cxx
index 944c3dbd598f5..53c98a0ac9e7a 100644
--- a/Detectors/FIT/FDD/raw/src/DataBlockFDD.cxx
+++ b/Detectors/FIT/FDD/raw/src/DataBlockFDD.cxx
@@ -10,4 +10,7 @@
 // or submit itself to any jurisdiction.
 
 #include "FDDRaw/DataBlockFDD.h"
-//using namespace o2::fdd;
+
+template class o2::fit::DataBlockPM, o2::fdd::RawHeaderPM, o2::fdd::RawDataPM>;
+template class o2::fit::DataBlockTCM, o2::fdd::RawHeaderTCM, o2::fdd::RawDataTCM>;
+template class o2::fit::DataBlockTCMext, o2::fdd::RawHeaderTCMext, o2::fdd::RawDataTCM, o2::fdd::RawDataTCMext>;
diff --git a/Detectors/FIT/FDD/raw/src/DigitBlockFDD.cxx b/Detectors/FIT/FDD/raw/src/DigitBlockFDD.cxx
index fe4948ab9480b..0e1bc72bef652 100644
--- a/Detectors/FIT/FDD/raw/src/DigitBlockFDD.cxx
+++ b/Detectors/FIT/FDD/raw/src/DigitBlockFDD.cxx
@@ -10,4 +10,6 @@
 // or submit itself to any jurisdiction.
 
 #include "FDDRaw/DigitBlockFDD.h"
-//using namespace o2::fdd;
+
+template class o2::fit::DigitBlockFIT;
+template class o2::fit::DigitBlockFIText;
\ No newline at end of file
diff --git a/Detectors/FIT/FDD/raw/src/RawReaderFDDBase.cxx b/Detectors/FIT/FDD/raw/src/RawReaderFDDBase.cxx
index 3708c53d3f03b..5c500b8dac5ca 100644
--- a/Detectors/FIT/FDD/raw/src/RawReaderFDDBase.cxx
+++ b/Detectors/FIT/FDD/raw/src/RawReaderFDDBase.cxx
@@ -10,4 +10,6 @@
 // or submit itself to any jurisdiction.
 
 #include "FDDRaw/RawReaderFDDBase.h"
-//using namespace o2::fdd;
+
+template class o2::fit::RawReaderBaseFIT;
+template class o2::fit::RawReaderBaseFIT;
diff --git a/Detectors/FIT/FDD/raw/src/RawWriterFDD.cxx b/Detectors/FIT/FDD/raw/src/RawWriterFDD.cxx
index 29b364ce400be..69d4e23a99815 100644
--- a/Detectors/FIT/FDD/raw/src/RawWriterFDD.cxx
+++ b/Detectors/FIT/FDD/raw/src/RawWriterFDD.cxx
@@ -10,3 +10,6 @@
 // or submit itself to any jurisdiction.
 
 #include "FDDRaw/RawWriterFDD.h"
+
+template class o2::fit::RawWriterFIT;
+template class o2::fit::RawWriterFIT;
diff --git a/Detectors/FIT/FDD/reconstruction/CMakeLists.txt b/Detectors/FIT/FDD/reconstruction/CMakeLists.txt
index bc4ad8c2c3f3f..4475ed5e3a404 100644
--- a/Detectors/FIT/FDD/reconstruction/CMakeLists.txt
+++ b/Detectors/FIT/FDD/reconstruction/CMakeLists.txt
@@ -14,6 +14,7 @@ o2_add_library(FDDReconstruction
                        src/CTFCoder.cxx
                PUBLIC_LINK_LIBRARIES O2::FDDBase
                                      O2::DataFormatsFDD
+                                     O2::DetectorsBase
                                      O2::DetectorsRaw)
 
 o2_target_root_dictionary(FDDReconstruction
diff --git a/Detectors/FIT/FDD/reconstruction/include/FDDReconstruction/CTFCoder.h b/Detectors/FIT/FDD/reconstruction/include/FDDReconstruction/CTFCoder.h
index 5082f4a8f79bf..c62e013447416 100644
--- a/Detectors/FIT/FDD/reconstruction/include/FDDReconstruction/CTFCoder.h
+++ b/Detectors/FIT/FDD/reconstruction/include/FDDReconstruction/CTFCoder.h
@@ -25,7 +25,6 @@
 #include "DataFormatsFDD/ChannelData.h"
 #include "DetectorsCommonDataFormats/DetID.h"
 #include "DetectorsBase/CTFCoderBase.h"
-#include "rANS/rans.h"
 
 class TTree;
 
@@ -34,10 +33,10 @@ namespace o2
 namespace fdd
 {
 
-class CTFCoder : public o2::ctf::CTFCoderBase
+class CTFCoder final : public o2::ctf::CTFCoderBase
 {
  public:
-  CTFCoder(o2::ctf::CTFCoderBase::OpType op) : o2::ctf::CTFCoderBase(op, CTF::getNBlocks(), o2::detectors::DetID::FDD) {}
+  CTFCoder(o2::ctf::CTFCoderBase::OpType op, const std::string& ctfdictOpt = "none") : o2::ctf::CTFCoderBase(op, CTF::getNBlocks(), o2::detectors::DetID::FDD, 1.f, ctfdictOpt) {}
   ~CTFCoder() final = default;
 
   /// entropy-encode digits to buffer with CTF
@@ -72,15 +71,15 @@ o2::ctf::CTFIOSize CTFCoder::encode(VEC& buff, const gsl::span& dig
   using MD = o2::ctf::Metadata::OptStore;
   // what to do which each field: see o2::ctd::Metadata explanation
   constexpr MD optField[CTF::getNBlocks()] = {
-    MD::EENCODE, // BLC_trigger
-    MD::EENCODE, // BLC_bcInc
-    MD::EENCODE, // BLC_orbitInc
-    MD::EENCODE, // BLC_nChan
+    MD::EENCODE_OR_PACK, // BLC_trigger
+    MD::EENCODE_OR_PACK, // BLC_bcInc
+    MD::EENCODE_OR_PACK, // BLC_orbitInc
+    MD::EENCODE_OR_PACK, // BLC_nChan
 
-    MD::EENCODE, // BLC_idChan
-    MD::EENCODE, // BLC_time
-    MD::EENCODE, // BLC_charge
-    MD::EENCODE  // BLC_feeBits
+    MD::EENCODE_OR_PACK, // BLC_idChan
+    MD::EENCODE_OR_PACK, // BLC_time
+    MD::EENCODE_OR_PACK, // BLC_charge
+    MD::EENCODE_OR_PACK  // BLC_feeBits
   };
   CompressedDigits cd;
   if (mExtHeader.isValidDictTimeStamp()) {
@@ -101,11 +100,11 @@ o2::ctf::CTFIOSize CTFCoder::encode(VEC& buff, const gsl::span& dig
 
   ec->setHeader(cd.header);
   assignDictVersion(static_cast(ec->getHeader()));
-  ec->getANSHeader().majorVersion = 0;
-  ec->getANSHeader().minorVersion = 1;
+
+  ec->setANSHeader(mANSVersion);
   // at every encoding the buffer might be autoexpanded, so we don't work with fixed pointer ec
   o2::ctf::CTFIOSize iosize;
-#define ENCODEFDD(part, slot, bits) CTF::get(buff.data())->encode(part, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)].get(), getMemMarginFactor());
+#define ENCODEFDD(part, slot, bits) CTF::get(buff.data())->encode(part, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)], getMemMarginFactor());
   // clang-format off
   iosize += ENCODEFDD(cd.trigger,   CTF::BLC_trigger,  0);
   iosize += ENCODEFDD(cd.bcInc,     CTF::BLC_bcInc,    0);
@@ -133,7 +132,7 @@ o2::ctf::CTFIOSize CTFCoder::decode(const CTF::base& ec, VDIG& digitVec, VCHAN&
   checkDictVersion(hd);
   ec.print(getPrefix(), mVerbosity);
   o2::ctf::CTFIOSize iosize;
-#define DECODEFDD(part, slot) ec.decode(part, int(slot), mCoders[int(slot)].get())
+#define DECODEFDD(part, slot) ec.decode(part, int(slot), mCoders[int(slot)])
   // clang-format off
   iosize += DECODEFDD(cd.trigger,   CTF::BLC_trigger);
   iosize += DECODEFDD(cd.bcInc,     CTF::BLC_bcInc);
@@ -228,12 +227,30 @@ void CTFCoder::compress(CompressedDigits& cd, const gsl::span& digi
 {
   // convert digits/channel to their compressed version
   cd.clear();
+  cd.header.det = mDet;
   if (!digitVec.size()) {
     return;
   }
-  const auto& dig0 = digitVec[0];
-  cd.header.det = mDet;
-  cd.header.nTriggers = digitVec.size();
+  uint32_t firstDig = digitVec.size(), nDigSel = digitVec.size(), nChanSel = channelVec.size();
+  std::vector reject(digitVec.size());
+  if (mIRFrameSelector.isSet()) {
+    for (size_t id = 0; id < digitVec.size(); id++) {
+      if (mIRFrameSelector.check(digitVec[id].mIntRecord) < 0) {
+        reject[id] = true;
+        nDigSel--;
+        nChanSel -= digitVec[id].ref.getEntries();
+      } else if (firstDig == digitVec.size()) {
+        firstDig = id;
+      }
+    }
+  } else {
+    firstDig = 0;
+  }
+  if (nDigSel == 0) { // nothing is selected
+    return;
+  }
+  const auto& dig0 = digitVec[firstDig];
+  cd.header.nTriggers = nDigSel;
   cd.header.firstOrbit = dig0.mIntRecord.orbit;
   cd.header.firstBC = dig0.mIntRecord.bc;
 
@@ -242,37 +259,41 @@ void CTFCoder::compress(CompressedDigits& cd, const gsl::span& digi
   cd.orbitInc.resize(cd.header.nTriggers);
   cd.nChan.resize(cd.header.nTriggers);
 
-  cd.idChan.resize(channelVec.size());
-  cd.time.resize(channelVec.size());
-  cd.charge.resize(channelVec.size());
-  cd.feeBits.resize(channelVec.size());
+  cd.idChan.resize(nChanSel);
+  cd.time.resize(nChanSel);
+  cd.charge.resize(nChanSel);
+  cd.feeBits.resize(nChanSel);
 
   uint16_t prevBC = cd.header.firstBC;
   uint32_t prevOrbit = cd.header.firstOrbit;
-  uint32_t ccount = 0;
-  for (uint32_t idig = 0; idig < cd.header.nTriggers; idig++) {
+  uint32_t ccount = 0, dcount = 0;
+  for (uint32_t idig = 0; idig < digitVec.size(); idig++) {
+    if (reject[idig]) {
+      continue;
+    }
     const auto& digit = digitVec[idig];
     const auto chanels = digit.getBunchChannelData(channelVec); // we assume the channels are sorted
 
     // fill trigger info
-    cd.trigger[idig] = digit.mTriggers.getTriggersignals();
+    cd.trigger[dcount] = digit.mTriggers.getTriggersignals();
     if (prevOrbit == digit.mIntRecord.orbit) {
-      cd.bcInc[idig] = digit.mIntRecord.bc - prevBC;
-      cd.orbitInc[idig] = 0;
+      cd.bcInc[dcount] = digit.mIntRecord.bc - prevBC;
+      cd.orbitInc[dcount] = 0;
     } else {
-      cd.bcInc[idig] = digit.mIntRecord.bc;
-      cd.orbitInc[idig] = digit.mIntRecord.orbit - prevOrbit;
+      cd.bcInc[dcount] = digit.mIntRecord.bc;
+      cd.orbitInc[dcount] = digit.mIntRecord.orbit - prevOrbit;
     }
     prevBC = digit.mIntRecord.bc;
     prevOrbit = digit.mIntRecord.orbit;
     // fill channels info
-    cd.nChan[idig] = chanels.size();
-    if (!cd.nChan[idig]) {
+    cd.nChan[dcount] = chanels.size();
+    if (!cd.nChan[dcount]) {
       LOG(debug) << "Digits with no channels";
+      dcount++;
       continue;
     }
     uint8_t prevChan = 0;
-    for (uint8_t ic = 0; ic < cd.nChan[idig]; ic++) {
+    for (uint8_t ic = 0; ic < cd.nChan[dcount]; ic++) {
       if constexpr (MINOR_VERSION == 0 && MAJOR_VERSION == 1) {
         cd.idChan[ccount] = chanels[ic].mPMNumber - prevChan; // Old method, lets keep it for a while
       } else {
@@ -284,6 +305,7 @@ void CTFCoder::compress(CompressedDigits& cd, const gsl::span& digi
       prevChan = chanels[ic].mPMNumber;
       ccount++;
     }
+    dcount++;
   }
 }
 
diff --git a/Detectors/FIT/FDD/reconstruction/include/FDDReconstruction/ReadRaw.h b/Detectors/FIT/FDD/reconstruction/include/FDDReconstruction/ReadRaw.h
deleted file mode 100644
index 54c8b7b203edb..0000000000000
--- a/Detectors/FIT/FDD/reconstruction/include/FDDReconstruction/ReadRaw.h
+++ /dev/null
@@ -1,68 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-/// \file ReadRaw.h
-/// \brief Reads raw data and converts to digits
-/// \author Maciej.Slupecki@cern.ch, arvind.khuntia@cern.ch, based on the FT0 code
-// RAW data format description: DataFormat/Detectors/FIT/FDD/RawEventData
-
-#ifndef ALICEO2_FDD_READRAW_H_
-#define ALICEO2_FDD_READRAW_H_
-
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include "TBranch.h"
-#include "TTree.h"
-#include "CommonDataFormat/InteractionRecord.h"
-#include "DataFormatsFDD/Digit.h"
-#include "DataFormatsFDD/ChannelData.h"
-#include "DataFormatsFDD/LookUpTable.h"
-#include "DataFormatsFDD/RawEventData.h"
-
-namespace o2
-{
-namespace fdd
-{
-class ReadRaw
-{
- public:
-  ReadRaw() = default;
-  ReadRaw(bool doConversionToDigits, const std::string inputRawFilePath = "fdd.raw", const std::string outputRawFilePath = "fdddigitsFromRaw.root");
-  void readRawData(const LookUpTable& lut);
-  void writeDigits(const std::string& outputDigitsFilePath);
-  void close();
-
- private:
-  std::ifstream mRawFileIn;
-  std::map> mDigitAccum; // digit accumulator
-
-  template 
-  TBranch* getOrMakeBranch(TTree& tree, std::string brname, T* ptr)
-  {
-    if (auto br = tree.GetBranch(brname.c_str())) {
-      br->SetAddress(static_cast(ptr));
-      return br;
-    }
-    // otherwise make it
-    return tree.Branch(brname.c_str(), ptr);
-  }
-
-  ClassDefNV(ReadRaw, 1);
-};
-
-} // namespace fdd
-} // namespace o2
-#endif
diff --git a/Detectors/FIT/FDD/reconstruction/include/FDDReconstruction/Reconstructor.h b/Detectors/FIT/FDD/reconstruction/include/FDDReconstruction/Reconstructor.h
index 91268881f74e6..8881605b652ac 100644
--- a/Detectors/FIT/FDD/reconstruction/include/FDDReconstruction/Reconstructor.h
+++ b/Detectors/FIT/FDD/reconstruction/include/FDDReconstruction/Reconstructor.h
@@ -17,6 +17,7 @@
 #include 
 #include "DataFormatsFDD/Digit.h"
 #include "DataFormatsFDD/RecPoint.h"
+#include "DataFormatsFIT/DeadChannelMap.h"
 namespace o2
 {
 namespace fdd
@@ -26,13 +27,20 @@ class Reconstructor
  public:
   Reconstructor() = default;
   ~Reconstructor() = default;
-  o2::fdd::RecPoint process(o2::fdd::Digit const& digitBC,
-                            gsl::span inChData,
-                            gsl::span outChData);
-
+  void process(o2::fdd::Digit const& digitBC,
+               gsl::span inChData,
+               std::vector& RecPoints,
+               std::vector& outChData);
   void finish();
 
+  void setDeadChannelMap(o2::fit::DeadChannelMap const* deadChannelMap)
+  {
+    LOG(info) << "Updated dead channel map";
+    mDeadChannelMap = deadChannelMap;
+  }
+
  private:
+  o2::fit::DeadChannelMap const* mDeadChannelMap = nullptr;
   ClassDefNV(Reconstructor, 3);
 };
 } // namespace fdd
diff --git a/Detectors/FIT/FDD/reconstruction/src/CTFCoder.cxx b/Detectors/FIT/FDD/reconstruction/src/CTFCoder.cxx
index 315f9ac346d89..ae20a299c8ebf 100644
--- a/Detectors/FIT/FDD/reconstruction/src/CTFCoder.cxx
+++ b/Detectors/FIT/FDD/reconstruction/src/CTFCoder.cxx
@@ -51,7 +51,7 @@ void CTFCoder::createCoders(const std::vector& bufVec, o2::ctf::CTFCoderBa
 {
   const auto ctf = CTF::getImage(bufVec.data());
   CompressedDigits cd; // just to get member types
-#define MAKECODER(part, slot) createCoder(op, ctf.getFrequencyTable(slot), int(slot))
+#define MAKECODER(part, slot) createCoder(op, std::get>(ctf.getDictionary(slot, mANSVersion)), int(slot))
   // clang-format off
   MAKECODER(cd.trigger,   CTF::BLC_trigger);
   MAKECODER(cd.bcInc,     CTF::BLC_bcInc);
@@ -69,22 +69,17 @@ void CTFCoder::createCoders(const std::vector& bufVec, o2::ctf::CTFCoderBa
 size_t CTFCoder::estimateCompressedSize(const CompressedDigits& cd)
 {
   size_t sz = 0;
-  // clang-format off
   // RS FIXME this is very crude estimate, instead, an empirical values should be used
-#define VTP(vec) typename std::remove_reference::type::value_type
-#define ESTSIZE(vec, slot) mCoders[int(slot)] ?                         \
-  rans::calculateMaxBufferSize(vec.size(), reinterpret_cast*>(mCoders[int(slot)].get())->getAlphabetRangeBits(), sizeof(VTP(vec)) ) : vec.size()*sizeof(VTP(vec))
-  sz += ESTSIZE(cd.trigger,   CTF::BLC_trigger);
-  sz += ESTSIZE(cd.bcInc,     CTF::BLC_bcInc);
-  sz += ESTSIZE(cd.orbitInc,  CTF::BLC_orbitInc);
-  sz += ESTSIZE(cd.nChan,     CTF::BLC_nChan);
+  sz += estimateBufferSize(static_cast(CTF::BLC_trigger), cd.trigger);
+  sz += estimateBufferSize(static_cast(CTF::BLC_bcInc), cd.bcInc);
+  sz += estimateBufferSize(static_cast(CTF::BLC_orbitInc), cd.orbitInc);
+  sz += estimateBufferSize(static_cast(CTF::BLC_nChan), cd.nChan);
 
-  sz += ESTSIZE(cd.idChan,    CTF::BLC_idChan);
-  sz += ESTSIZE(cd.time,      CTF::BLC_time);
-  sz += ESTSIZE(cd.charge,    CTF::BLC_charge);
-  sz += ESTSIZE(cd.feeBits,   CTF::BLC_feeBits);
-  // clang-format on
+  sz += estimateBufferSize(static_cast(CTF::BLC_idChan), cd.idChan);
+  sz += estimateBufferSize(static_cast(CTF::BLC_time), cd.time);
+  sz += estimateBufferSize(static_cast(CTF::BLC_charge), cd.charge);
+  sz += estimateBufferSize(static_cast(CTF::BLC_feeBits), cd.feeBits);
 
-  LOG(info) << "Estimated output size is " << sz << " bytes";
+  LOG(debug) << "Estimated output size is " << sz << " bytes";
   return sz;
 }
diff --git a/Detectors/FIT/FDD/reconstruction/src/Reconstructor.cxx b/Detectors/FIT/FDD/reconstruction/src/Reconstructor.cxx
index ad4ad6cdb3866..7d133e30df08e 100644
--- a/Detectors/FIT/FDD/reconstruction/src/Reconstructor.cxx
+++ b/Detectors/FIT/FDD/reconstruction/src/Reconstructor.cxx
@@ -17,53 +17,79 @@
 #include "FDDBase/Constants.h"
 #include 
 #include 
-#include "FairLogger.h"
+#include 
 
 using namespace o2::fdd;
 
 //_____________________________________________________________________
-o2::fdd::RecPoint Reconstructor::process(o2::fdd::Digit const& digitBC,
-                                         gsl::span inChData,
-                                         gsl::span outChData)
+void Reconstructor::process(o2::fdd::Digit const& digitBC, gsl::span inChData,
+                            std::vector& RecPoints, std::vector& outChData)
+// gsl::span outChData)
 {
   // Compute charge weighted average time
   Double_t timeFDA = 0, timeFDC = 0;
   Double_t weightFDA = 0.0, weightFDC = 0.0;
-
+  Int_t nInTimeA = 0, nInTimeC = 0;
+  int firstEntry = outChData.size();
+  int nStored = 0;
   int nch = inChData.size();
+
   for (int ich = 0; ich < nch; ich++) {
-    outChData[ich] = o2::fdd::ChannelDataFloat{inChData[ich].mPMNumber,
-                                               (inChData[ich].mTime) * timePerTDC,
-                                               (double)inChData[ich].mChargeADC,
-                                               inChData[ich].mFEEBits};
+    if (mDeadChannelMap && !mDeadChannelMap->isChannelAlive(inChData[ich].mPMNumber)) {
+      LOG(debug) << "Channel " << ich << " is dead - discarding data";
+      continue;
+    }
+    bool inTime = inChData[ich].getFlag(ChannelData::EEventDataBit::kIsEventInTVDC);
+    bool inAdcGate = inChData[ich].getFlag(ChannelData::EEventDataBit::kIsCFDinADCgate);
+    if (inAdcGate) {
+      outChData.emplace_back((int)inChData[ich].mPMNumber, (inChData[ich].mTime) * timePerTDC,
+                             (double)inChData[ich].mChargeADC, inChData[ich].mFEEBits);
+      nStored++;
+    }
 
-    Float_t adc = outChData[ich].mChargeADC;
-    Float_t time = outChData[ich].mTime;
+    Float_t adc = inChData[ich].mChargeADC;
+    Float_t time = (inChData[ich].mTime) * timePerTDC;
     if (time == o2::InteractionRecord::DummyTime) {
       continue;
     }
     Float_t timeErr = 1;
-    if (adc > 1) {
-      timeErr = 1 / adc;
+    if (adc > 3) {
+      timeErr = 1. / adc;
     }
-    if (outChData[ich].mPMNumber < 8) {
+    if ((int)inChData[ich].mPMNumber < 8) {
+      nInTimeC += inTime;
       timeFDC += time / (timeErr * timeErr);
       weightFDC += 1. / (timeErr * timeErr);
     } else {
+      nInTimeA += inTime;
       timeFDA += time / (timeErr * timeErr);
       weightFDA += 1. / (timeErr * timeErr);
     }
   }
   const int nsToPs = 1e3;
   std::array mCollisionTime = {o2::fdd::RecPoint::sDummyCollissionTime, o2::fdd::RecPoint::sDummyCollissionTime};
-
-  mCollisionTime[o2::fdd::RecPoint::TimeA] = (weightFDA > 1) ? round(timeFDA / weightFDA * nsToPs) : o2::fdd::RecPoint::sDummyCollissionTime;
-  mCollisionTime[o2::fdd::RecPoint::TimeC] = (weightFDC > 1) ? round(timeFDC / weightFDC * nsToPs) : o2::fdd::RecPoint::sDummyCollissionTime;
-  return RecPoint{mCollisionTime, digitBC.ref.getFirstEntry(), digitBC.ref.getEntries(), digitBC.getIntRecord(), digitBC.mTriggers};
+  /// Avg time for each side, only if one channel satisfy the TVDC condition (if not, also avg time is propagated for background study using AO2D)
+  if (nInTimeA > 0) {
+    mCollisionTime[o2::fdd::RecPoint::TimeA] = (weightFDA > 1) ? round(timeFDA / weightFDA * nsToPs)
+                                                               : o2::fdd::RecPoint::sDummyCollissionTime;
+  } else {
+    if (weightFDA > 0) {
+      mCollisionTime[o2::fdd::RecPoint::TimeA] = round(timeFDA / weightFDA * nsToPs);
+    }
+  }
+  if (nInTimeC > 0) {
+    mCollisionTime[o2::fdd::RecPoint::TimeC] = (weightFDC > 1) ? round(timeFDC / weightFDC * nsToPs)
+                                                               : o2::fdd::RecPoint::sDummyCollissionTime;
+  } else {
+    if (weightFDC > 0) {
+      mCollisionTime[o2::fdd::RecPoint::TimeC] = round(timeFDC / weightFDC * nsToPs);
+    }
+  }
+  RecPoints.emplace_back(mCollisionTime, firstEntry, nStored, digitBC.getIntRecord(), digitBC.mTriggers);
 }
 //________________________________________________________
 void Reconstructor::finish()
 {
-  // finalize digitization, if needed, flash remaining digits
+  // finalize reconstruction, if needed, flash remaining recpoints
   // if (!mContinuous)   return;
 }
diff --git a/Detectors/FIT/FDD/simulation/CMakeLists.txt b/Detectors/FIT/FDD/simulation/CMakeLists.txt
index e5de98b6b8ece..ec217c4419830 100644
--- a/Detectors/FIT/FDD/simulation/CMakeLists.txt
+++ b/Detectors/FIT/FDD/simulation/CMakeLists.txt
@@ -11,27 +11,25 @@
 
 o2_add_library(FDDSimulation
         SOURCES src/Detector.cxx
-        src/Digitizer.cxx
-        src/Digits2Raw.cxx
-        PUBLIC_LINK_LIBRARIES O2::SimulationDataFormat
-        O2::FDDBase
-        O2::DataFormatsFDD
-        O2::DetectorsRaw
-        ROOT::Physics)
+                src/Digitizer.cxx
+        PUBLIC_LINK_LIBRARIES O2::CommonUtils
+                              O2::SimulationDataFormat
+                              O2::FDDBase
+                              O2::DataFormatsFDD
+                              O2::DetectorsRaw
+                              O2::DetectorsBase
+                              ROOT::Physics)
 
 o2_target_root_dictionary(FDDSimulation
         HEADERS include/FDDSimulation/Detector.h
-        include/FDDSimulation/Digitizer.h
-        include/FDDSimulation/DigitizationParameters.h
-        include/FDDSimulation/Digits2Raw.h)
+                include/FDDSimulation/Digitizer.h
+                include/FDDSimulation/DigitizationParameters.h
+                include/FDDSimulation/FDDDigParam.h)
 
 o2_add_executable(digit2raw
         COMPONENT_NAME fdd
         SOURCES src/digit2raw.cxx
-        PUBLIC_LINK_LIBRARIES O2::FDDSimulation
-        O2::DetectorsRaw
-        O2::DetectorsCommonDataFormats
-        O2::CommonUtils
-        Boost::program_options
-        O2::FDDRaw)
+        PUBLIC_LINK_LIBRARIES O2::FDDRaw
+                              Boost::program_options)
+
 o2_data_file(COPY data DESTINATION Detectors/FDD/simulation)
\ No newline at end of file
diff --git a/Detectors/FIT/FDD/simulation/include/FDDSimulation/DigitizationParameters.h b/Detectors/FIT/FDD/simulation/include/FDDSimulation/DigitizationParameters.h
index 2a0d25839014a..827079b1a97c8 100644
--- a/Detectors/FIT/FDD/simulation/include/FDDSimulation/DigitizationParameters.h
+++ b/Detectors/FIT/FDD/simulation/include/FDDSimulation/DigitizationParameters.h
@@ -16,17 +16,17 @@ namespace o2::fdd
 {
 struct DigitizationParameters {
   static constexpr float LightYield = 0.01;
-  static constexpr float PmGain = 1e6;
   static constexpr float ShapeAlpha = -0.445;
   static constexpr float ShapeN = 2.65;
   static constexpr float ShapeSigma = 3.25;
-  //static constexpr float Pedestal = 0;
+  // static constexpr float Pedestal = 0;
   static constexpr float TimeShiftCFD = 1.42;
   static constexpr float TimeDelayFDA = 30.0;
   static constexpr float TimeDelayFDC = 30.0;
-  static constexpr int PheRRSize = 1e5;     // size of random ring to be used inside photoelectron loop
-  static constexpr int HitRRSize = 1e4;     // size of random ring to be used inside hit loop
-  static constexpr int NResponseTables = 9; // number of PMT response tables
+  static constexpr int PheRRSize = 1e5;           // size of random ring to be used inside photoelectron loop
+  static constexpr int HitRRSize = 1e4;           // size of random ring to be used inside hit loop
+  static constexpr int NResponseTables = 9;       // number of PMT response tables
+  static constexpr uint8_t defaultFEEbits = 0x48; // (01001000) only 2 flags are set by default in simulation:kIsCFDinADCgate and kIsEventInTVDC
 };
 } // namespace o2::fdd
 #endif
diff --git a/Detectors/FIT/FDD/simulation/include/FDDSimulation/Digitizer.h b/Detectors/FIT/FDD/simulation/include/FDDSimulation/Digitizer.h
index c7bd444d5ce63..943babc0a23d0 100644
--- a/Detectors/FIT/FDD/simulation/include/FDDSimulation/Digitizer.h
+++ b/Detectors/FIT/FDD/simulation/include/FDDSimulation/Digitizer.h
@@ -16,6 +16,7 @@
 #include "DataFormatsFDD/ChannelData.h"
 #include "DataFormatsFDD/Digit.h"
 #include "DataFormatsFDD/MCLabel.h"
+#include "DataFormatsFIT/DeadChannelMap.h"
 #include "FDDSimulation/Detector.h"
 #include "SimulationDataFormat/MCTruthContainer.h"
 #include "FDDSimulation/DigitizationParameters.h"
@@ -87,6 +88,8 @@ class Digitizer
   void init();
   void finish();
 
+  void setDeadChannelMap(o2::fit::DeadChannelMap const* deadChannelMap) { mDeadChannelMap = deadChannelMap; };
+
  private:
   static constexpr int BCCacheMin = -1, BCCacheMax = 10, NBC2Cache = 1 + BCCacheMax - BCCacheMin;
 
@@ -125,6 +128,8 @@ class Digitizer
   static Double_t PMResponse(Double_t* x, Double_t*);
   static Double_t SinglePhESpectrum(Double_t* x, Double_t* par);
 
+  o2::fit::DeadChannelMap const* mDeadChannelMap = nullptr;
+
   ClassDefNV(Digitizer, 4);
 };
 } // namespace fdd
diff --git a/Detectors/FIT/FDD/simulation/include/FDDSimulation/Digits2Raw.h b/Detectors/FIT/FDD/simulation/include/FDDSimulation/Digits2Raw.h
deleted file mode 100644
index 10cec83c8afba..0000000000000
--- a/Detectors/FIT/FDD/simulation/include/FDDSimulation/Digits2Raw.h
+++ /dev/null
@@ -1,74 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-/// \file Digits2Raw.h
-/// \brief converts digits to raw format
-/// \author Maciej.Slupecki@cern.ch
-// based on FV0
-
-#ifndef ALICEO2_FDD_DIGITS2RAW_H_
-#define ALICEO2_FDD_DIGITS2RAW_H_
-
-#include "Headers/RAWDataHeader.h"
-#include "CommonDataFormat/InteractionRecord.h"
-#include "DataFormatsFDD/RawEventData.h"
-#include "DataFormatsFDD/LookUpTable.h"
-#include "DataFormatsFDD/ChannelData.h"
-#include "DataFormatsFDD/Digit.h"
-#include "DetectorsRaw/HBFUtils.h"
-#include "DetectorsRaw/RawFileWriter.h"
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-
-namespace o2
-{
-namespace fdd
-{
-class Digits2Raw
-{
- public:
-  Digits2Raw() = default;
-  void readDigits(const std::string& outDir, const std::string& fileDigitsName);
-  void convertDigits(o2::fdd::Digit bcdigits,
-                     gsl::span pmchannels,
-                     const o2::fdd::LookUpTable& lut);
-
-  o2::raw::RawFileWriter& getWriter() { return mWriter; }
-  void setFilePerLink(bool v) { mOutputPerLink = v; }
-  bool getFilePerLink() const { return mOutputPerLink; }
-
-  int carryOverMethod(const header::RDHAny* rdh, const gsl::span data,
-                      const char* ptr, int maxSize, int splitID,
-                      std::vector& trailer, std::vector& header) const;
-
- private:
-  static constexpr uint32_t sTcmLink = 2;
-  static constexpr uint16_t sCruId = 0;
-  static constexpr uint32_t sEndPointId = sCruId;
-
-  void makeGBTHeader(EventHeader& eventHeader, int link, o2::InteractionRecord const& mIntRecord);
-  void fillSecondHalfWordAndAddData(int iChannelPerLink, int prevPmLink, const o2::InteractionRecord& ir);
-  RawEventData mRawEventData;
-  o2::fdd::Triggers mTriggers;
-  o2::raw::RawFileWriter mWriter{"FDD"};
-  bool mOutputPerLink = false;
-  /////////////////////////////////////////////////
-
-  ClassDefNV(Digits2Raw, 1);
-};
-
-} // namespace fdd
-} // namespace o2
-#endif
diff --git a/Detectors/FIT/FDD/simulation/include/FDDSimulation/FDDDigParam.h b/Detectors/FIT/FDD/simulation/include/FDDSimulation/FDDDigParam.h
new file mode 100644
index 0000000000000..36f85bff24d6f
--- /dev/null
+++ b/Detectors/FIT/FDD/simulation/include/FDDSimulation/FDDDigParam.h
@@ -0,0 +1,34 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file  FDDDigParam.h
+/// \brief Configurable digitization parameters
+///
+/// \author Andreas Molander 
+
+#ifndef ALICEO2_FDD_DIG_PARAM
+#define ALICEO2_FDD_DIG_PARAM
+
+#include "CommonUtils/ConfigurableParamHelper.h"
+
+namespace o2::fdd
+{
+struct FDDDigParam : o2::conf::ConfigurableParamHelper {
+  float hitTimeOffsetA = 0; ///< Hit time offset on the A side [ns]
+  float hitTimeOffsetC = 0; ///< Hit time offset on the C side [ns]
+
+  float pmGain = 1e6; ///< PM gain
+
+  O2ParamDef(FDDDigParam, "FDDDigParam");
+};
+} // namespace o2::fdd
+
+#endif // ALICEO2_FDD_DIG_PARAM
diff --git a/Detectors/FIT/FDD/simulation/src/Detector.cxx b/Detectors/FIT/FDD/simulation/src/Detector.cxx
index fb64b4e67d9af..3bb945ddd1bba 100644
--- a/Detectors/FIT/FDD/simulation/src/Detector.cxx
+++ b/Detectors/FIT/FDD/simulation/src/Detector.cxx
@@ -20,7 +20,7 @@
 
 #include "FDDSimulation/Detector.h"
 
-#include "SimulationDataFormat/Stack.h"
+#include "DetectorsBase/Stack.h"
 #include "Field/MagneticField.h"
 
 #include "TVirtualMC.h"
@@ -29,7 +29,7 @@
 #include "TGeoManager.h"
 #include "TRandom.h"
 
-#include "FairLogger.h"
+#include 
 #include "FairRootManager.h"
 #include "FairVolume.h"
 #include "FairRootManager.h"
diff --git a/Detectors/FIT/FDD/simulation/src/Digitizer.cxx b/Detectors/FIT/FDD/simulation/src/Digitizer.cxx
index 7b402da039e62..daa3ae33f3c23 100644
--- a/Detectors/FIT/FDD/simulation/src/Digitizer.cxx
+++ b/Detectors/FIT/FDD/simulation/src/Digitizer.cxx
@@ -10,8 +10,10 @@
 // or submit itself to any jurisdiction.
 
 #include "FDDSimulation/Digitizer.h"
+
+#include "CommonDataFormat/InteractionRecord.h"
+#include "FDDSimulation/FDDDigParam.h"
 #include "SimulationDataFormat/MCTruthContainer.h"
-#include 
 
 #include "TMath.h"
 #include "TRandom.h"
@@ -49,6 +51,13 @@ void Digitizer::process(const std::vector& hits,
   // LOG(info) << "Pulse";
   // Conversion of hits to the analogue pulse shape
   for (auto& hit : sorted_hits) {
+    int iChannel = hit.GetDetectorID();
+
+    // If the dead channel map is used, and the channel with ID 'hit_ch' is dead, don't process this hit.
+    if (mDeadChannelMap && !mDeadChannelMap->isChannelAlive(iChannel)) {
+      continue;
+    }
+
     if (hit.GetTime() > 20e3) {
       const int maxWarn = 10;
       static int warnNo = 0;
@@ -60,13 +69,16 @@ void Digitizer::process(const std::vector& hits,
     }
 
     std::array cachedIR;
-    int iChannel = hit.GetDetectorID();
     int nPhotoElectrons = simulateLightYield(iChannel, hit.GetNphot());
 
     double delayScintillator = mRndScintDelay.getNextValue();
     double timeHit = delayScintillator + hit.GetTime();
 
-    timeHit -= getTOFCorrection(int(iChannel / 4)); // account for TOF to detector
+    // Subtract time-of-flight from hit time
+    const float timeOfFlight = hit.GetPos().R() / o2::constants::physics::LightSpeedCm2NS;
+    const float timeOffset = iChannel < 8 ? FDDDigParam::Instance().hitTimeOffsetC : FDDDigParam::Instance().hitTimeOffsetA;
+
+    timeHit += -timeOfFlight + timeOffset;
     timeHit += mIntRecord.getTimeNS();
     o2::InteractionRecord irHit(timeHit); // BC in which the hit appears (might be different from interaction BC for slow particles)
 
@@ -100,7 +112,7 @@ void Digitizer::createPulse(int nPhE, int parID, double timeHit, std::array& labels)
 {
   // LOG(info) << "Storing BC " << bc;
-
   int first = digitsCh.size(), nStored = 0;
+  float totalChargeA = 0, totalChargeC = 0;
+  int n_hit_A = 0, n_hit_C = 0, total_time_A = 0, total_time_C = 0;
+  bool nChCside = 8;
   for (int ic = 0; ic < Nchannels; ic++) {
     float chargeADC = integrateCharge(bc.pulse[ic]);
+    int cfdTime = int(simulateTimeCFD(bc.pulse[ic]));
+
     if (chargeADC != 0) {
-      digitsCh.emplace_back(ic, int(simulateTimeCFD(bc.pulse[ic])), int(chargeADC), std::rand() % (1 << 8));
+      if (ic < nChCside) {
+        totalChargeC += chargeADC;
+        total_time_C += cfdTime;
+        n_hit_C++;
+      } else {
+        totalChargeA += chargeADC;
+        total_time_A += cfdTime;
+        n_hit_A++;
+      }
+      uint8_t channelBits = parameters.defaultFEEbits;
+      if (std::rand() % 2) {
+        ChannelData::setFlag(ChannelData::kNumberADC, channelBits);
+      }
+      digitsCh.emplace_back(ic, cfdTime, int(chargeADC), channelBits);
       nStored++;
     }
   }
-  // bc.print();
-
+  // SET TRIGGERS
+  Bool_t is_A, is_C, isVertex, is_Central, is_SemiCentral = 0;
+  is_A = n_hit_A > 0;
+  is_C = n_hit_C > 0;
+  uint32_t amplA = is_A ? totalChargeA * 0.125 : -5000; // sum amplitude A side / 8 (hardware)
+  uint32_t amplC = is_C ? totalChargeC * 0.125 : -5000; // sum amplitude C side / 8 (hardware)
+  int timeA = is_A ? total_time_A / n_hit_A : -5000;    // average time A side
+  int timeC = is_C ? total_time_C / n_hit_C : -5000;    // average time C side
+  isVertex = is_A && is_C;
+
+  bool isLaser = false;
+  bool isOutputsAreBlocked = false;
+  bool isDataValid = true;
+  mTriggers.setTriggers(is_A, is_C, isVertex, is_Central, is_SemiCentral, int8_t(n_hit_A), int8_t(n_hit_C),
+                        amplA, amplC, timeA, timeC, isLaser, isOutputsAreBlocked, isDataValid);
   if (nStored != 0) {
     int nBC = digitsBC.size();
     digitsBC.emplace_back(first, nStored, bc, mTriggers);
-    digitsTrig.emplace_back(bc, 0, 0, 0, 0, 0);
+    digitsTrig.emplace_back(bc, is_A, is_C, isVertex, is_Central, is_SemiCentral);
 
     for (const auto& lbl : bc.labels) {
       labels.addElement(nBC, lbl);
@@ -390,3 +432,5 @@ void Digitizer::BCCache::print() const
     printf("\n");
   }
 }
+
+O2ParamImpl(FDDDigParam);
\ No newline at end of file
diff --git a/Detectors/FIT/FDD/simulation/src/Digits2Raw.cxx b/Detectors/FIT/FDD/simulation/src/Digits2Raw.cxx
deleted file mode 100644
index 77657df311705..0000000000000
--- a/Detectors/FIT/FDD/simulation/src/Digits2Raw.cxx
+++ /dev/null
@@ -1,213 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-//
-//  RAW data format - DataFormat/Detectors/FIT/FDD/RawEventData
-
-#include "FDDBase/Constants.h"
-#include "FDDSimulation/Digits2Raw.h"
-#include 
-#include 
-
-using namespace o2::fdd;
-
-ClassImp(Digits2Raw);
-
-//_____________________________________________________________________________________
-void Digits2Raw::readDigits(const std::string& outDir, const std::string& fileDigitsName)
-{
-  LOG(info) << "==============FDD: Digits2Raw::convertDigits" << std::endl;
-  mWriter.setCarryOverCallBack(this);
-  LookUpTable lut(true);
-
-  std::string outd = outDir;
-  if (outd.back() != '/') {
-    outd += '/';
-  }
-
-  // Register PM links linearly
-  for (int iPmLink = 0; iPmLink < Nmodules; ++iPmLink) {
-    uint16_t feeId = uint16_t(iPmLink);
-    uint8_t linkId = uint8_t(iPmLink);
-    std::string outFileLink = mOutputPerLink ? (outd + "fdd_link" + std::to_string(iPmLink) + ".raw") : (outd + "fdd.raw");
-    LOG(info) << " Register PM link: " << iPmLink << " to file: " << outFileLink;
-    mWriter.registerLink(feeId, sCruId, linkId, sEndPointId, outFileLink);
-  }
-
-  // Register TCM link separately
-  std::string outFileLink = mOutputPerLink ? (outd + "fdd_link" + std::to_string(sTcmLink) + ".raw") : (outd + "fdd.raw");
-  LOG(info) << " Register TCM link: " << outFileLink;
-  mWriter.registerLink(uint16_t(sTcmLink), sCruId, sTcmLink, sEndPointId, outFileLink);
-
-  TFile* fdig = TFile::Open(fileDigitsName.data());
-  assert(fdig != nullptr);
-  LOG(info) << "Open digits file: " << fileDigitsName.data();
-
-  TTree* digTree = (TTree*)fdig->Get("o2sim");
-  std::vector digitsBC, *fddDigitPtr = &digitsBC;
-  std::vector digitsCh, *fddChDataPtr = &digitsCh;
-  digTree->SetBranchAddress("FDDDigit", &fddDigitPtr);
-  digTree->SetBranchAddress("FDDDigitCh", &fddChDataPtr);
-
-  for (int ient = 0; ient < digTree->GetEntries(); ient++) {
-    digTree->GetEntry(ient);
-    int nbc = digitsBC.size();
-    for (int ibc = 0; ibc < nbc; ibc++) {
-      auto& bcd = digitsBC[ibc];
-      auto channels = bcd.getBunchChannelData(digitsCh);
-
-      if (!channels.empty()) {
-        LOG(debug) << "o2::fdd::Digits2Raw::readDigits(): Start to convertDigits() at ibc = " << ibc << "  " << bcd.mIntRecord
-                   << " iCh0:" << bcd.ref.getFirstEntry() << "  nentries:" << bcd.ref.getEntries();
-        convertDigits(bcd, channels, lut);
-      }
-    }
-  }
-}
-//_____________________________________________________________________________________
-void Digits2Raw::convertDigits(o2::fdd::Digit bcdigits, gsl::span pmchannels,
-                               const o2::fdd::LookUpTable& lut)
-{
-  const o2::InteractionRecord intRecord = bcdigits.getIntRecord();
-  int prevPmLink = -1;
-  int iChannelPerLink = 0;
-  int nch = pmchannels.size();
-
-  std::stringstream ss;
-  ss << "  Number of channels: " << nch << "   (Ch, PMT, Q, T)\n";
-  for (int ich = 0; ich < nch; ich++) {
-    if (pmchannels[ich].mChargeADC != 0) {
-      ss << "          " << std::setw(2) << ich
-         << std::setw(3) << pmchannels[ich].mPMNumber
-         << std::setw(5) << pmchannels[ich].mChargeADC
-         << std::setw(7) << std::setprecision(3) << pmchannels[ich].mTime << "\n";
-    }
-  }
-  LOG(debug) << ss.str().substr(0, ss.str().size() - 1);
-
-  for (int ich = 0; ich < nch; ich++) {
-    int nLinkPm = lut.getLink(pmchannels[ich].mPMNumber);
-    if (nLinkPm != prevPmLink) {
-      if (prevPmLink >= 0) {
-        fillSecondHalfWordAndAddData(iChannelPerLink, prevPmLink, intRecord);
-      }
-      makeGBTHeader(mRawEventData.mEventHeader, nLinkPm, intRecord);
-      iChannelPerLink = 0;
-      prevPmLink = nLinkPm;
-    }
-    LOG(debug) << "    Store data for channel: " << ich << " PmLink = " << nLinkPm << "  ";
-    auto& newData = mRawEventData.mEventData[iChannelPerLink];
-    newData.charge = pmchannels[ich].mChargeADC;
-    newData.time = pmchannels[ich].mTime;
-
-    newData.numberADC = bool(pmchannels[ich].mFEEBits & ChannelData::kNumberADC);
-    newData.isDoubleEvent = bool(pmchannels[ich].mFEEBits & ChannelData::kIsDoubleEvent);
-    newData.isTimeInfoNOTvalid = bool(pmchannels[ich].mFEEBits & ChannelData::kIsTimeInfoNOTvalid);
-    newData.isCFDinADCgate = bool(pmchannels[ich].mFEEBits & ChannelData::kIsCFDinADCgate);
-    newData.isTimeInfoLate = bool(pmchannels[ich].mFEEBits & ChannelData::kIsTimeInfoLate);
-    newData.isAmpHigh = bool(pmchannels[ich].mFEEBits & ChannelData::kIsAmpHigh);
-    newData.isEventInTVDC = bool(pmchannels[ich].mFEEBits & ChannelData::kIsEventInTVDC);
-    newData.isTimeInfoLost = bool(pmchannels[ich].mFEEBits & ChannelData::kIsTimeInfoLost);
-
-    newData.channelID = lut.getModChannel(pmchannels[ich].mPMNumber);
-    iChannelPerLink++;
-    if (ich == nch - 1) {
-      fillSecondHalfWordAndAddData(iChannelPerLink, prevPmLink, intRecord);
-    }
-  }
-
-  // TCM
-  makeGBTHeader(mRawEventData.mEventHeader, sTcmLink, intRecord);
-  mRawEventData.mEventHeader.nGBTWords = 1;
-  auto& tcmdata = mRawEventData.mTCMdata;
-  mTriggers = bcdigits.mTriggers;
-
-  float ampA = mTriggers.getAmplA();
-  float ampC = mTriggers.getAmplC();
-  if (ampA > 131071) {
-    ampA = 131071; //2^17
-  }
-  if (ampC > 131071) {
-    ampC = 131071; //2^17
-  }
-  tcmdata.vertex = mTriggers.getVertex();
-  tcmdata.orA = mTriggers.getOrA();
-  tcmdata.orC = mTriggers.getOrC();
-  tcmdata.sCen = mTriggers.getSCen();
-  tcmdata.cen = mTriggers.getCen();
-  tcmdata.nChanA = mTriggers.getNChanA();
-  tcmdata.nChanC = mTriggers.getNChanC();
-  tcmdata.amplA = ampA;
-  tcmdata.amplC = ampC;
-  tcmdata.timeA = mTriggers.getTimeA();
-  tcmdata.timeC = mTriggers.getTimeC();
-  LOG(debug) << " TCM  triggers read "
-             << " time A " << mTriggers.getTimeA() << " time C " << mTriggers.getTimeC()
-             << " amp A " << ampA << " amp C " << ampC
-             << " N A " << int(mTriggers.getNChanA()) << " N C " << int(mTriggers.getNChanC())
-             << " trig "
-             << " ver " << mTriggers.getVertex() << " A " << mTriggers.getOrA() << " C " << mTriggers.getOrC();
-
-  LOG(debug) << "TCMdata"
-             << " time A " << tcmdata.timeA << " time C " << tcmdata.timeC
-             << " amp A " << tcmdata.amplA << " amp C " << tcmdata.amplC
-             << " N A " << int(tcmdata.nChanA) << " N C " << int(tcmdata.nChanC)
-             << " trig "
-             << " ver " << tcmdata.vertex << " A " << tcmdata.orA << " C " << tcmdata.orC
-             << " size " << sizeof(tcmdata);
-
-  auto data = mRawEventData.to_vector(kTRUE); //for tcm module
-  uint32_t linkId = uint32_t(sTcmLink);
-  uint64_t feeId = uint64_t(sTcmLink);
-  mWriter.addData(feeId, sCruId, linkId, sEndPointId, intRecord, data);
-
-  // fill mEventData[iChannelPerLink] with 0s to flag that this is a dummy data
-  uint nGBTWords = uint((iChannelPerLink + 1) / 2);
-  if ((iChannelPerLink % 2) == 1) {
-    mRawEventData.mEventData[iChannelPerLink] = {};
-  }
-  mRawEventData.mEventHeader.nGBTWords = nGBTWords;
-  //  LOG(debug) << " last link: " << prevPmLink;
-}
-//_____________________________________________________________________________________
-void Digits2Raw::makeGBTHeader(EventHeader& eventHeader, int link, o2::InteractionRecord const& mIntRecord)
-{
-  eventHeader.startDescriptor = 0xf;
-  eventHeader.reservedField1 = 0;
-  eventHeader.reservedField2 = 0;
-  eventHeader.reservedField3 = 0;
-  eventHeader.bc = mIntRecord.bc;
-  eventHeader.orbit = mIntRecord.orbit;
-  LOG(debug) << "  makeGBTHeader for link: " << link;
-}
-//_____________________________________________________________________________________
-void Digits2Raw::fillSecondHalfWordAndAddData(int iChannelPerLink, int prevPmLink, const o2::InteractionRecord& ir)
-{
-  uint nGBTWords = uint((iChannelPerLink + 1) / 2);
-  if ((iChannelPerLink % 2) == 1) {
-    mRawEventData.mEventData[iChannelPerLink] = {};
-    LOG(debug) << "    Fill up empty second half-word.";
-  }
-  mRawEventData.mEventHeader.nGBTWords = nGBTWords;
-  auto data = mRawEventData.to_vector(false);
-  uint32_t linkId = uint32_t(prevPmLink);
-  uint64_t feeId = uint64_t(prevPmLink);
-  mWriter.addData(feeId, sCruId, linkId, sEndPointId, ir, data);
-  LOG(debug) << "  Switch prevPmLink:  " << prevPmLink << ". Save data with nGBTWords="
-             << nGBTWords << " in header. Last channel: " << iChannelPerLink;
-}
-
-//_____________________________________________________________________________________
-int Digits2Raw::carryOverMethod(const header::RDHAny* rdh, const gsl::span data,
-                                const char* ptr, int maxSize, int splitID,
-                                std::vector& trailer, std::vector& header) const
-{
-  return 0; // do not split, always start new CRU page
-}
diff --git a/Detectors/FIT/FDD/simulation/src/FDDSimulationLinkDef.h b/Detectors/FIT/FDD/simulation/src/FDDSimulationLinkDef.h
index 44059b3c5d75e..30f111c5ad2bf 100644
--- a/Detectors/FIT/FDD/simulation/src/FDDSimulationLinkDef.h
+++ b/Detectors/FIT/FDD/simulation/src/FDDSimulationLinkDef.h
@@ -19,7 +19,7 @@
 #pragma link C++ class o2::base::DetImpl < o2::fdd::Detector> + ;
 #pragma link C++ class o2::fdd::Digitizer + ;
 #pragma link C++ class o2::fdd::DigitizationParameters + ;
+#pragma link C++ class o2::fdd::FDDDigParam + ;
 #pragma link C++ class o2::dataformats::MCTruthContainer < o2::fdd::MCLabel> + ;
-#pragma link C++ class o2::fdd::Digits2Raw + ;
 
 #endif
diff --git a/Detectors/FIT/FDD/simulation/src/digit2raw.cxx b/Detectors/FIT/FDD/simulation/src/digit2raw.cxx
index e7fea9bea241b..de24fc92a3976 100644
--- a/Detectors/FIT/FDD/simulation/src/digit2raw.cxx
+++ b/Detectors/FIT/FDD/simulation/src/digit2raw.cxx
@@ -10,29 +10,16 @@
 // or submit itself to any jurisdiction.
 
 /// \file digit2raw.cxx
-/// \author ruben.shahoyan@cern.ch
+/// \author ruben.shahoyan@cern.ch afurs@cern.ch
 
 #include 
-#include 
-#include 
-#include 
-#include "Framework/Logger.h"
 #include 
-#include 
-#include "CommonUtils/StringUtils.h"
-#include "CommonUtils/ConfigurableParam.h"
-#include "CommonUtils/NameConf.h"
-#include "DetectorsRaw/HBFUtils.h"
 #include "FDDRaw/RawWriterFDD.h"
-#include "DataFormatsParameters/GRPObject.h"
 
 /// MC->raw conversion for FDD
 
 namespace bpo = boost::program_options;
 
-void digit2raw(const std::string& inpName, const std::string& outDir, int verbosity, const std::string& fileFor, uint32_t rdhV, bool noEmptyHBF, const std::string& flpName,
-               const std::string& ccdbUrl, const std::string& lutPath, int superPageSizeInB = 1024 * 1024);
-
 int main(int argc, char** argv)
 {
   bpo::variables_map vm;
@@ -43,21 +30,10 @@ int main(int argc, char** argv)
   bpo::positional_options_description opt_pos;
 
   try {
-    auto add_option = opt_general.add_options();
-    add_option("help,h", "Print this help message");
-    add_option("verbosity,v", bpo::value()->default_value(0), "verbosity level");
-    //    add_option("input-file,i", bpo::value()->default_value(o2::base::NameConf::getDigitsFileName(o2::detectors::DetID::FDD)),"input FDD digits file"); // why not used?
-    add_option("input-file,i", bpo::value()->default_value("fdddigits.root"), "input FDD digits file");
-    add_option("flp-name", bpo::value()->default_value("alio2-cr1-flp201"), "single file per: all,flp,cru,link"); // temporary, beacause FIT deployed only on one node
-    add_option("file-for,f", bpo::value()->default_value("all"), "single file per: all,flp,cru,link");
-    add_option("output-dir,o", bpo::value()->default_value("./"), "output directory for raw data");
-    uint32_t defRDH = o2::raw::RDHUtils::getVersion();
-    add_option("rdh-version,r", bpo::value()->default_value(defRDH), "RDH version to use");
-    add_option("no-empty-hbf,e", bpo::value()->default_value(false)->implicit_value(true), "do not create empty HBF pages (except for HBF starting TF)");
-    add_option("hbfutils-config,u", bpo::value()->default_value(std::string(o2::base::NameConf::DIGITIZATIONCONFIGFILE)), "config file for HBFUtils (or none)");
-    add_option("configKeyValues", bpo::value()->default_value(""), "comma-separated configKeyValues");
-    add_option("ccdb-path", bpo::value()->default_value(""), "CCDB url which contains LookupTable");
-    add_option("lut-path", bpo::value()->default_value(""), "LookupTable path, e.g. FDD/LookupTable");
+    opt_general.add_options()("help,h", "Print this help message");
+    // config with FDD defaults
+    o2::fit::DigitToRawConfig::configureExecOptions(opt_general, "fdddigits.root", "alio2-cr1-flp201");
+    // common part
     opt_all.add(opt_general).add(opt_hidden);
     bpo::store(bpo::command_line_parser(argc, argv).options(opt_all).positional(opt_pos).run(), vm);
 
@@ -76,59 +52,12 @@ int main(int argc, char** argv)
     std::cerr << e.what() << ", application will now exit" << std::endl;
     exit(2);
   }
-
-  std::string confDig = vm["hbfutils-config"].as();
-  if (!confDig.empty() && confDig != "none") {
-    o2::conf::ConfigurableParam::updateFromFile(confDig, "HBFUtils");
+  const o2::fit::DigitToRawConfig cfg(vm);
+  if (!cfg.mEnablePadding) {
+    o2::fit::DigitToRawDevice::digit2raw(cfg);
+  } else {
+    o2::fit::DigitToRawDevice::digit2raw(cfg);
   }
-  o2::conf::ConfigurableParam::updateFromString(vm["configKeyValues"].as());
-  digit2raw(vm["input-file"].as(),
-            vm["output-dir"].as(),
-            vm["verbosity"].as(),
-            vm["file-for"].as(),
-            vm["rdh-version"].as(),
-            vm["no-empty-hbf"].as(),
-            vm["flp-name"].as(),
-            vm["ccdb-path"].as(),
-            vm["lut-path"].as());
-
   o2::raw::HBFUtils::Instance().print();
-
   return 0;
 }
-
-void digit2raw(const std::string& inpName, const std::string& outDir, int verbosity, const std::string& fileFor, uint32_t rdhV, bool noEmptyHBF, const std::string& flpName, const std::string& ccdbUrl, const std::string& lutPath, int superPageSizeInB)
-{
-  TStopwatch swTot;
-  swTot.Start();
-  o2::fdd::RawWriterFDD m2r;
-  m2r.setFileFor(fileFor);
-  m2r.setFlpName(flpName);
-  m2r.setVerbosity(verbosity);
-  if (ccdbUrl != "") {
-    m2r.setCCDBurl(ccdbUrl);
-  }
-  if (lutPath != "") {
-    m2r.setLUTpath(lutPath);
-  }
-  auto& wr = m2r.getWriter();
-  std::string inputGRP = o2::base::NameConf::getGRPFileName();
-  const auto grp = o2::parameters::GRPObject::loadFrom(inputGRP);
-  wr.setContinuousReadout(grp->isDetContinuousReadOut(o2::detectors::DetID::FDD)); // must be set explicitly
-  wr.setSuperPageSize(superPageSizeInB);
-  wr.useRDHVersion(rdhV);
-  wr.setDontFillEmptyHBF(noEmptyHBF);
-
-  o2::raw::assertOutputDirectory(outDir);
-
-  std::string outDirName(outDir);
-  if (outDirName.back() != '/') {
-    outDirName += '/';
-  }
-
-  m2r.convertDigitsToRaw(outDirName, inpName);
-  wr.writeConfFile(wr.getOrigin().str, "RAWDATA", o2::utils::Str::concat_string(outDirName, wr.getOrigin().str, "raw.cfg"));
-  //
-  swTot.Stop();
-  swTot.Print();
-}
diff --git a/Detectors/FIT/FDD/workflow/CMakeLists.txt b/Detectors/FIT/FDD/workflow/CMakeLists.txt
index c3e47a0d58536..a4bcc6f0de6fb 100644
--- a/Detectors/FIT/FDD/workflow/CMakeLists.txt
+++ b/Detectors/FIT/FDD/workflow/CMakeLists.txt
@@ -52,6 +52,31 @@ o2_add_executable(flp-dpl-workflow
                   PUBLIC_LINK_LIBRARIES O2::FDDWorkflow O2::FDDRaw O2::FITWorkflow
                   TARGETVARNAME fddflpexe)
 
+o2_add_executable(recpoints-reader-workflow
+                  SOURCES src/recpoints-reader-workflow.cxx
+                  COMPONENT_NAME fdd
+                  PUBLIC_LINK_LIBRARIES O2::FDDWorkflow)
+
+o2_add_executable(recpoints-writer-workflow
+                  SOURCES src/recpoints-writer-workflow.cxx
+                  COMPONENT_NAME fdd
+                  PUBLIC_LINK_LIBRARIES O2::FDDWorkflow)
+
+o2_add_executable(integrate-cluster-workflow
+                  SOURCES src/cluster-integrator.cxx
+                  COMPONENT_NAME fdd
+                  PUBLIC_LINK_LIBRARIES O2::FDDWorkflow O2::FITWorkflow)
+
+o2_add_executable(integrate-cluster-reader-workflow
+                  SOURCES src/cluster-integrator-reader.cxx
+                  COMPONENT_NAME fdd
+                  PUBLIC_LINK_LIBRARIES O2::FDDWorkflow O2::FITWorkflow)
+
+o2_add_executable(merge-integrate-cluster-workflow
+                  SOURCES src/cluster-merge-integrator.cxx
+                  COMPONENT_NAME fdd
+                  PUBLIC_LINK_LIBRARIES O2::FDDWorkflow O2::FITWorkflow)
+
 if(NOT APPLE)
 
  set_property(TARGET ${fddrecoexe} PROPERTY LINK_WHAT_YOU_USE ON)
diff --git a/Detectors/FIT/FDD/workflow/include/FDDWorkflow/EntropyDecoderSpec.h b/Detectors/FIT/FDD/workflow/include/FDDWorkflow/EntropyDecoderSpec.h
index a6ee132ee0c34..1fd3cd7835cd9 100644
--- a/Detectors/FIT/FDD/workflow/include/FDDWorkflow/EntropyDecoderSpec.h
+++ b/Detectors/FIT/FDD/workflow/include/FDDWorkflow/EntropyDecoderSpec.h
@@ -28,7 +28,7 @@ namespace fdd
 class EntropyDecoderSpec : public o2::framework::Task
 {
  public:
-  EntropyDecoderSpec(int verbosity);
+  EntropyDecoderSpec(int verbosity, const std::string& ctfdictOpt = "none");
   ~EntropyDecoderSpec() override = default;
   void run(o2::framework::ProcessingContext& pc) final;
   void init(o2::framework::InitContext& ic) final;
@@ -41,7 +41,7 @@ class EntropyDecoderSpec : public o2::framework::Task
 };
 
 /// create a processor spec
-framework::DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec);
+framework::DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec, const std::string& ctfdictOpt);
 
 } // namespace fdd
 } // namespace o2
diff --git a/Detectors/FIT/FDD/workflow/include/FDDWorkflow/EntropyEncoderSpec.h b/Detectors/FIT/FDD/workflow/include/FDDWorkflow/EntropyEncoderSpec.h
index 4c64beaa46dc1..37d43f477e836 100644
--- a/Detectors/FIT/FDD/workflow/include/FDDWorkflow/EntropyEncoderSpec.h
+++ b/Detectors/FIT/FDD/workflow/include/FDDWorkflow/EntropyEncoderSpec.h
@@ -28,7 +28,7 @@ namespace fdd
 class EntropyEncoderSpec : public o2::framework::Task
 {
  public:
-  EntropyEncoderSpec();
+  EntropyEncoderSpec(bool selIR, const std::string& ctfdictOpt = "none");
   ~EntropyEncoderSpec() override = default;
   void run(o2::framework::ProcessingContext& pc) final;
   void init(o2::framework::InitContext& ic) final;
@@ -37,11 +37,12 @@ class EntropyEncoderSpec : public o2::framework::Task
 
  private:
   o2::fdd::CTFCoder mCTFCoder;
+  bool mSelIR = false;
   TStopwatch mTimer;
 };
 
 /// create a processor spec
-framework::DataProcessorSpec getEntropyEncoderSpec();
+framework::DataProcessorSpec getEntropyEncoderSpec(bool selIR = false, const std::string& ctfdictOpt = "none");
 
 } // namespace fdd
 } // namespace o2
diff --git a/Detectors/FIT/FDD/workflow/include/FDDWorkflow/RawDataProcessSpec.h b/Detectors/FIT/FDD/workflow/include/FDDWorkflow/RawDataProcessSpec.h
deleted file mode 100644
index 6ed465b6181dd..0000000000000
--- a/Detectors/FIT/FDD/workflow/include/FDDWorkflow/RawDataProcessSpec.h
+++ /dev/null
@@ -1,61 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-/// @file   RawDataProcessSpec.h
-
-#ifndef O2_FDD_RAWDATAPROCESSSPEC_H
-#define O2_FDD_RAWDATAPROCESSSPEC_H
-
-#include "Framework/CallbackService.h"
-#include "Framework/ConfigParamRegistry.h"
-#include "Framework/ControlService.h"
-#include "Framework/Lifetime.h"
-#include "Framework/Output.h"
-#include "Framework/WorkflowSpec.h"
-#include "Framework/SerializationMethods.h"
-#include "Framework/DataProcessorSpec.h"
-#include "Framework/Task.h"
-
-#include "FDDRaw/DigitBlockFDD.h"
-#include "DataFormatsFDD/Digit.h"
-#include "DataFormatsFDD/ChannelData.h"
-
-#include 
-#include 
-#include 
-
-using namespace o2::framework;
-
-namespace o2
-{
-namespace fdd
-{
-
-class RawDataProcessSpec : public Task
-{
- public:
-  RawDataProcessSpec(bool dumpEventBlocks) : mDumpEventBlocks(dumpEventBlocks) {}
-  ~RawDataProcessSpec() override = default;
-  void init(InitContext& ic) final;
-  void run(ProcessingContext& pc) final;
-
- private:
-  bool mDumpEventBlocks;
-
-  o2::header::DataOrigin mOrigin = o2::header::gDataOriginFDD;
-};
-
-framework::DataProcessorSpec getFDDRawDataProcessSpec(bool dumpProcessor);
-
-} // namespace fdd
-} // namespace o2
-
-#endif /* O2_FDDDATAPROCESSDPL_H */
diff --git a/Detectors/FIT/FDD/workflow/include/FDDWorkflow/RawDataReaderSpec.h b/Detectors/FIT/FDD/workflow/include/FDDWorkflow/RawDataReaderSpec.h
deleted file mode 100644
index f3bff1d32f67a..0000000000000
--- a/Detectors/FIT/FDD/workflow/include/FDDWorkflow/RawDataReaderSpec.h
+++ /dev/null
@@ -1,83 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-/// @file   RawDataReaderSpec.h
-
-#ifndef O2_FDD_RAWDATAREADERSPEC_H
-#define O2_FDD_RAWDATAREADERSPEC_H
-
-#include "DataFormatsFDD/LookUpTable.h"
-#include "Framework/DataProcessorSpec.h"
-#include "Framework/Task.h"
-#include "Framework/CallbackService.h"
-#include "Framework/ConfigParamRegistry.h"
-#include "Framework/ControlService.h"
-#include "Framework/Lifetime.h"
-#include "Framework/Output.h"
-#include "Framework/WorkflowSpec.h"
-#include "Framework/SerializationMethods.h"
-#include "DPLUtils/DPLRawParser.h"
-
-#include 
-#include 
-#include 
-using namespace o2::framework;
-
-namespace o2
-{
-namespace fdd
-{
-template 
-class RawDataReaderSpec : public Task
-{
- public:
-  RawDataReaderSpec(const RawReader& rawReader) : mRawReader(rawReader) {}
-  RawDataReaderSpec() = default;
-  ~RawDataReaderSpec() override = default;
-  void init(InitContext& ic) final { o2::fdd::SingleLUT::Instance().printFullMap(); }
-  void run(ProcessingContext& pc) final
-  {
-    DPLRawParser parser(pc.inputs());
-    mRawReader.clear();
-    LOG(info) << "FDD RawDataReaderSpec";
-    uint64_t count = 0;
-    for (auto it = parser.begin(), end = parser.end(); it != end; ++it) {
-      //Proccessing each page
-      count++;
-      auto rdhPtr = it.get_if();
-      gsl::span payload(it.data(), it.size());
-      mRawReader.process(payload, rdhPtr->linkID, int(0));
-    }
-    LOG(info) << "Pages: " << count;
-    mRawReader.accumulateDigits();
-    mRawReader.makeSnapshot(pc);
-  }
-  RawReader mRawReader;
-};
-
-template 
-framework::DataProcessorSpec getFDDRawDataReaderSpec(const RawReader& rawReader)
-{
-  LOG(info) << "DataProcessorSpec initDataProcSpec() for RawReaderFDD";
-  std::vector outputSpec;
-  RawReader::prepareOutputSpec(outputSpec);
-  return DataProcessorSpec{
-    "fdd-datareader-dpl",
-    o2::framework::select("TF:FDD/RAWDATA"),
-    outputSpec,
-    adaptFromTask>(rawReader),
-    Options{}};
-}
-
-} // namespace fdd
-} // namespace o2
-
-#endif /* O2_FDDDATAREADERDPL_H */
diff --git a/Detectors/FIT/FDD/workflow/include/FDDWorkflow/RawReaderFDD.h b/Detectors/FIT/FDD/workflow/include/FDDWorkflow/RawReaderFDD.h
index e9612ae8c8bda..2d92c907c1d79 100644
--- a/Detectors/FIT/FDD/workflow/include/FDDWorkflow/RawReaderFDD.h
+++ b/Detectors/FIT/FDD/workflow/include/FDDWorkflow/RawReaderFDD.h
@@ -65,8 +65,8 @@ class RawReaderFDD : public RawReaderFDDBaseNorm
   }
   void makeSnapshot(o2::framework::ProcessingContext& pc)
   {
-    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFDD, "DIGITSBC", 0, o2::framework::Lifetime::Timeframe}, mVecDigits);
-    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFDD, "DIGITSCH", 0, o2::framework::Lifetime::Timeframe}, mVecChannelData);
+    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFDD, "DIGITSBC", 0}, mVecDigits);
+    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFDD, "DIGITSCH", 0}, mVecChannelData);
   }
   bool mDumpData;
   std::vector mVecDigits;
diff --git a/Detectors/FIT/FDD/workflow/include/FDDWorkflow/RawWorkflow.h b/Detectors/FIT/FDD/workflow/include/FDDWorkflow/RawWorkflow.h
deleted file mode 100644
index 3bbab66d16497..0000000000000
--- a/Detectors/FIT/FDD/workflow/include/FDDWorkflow/RawWorkflow.h
+++ /dev/null
@@ -1,28 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-#ifndef O2_FDD_RAWWORKFLOW_H
-#define O2_FDD_RAWWORKFLOW_H
-
-/// @file   RawWorkflow.h
-
-#include "Framework/WorkflowSpec.h"
-
-namespace o2
-{
-namespace fdd
-{
-framework::WorkflowSpec getFDDRawWorkflow(bool useProcess,
-                                          bool dumpProcessor, bool dumpReader,
-                                          bool disableRootOut);
-} // namespace fdd
-} // namespace o2
-#endif
diff --git a/Detectors/FIT/FDD/workflow/include/FDDWorkflow/RecPointReaderSpec.h b/Detectors/FIT/FDD/workflow/include/FDDWorkflow/RecPointReaderSpec.h
index 500883d5badfa..6c3c9694f3e1c 100644
--- a/Detectors/FIT/FDD/workflow/include/FDDWorkflow/RecPointReaderSpec.h
+++ b/Detectors/FIT/FDD/workflow/include/FDDWorkflow/RecPointReaderSpec.h
@@ -31,7 +31,7 @@ namespace fdd
 class RecPointReader : public Task
 {
  public:
-  RecPointReader(bool useMC = true);
+  RecPointReader(bool useMC = false);
   ~RecPointReader() override = default;
   void init(InitContext& ic) final;
   void run(ProcessingContext& pc) final;
@@ -42,7 +42,7 @@ class RecPointReader : public Task
   std::unique_ptr mFile;
   std::unique_ptr mTree;
 
-  bool mUseMC = true; // use MC truth
+  bool mUseMC = false; // use MC truth
   o2::header::DataOrigin mOrigin = o2::header::gDataOriginFDD;
 
   std::vector* mRecPoints = nullptr;
diff --git a/Detectors/FIT/FDD/workflow/include/FDDWorkflow/RecoWorkflow.h b/Detectors/FIT/FDD/workflow/include/FDDWorkflow/RecoWorkflow.h
index 2dbd854e34eee..0d5d308216bb0 100644
--- a/Detectors/FIT/FDD/workflow/include/FDDWorkflow/RecoWorkflow.h
+++ b/Detectors/FIT/FDD/workflow/include/FDDWorkflow/RecoWorkflow.h
@@ -20,7 +20,7 @@ namespace o2
 {
 namespace fdd
 {
-framework::WorkflowSpec getRecoWorkflow(bool useMC, bool disableRootInp, bool disableRootOut);
+framework::WorkflowSpec getRecoWorkflow(bool useMC, bool disableRootInp, bool disableRootOut, bool useDeadChannelMap);
 } // namespace fdd
 } // namespace o2
 #endif
diff --git a/Detectors/FIT/FDD/workflow/include/FDDWorkflow/ReconstructorSpec.h b/Detectors/FIT/FDD/workflow/include/FDDWorkflow/ReconstructorSpec.h
index 7dcb5d9aaba40..8f20ff1513ab4 100644
--- a/Detectors/FIT/FDD/workflow/include/FDDWorkflow/ReconstructorSpec.h
+++ b/Detectors/FIT/FDD/workflow/include/FDDWorkflow/ReconstructorSpec.h
@@ -18,6 +18,8 @@
 #include "Framework/Task.h"
 #include "FDDReconstruction/Reconstructor.h"
 #include "DataFormatsFDD/RecPoint.h"
+#include "DataFormatsFIT/DeadChannelMap.h"
+#include "Framework/ConcreteDataMatcher.h"
 
 using namespace o2::framework;
 
@@ -29,21 +31,25 @@ namespace fdd
 class FDDReconstructorDPL : public Task
 {
  public:
-  FDDReconstructorDPL(bool useMC) : mUseMC(useMC) {}
+  FDDReconstructorDPL(bool useMC, bool useDeadChannelMap) : mUseMC(useMC), mUseDeadChannelMap(useDeadChannelMap) {}
   ~FDDReconstructorDPL() override = default;
   void init(InitContext& ic) final;
   void run(ProcessingContext& pc) final;
+  void finaliseCCDB(ConcreteDataMatcher& matcher, void* obj) final;
 
  private:
   bool mUseMC = true;
+  bool mUseDeadChannelMap = true;
+  bool mUpdateDeadChannelMap = true;
   std::vector mRecPoints;
   std::vector mRecChData;
+  o2::fit::DeadChannelMap const* mDeadChannelMap;
   o2::fdd::Reconstructor mReco;
   o2::header::DataOrigin mOrigin = o2::header::gDataOriginFDD;
 };
 
 /// create a processor spec
-framework::DataProcessorSpec getFDDReconstructorSpec(bool useMC = true);
+framework::DataProcessorSpec getFDDReconstructorSpec(bool useMC = true, bool useDeadChannelMap = true);
 
 } // namespace fdd
 } // namespace o2
diff --git a/Detectors/FIT/FDD/workflow/src/DigitReaderSpec.cxx b/Detectors/FIT/FDD/workflow/src/DigitReaderSpec.cxx
index b14d891d44a00..628a2160c6d0c 100644
--- a/Detectors/FIT/FDD/workflow/src/DigitReaderSpec.cxx
+++ b/Detectors/FIT/FDD/workflow/src/DigitReaderSpec.cxx
@@ -42,7 +42,7 @@ void DigitReader::init(InitContext& ic)
   mInputFileName = o2::utils::Str::concat_string(o2::utils::Str::rectifyDirectory(ic.options().get("input-dir")),
                                                  ic.options().get("fdd-digits-infile"));
 
-  mFile = std::make_unique(mInputFileName.c_str(), "OLD");
+  mFile.reset(TFile::Open(mInputFileName.c_str()));
   if (!mFile->IsOpen()) {
     LOG(error) << "Cannot open the " << mInputFileName.c_str() << " file !";
     throw std::runtime_error("cannot open input digits file");
@@ -81,18 +81,18 @@ void DigitReader::run(ProcessingContext& pc)
   mTree->GetEntry(ent);
 
   LOG(info) << "FDD DigitReader pushes " << digitsBC->size() << " digits";
-  pc.outputs().snapshot(Output{mOrigin, "DIGITSBC", 0, Lifetime::Timeframe}, *digitsBC);
-  pc.outputs().snapshot(Output{mOrigin, "DIGITSCH", 0, Lifetime::Timeframe}, *digitsCh);
+  pc.outputs().snapshot(Output{mOrigin, "DIGITSBC", 0}, *digitsBC);
+  pc.outputs().snapshot(Output{mOrigin, "DIGITSCH", 0}, *digitsCh);
 
   if (mUseMC) {
     // TODO: To be replaced with sending ConstMCTruthContainer as soon as reco workflow supports it
-    pc.outputs().snapshot(Output{mOrigin, "TRIGGERINPUT", 0, Lifetime::Timeframe}, *digitsTrig);
+    pc.outputs().snapshot(Output{mOrigin, "TRIGGERINPUT", 0}, *digitsTrig);
 
     std::vector flatbuffer;
     mcTruthRootBuffer->copyandflatten(flatbuffer);
     o2::dataformats::MCTruthContainer mcTruth;
     mcTruth.restore_from(flatbuffer.data(), flatbuffer.size());
-    pc.outputs().snapshot(Output{mOrigin, "DIGITLBL", 0, Lifetime::Timeframe}, mcTruth);
+    pc.outputs().snapshot(Output{mOrigin, "DIGITLBL", 0}, mcTruth);
   }
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get().endOfStream();
diff --git a/Detectors/FIT/FDD/workflow/src/EntropyDecoderSpec.cxx b/Detectors/FIT/FDD/workflow/src/EntropyDecoderSpec.cxx
index d9e2761b61dbc..33c140b5bc198 100644
--- a/Detectors/FIT/FDD/workflow/src/EntropyDecoderSpec.cxx
+++ b/Detectors/FIT/FDD/workflow/src/EntropyDecoderSpec.cxx
@@ -24,12 +24,12 @@ namespace o2
 {
 namespace fdd
 {
-
-EntropyDecoderSpec::EntropyDecoderSpec(int verbosity) : mCTFCoder(o2::ctf::CTFCoderBase::OpType::Decoder)
+EntropyDecoderSpec::EntropyDecoderSpec(int verbosity, const std::string& ctfdictOpt) : mCTFCoder(o2::ctf::CTFCoderBase::OpType::Decoder, ctfdictOpt)
 {
   mTimer.Stop();
   mTimer.Reset();
   mCTFCoder.setVerbosity(verbosity);
+  mCTFCoder.setDictBinding("ctfdict_FDD");
 }
 
 void EntropyDecoderSpec::finaliseCCDB(o2::framework::ConcreteDataMatcher& matcher, void* obj)
@@ -50,8 +50,8 @@ void EntropyDecoderSpec::run(ProcessingContext& pc)
   mTimer.Start(false);
   o2::ctf::CTFIOSize iosize;
 
-  mCTFCoder.updateTimeDependentParams(pc);
-  auto buff = pc.inputs().get>("ctf");
+  mCTFCoder.updateTimeDependentParams(pc, true);
+  auto buff = pc.inputs().get>("ctf_FDD");
 
   auto& digits = pc.outputs().make>(OutputRef{"digits"});
   auto& channels = pc.outputs().make>(OutputRef{"channels"});
@@ -72,7 +72,7 @@ void EntropyDecoderSpec::endOfStream(EndOfStreamContext& ec)
        mTimer.CpuTime(), mTimer.RealTime(), mTimer.Counter() - 1);
 }
 
-DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec)
+DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec, const std::string& ctfdictOpt)
 {
   std::vector outputs{
     OutputSpec{{"digits"}, "FDD", "DIGITSBC", 0, Lifetime::Timeframe},
@@ -80,16 +80,19 @@ DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec)
     OutputSpec{{"ctfrep"}, "FDD", "CTFDECREP", 0, Lifetime::Timeframe}};
 
   std::vector inputs;
-  inputs.emplace_back("ctf", "FDD", "CTFDATA", sspec, Lifetime::Timeframe);
-  inputs.emplace_back("ctfdict", "FDD", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("FDD/Calib/CTFDictionary"));
+  inputs.emplace_back("ctf_FDD", "FDD", "CTFDATA", sspec, Lifetime::Timeframe);
+
+  if (ctfdictOpt.empty() || ctfdictOpt == "ccdb") {
+    inputs.emplace_back("ctfdict_FDD", "FDD", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("FDD/Calib/CTFDictionaryTree"));
+  }
+  inputs.emplace_back("trigoffset", "CTP", "Trig_Offset", 0, Lifetime::Condition, ccdbParamSpec("CTP/Config/TriggerOffsets"));
 
   return DataProcessorSpec{
     "fdd-entropy-decoder",
     inputs,
     outputs,
-    AlgorithmSpec{adaptFromTask(verbosity)},
-    Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}}};
+    AlgorithmSpec{adaptFromTask(verbosity, ctfdictOpt)},
+    Options{{"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}};
 }
-
 } // namespace fdd
 } // namespace o2
diff --git a/Detectors/FIT/FDD/workflow/src/EntropyEncoderSpec.cxx b/Detectors/FIT/FDD/workflow/src/EntropyEncoderSpec.cxx
index 811a45dc93c25..be81f7ca7d3d4 100644
--- a/Detectors/FIT/FDD/workflow/src/EntropyEncoderSpec.cxx
+++ b/Detectors/FIT/FDD/workflow/src/EntropyEncoderSpec.cxx
@@ -25,8 +25,7 @@ namespace o2
 {
 namespace fdd
 {
-
-EntropyEncoderSpec::EntropyEncoderSpec() : mCTFCoder(o2::ctf::CTFCoderBase::OpType::Encoder)
+EntropyEncoderSpec::EntropyEncoderSpec(bool selIR, const std::string& ctfdictOpt) : mCTFCoder(o2::ctf::CTFCoderBase::OpType::Encoder, ctfdictOpt), mSelIR(selIR)
 {
   mTimer.Stop();
   mTimer.Reset();
@@ -48,12 +47,17 @@ void EntropyEncoderSpec::run(ProcessingContext& pc)
 {
   auto cput = mTimer.CpuTime();
   mTimer.Start(false);
-  mCTFCoder.updateTimeDependentParams(pc);
+  mCTFCoder.updateTimeDependentParams(pc, true);
   auto digits = pc.inputs().get>("digits");
   auto channels = pc.inputs().get>("channels");
-
-  auto& buffer = pc.outputs().make>(Output{"FDD", "CTFDATA", 0, Lifetime::Timeframe});
+  if (mSelIR) {
+    mCTFCoder.setSelectedIRFrames(pc.inputs().get>("selIRFrames"));
+  }
+  auto& buffer = pc.outputs().make>(Output{"FDD", "CTFDATA", 0});
   auto iosize = mCTFCoder.encode(buffer, digits, channels);
+  if (mSelIR) {
+    mCTFCoder.getIRFramesSelector().clear();
+  }
   mTimer.Stop();
   LOG(info) << iosize.asString() << " in " << mTimer.CpuTime() - cput << " s";
 }
@@ -64,21 +68,27 @@ void EntropyEncoderSpec::endOfStream(EndOfStreamContext& ec)
        mTimer.CpuTime(), mTimer.RealTime(), mTimer.Counter() - 1);
 }
 
-DataProcessorSpec getEntropyEncoderSpec()
+DataProcessorSpec getEntropyEncoderSpec(bool selIR, const std::string& ctfdictOpt)
 {
   std::vector inputs;
   inputs.emplace_back("digits", "FDD", "DIGITSBC", 0, Lifetime::Timeframe);
   inputs.emplace_back("channels", "FDD", "DIGITSCH", 0, Lifetime::Timeframe);
-  inputs.emplace_back("ctfdict", "FDD", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("FDD/Calib/CTFDictionary"));
 
+  if (ctfdictOpt.empty() || ctfdictOpt == "ccdb") {
+    inputs.emplace_back("ctfdict", "FDD", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("FDD/Calib/CTFDictionaryTree"));
+  }
+  if (selIR) {
+    inputs.emplace_back("selIRFrames", "CTF", "SELIRFRAMES", 0, Lifetime::Timeframe);
+  }
   return DataProcessorSpec{
     "fdd-entropy-encoder",
     inputs,
     Outputs{{"FDD", "CTFDATA", 0, Lifetime::Timeframe}},
-    AlgorithmSpec{adaptFromTask()},
-    Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}},
-            {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}}}};
+    AlgorithmSpec{adaptFromTask(selIR, ctfdictOpt)},
+    Options{{"irframe-margin-bwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame lower boundary when selection is requested"}},
+            {"irframe-margin-fwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame upper boundary when selection is requested"}},
+            {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}},
+            {"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}};
 }
-
 } // namespace fdd
 } // namespace o2
diff --git a/Detectors/FIT/FDD/workflow/src/RawDataProcessSpec.cxx b/Detectors/FIT/FDD/workflow/src/RawDataProcessSpec.cxx
deleted file mode 100644
index bf18db67672c2..0000000000000
--- a/Detectors/FIT/FDD/workflow/src/RawDataProcessSpec.cxx
+++ /dev/null
@@ -1,52 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-/// @file   RawDataProcessSpec.cxx
-
-#include "FDDWorkflow/RawDataProcessSpec.h"
-
-using namespace o2::framework;
-
-namespace o2
-{
-namespace fdd
-{
-using namespace std;
-void RawDataProcessSpec::init(InitContext& ic)
-{
-}
-
-void RawDataProcessSpec::run(ProcessingContext& pc)
-{
-  LOG(info) << "RawDataProcessSpec running...";
-  auto vecDigits = pc.inputs().get>("digits");
-  auto vecChannelData = pc.inputs().get>("digch");
-  if (mDumpEventBlocks) {
-    DigitBlockFDD::print(vecDigits, vecChannelData);
-  }
-}
-
-DataProcessorSpec getFDDRawDataProcessSpec(bool dumpProcessor)
-{
-  std::vector inputSpec;
-  inputSpec.emplace_back("digits", o2::header::gDataOriginFDD, "DIGITSBC", 0, Lifetime::Timeframe);
-  inputSpec.emplace_back("digch", o2::header::gDataOriginFDD, "DIGITSCH", 0, Lifetime::Timeframe);
-  LOG(info) << "DataProcessorSpec getRawDataProcessSpec";
-  return DataProcessorSpec{
-    "fdd-dataprocess-dpl-flp",
-    inputSpec,
-    Outputs{},
-    AlgorithmSpec{adaptFromTask(dumpProcessor)},
-    Options{}};
-}
-
-} // namespace fdd
-} // namespace o2
diff --git a/Detectors/FIT/FDD/workflow/src/RawDataReaderSpec.cxx b/Detectors/FIT/FDD/workflow/src/RawDataReaderSpec.cxx
deleted file mode 100644
index 631655d3038ec..0000000000000
--- a/Detectors/FIT/FDD/workflow/src/RawDataReaderSpec.cxx
+++ /dev/null
@@ -1,24 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-/// @file   RawDataReaderSpec.cxx
-
-#include "FDDWorkflow/RawDataReaderSpec.h"
-
-using namespace o2::framework;
-
-namespace o2
-{
-namespace fdd
-{
-
-} // namespace fdd
-} // namespace o2
diff --git a/Detectors/FIT/FDD/workflow/src/RawWorkflow.cxx b/Detectors/FIT/FDD/workflow/src/RawWorkflow.cxx
deleted file mode 100644
index c9e5c5be0c81d..0000000000000
--- a/Detectors/FIT/FDD/workflow/src/RawWorkflow.cxx
+++ /dev/null
@@ -1,42 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-/// @file   RawWorkflow.cxx
-
-#include "FDDWorkflow/RawWorkflow.h"
-#include "FDDWorkflow/RawDataProcessSpec.h"
-#include "FDDWorkflow/RawDataReaderSpec.h"
-#include "FDDWorkflow/DigitWriterSpec.h"
-#include "FDDWorkflow/RawReaderFDD.h"
-namespace o2
-{
-namespace fdd
-{
-
-framework::WorkflowSpec getFDDRawWorkflow(bool useProcess,
-                                          bool dumpProcessor, bool dumpReader,
-                                          bool disableRootOut)
-{
-  LOG(info) << "framework::WorkflowSpec getFDDWorkflow";
-  framework::WorkflowSpec specs;
-  specs.emplace_back(o2::fdd::getFDDRawDataReaderSpec(RawReaderFDD{dumpReader}));
-
-  if (useProcess) {
-    specs.emplace_back(o2::fdd::getFDDRawDataProcessSpec(dumpProcessor));
-  }
-  if (!disableRootOut) {
-    specs.emplace_back(o2::fdd::getFDDDigitWriterSpec(false, false));
-  }
-  return specs;
-}
-
-} // namespace fdd
-} // namespace o2
diff --git a/Detectors/FIT/FDD/workflow/src/RecPointReaderSpec.cxx b/Detectors/FIT/FDD/workflow/src/RecPointReaderSpec.cxx
index 3da258a672611..3c4812c75b251 100644
--- a/Detectors/FIT/FDD/workflow/src/RecPointReaderSpec.cxx
+++ b/Detectors/FIT/FDD/workflow/src/RecPointReaderSpec.cxx
@@ -11,16 +11,14 @@
 
 /// @file  RecPointReaderSpec.cxx
 
-#include 
-
-#include "TTree.h"
-
-#include "Framework/ControlService.h"
 #include "Framework/ConfigParamRegistry.h"
+#include "Framework/ControlService.h"
 #include "Framework/Logger.h"
 #include "FDDWorkflow/RecPointReaderSpec.h"
 #include "CommonUtils/NameConf.h"
 
+#include 
+
 using namespace o2::framework;
 using namespace o2::fdd;
 
@@ -51,8 +49,8 @@ void RecPointReader::run(ProcessingContext& pc)
   mTree->GetEntry(ent);
 
   LOG(info) << "FDD RecPointReader pushes " << mRecPoints->size() << " recpoints with " << mChannelData->size() << " channels at entry " << ent;
-  pc.outputs().snapshot(Output{mOrigin, "RECPOINTS", 0, Lifetime::Timeframe}, *mRecPoints);
-  pc.outputs().snapshot(Output{mOrigin, "RECCHDATA", 0, Lifetime::Timeframe}, *mChannelData);
+  pc.outputs().snapshot(Output{mOrigin, "RECPOINTS", 0}, *mRecPoints);
+  pc.outputs().snapshot(Output{mOrigin, "RECCHDATA", 0}, *mChannelData);
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get().endOfStream();
diff --git a/Detectors/FIT/FDD/workflow/src/RecoWorkflow.cxx b/Detectors/FIT/FDD/workflow/src/RecoWorkflow.cxx
index a7d4c15af81bb..b464e689f7a75 100644
--- a/Detectors/FIT/FDD/workflow/src/RecoWorkflow.cxx
+++ b/Detectors/FIT/FDD/workflow/src/RecoWorkflow.cxx
@@ -22,14 +22,14 @@ namespace o2
 namespace fdd
 {
 
-framework::WorkflowSpec getRecoWorkflow(bool useMC, bool disableRootInp, bool disableRootOut)
+framework::WorkflowSpec getRecoWorkflow(bool useMC, bool disableRootInp, bool disableRootOut, bool useDeadChannelMap)
 {
   framework::WorkflowSpec specs;
 
   if (!disableRootInp) {
     specs.emplace_back(o2::fdd::getFDDDigitReaderSpec(useMC));
   }
-  specs.emplace_back(o2::fdd::getFDDReconstructorSpec(useMC));
+  specs.emplace_back(o2::fdd::getFDDReconstructorSpec(useMC, useDeadChannelMap));
   if (!disableRootOut) {
     specs.emplace_back(o2::fdd::getFDDRecPointWriterSpec(useMC));
   }
diff --git a/Detectors/FIT/FDD/workflow/src/ReconstructorSpec.cxx b/Detectors/FIT/FDD/workflow/src/ReconstructorSpec.cxx
index 566a5f2bf9bab..1d5d599b5ee31 100644
--- a/Detectors/FIT/FDD/workflow/src/ReconstructorSpec.cxx
+++ b/Detectors/FIT/FDD/workflow/src/ReconstructorSpec.cxx
@@ -18,6 +18,7 @@
 #include "FDDWorkflow/ReconstructorSpec.h"
 #include "DataFormatsFDD/Digit.h"
 #include "DataFormatsFDD/MCLabel.h"
+#include "Framework/CCDBParamSpec.h"
 
 using namespace o2::framework;
 
@@ -37,41 +38,55 @@ void FDDReconstructorDPL::run(ProcessingContext& pc)
   auto digitsBC = pc.inputs().get>("digitsBC");
   auto digitsCh = pc.inputs().get>("digitsCh");
   // RS: if we need to process MC truth, uncomment lines below
-  //std::unique_ptr> labels;
-  //const o2::dataformats::MCTruthContainer* lblPtr = nullptr;
+  // std::unique_ptr> labels;
+  // const o2::dataformats::MCTruthContainer* lblPtr = nullptr;
   if (mUseMC) {
-    //labels = pc.inputs().get*>("labels");
-    //lblPtr = labels.get();
+    // labels = pc.inputs().get*>("labels");
+    // lblPtr = labels.get();
     LOG(info) << "Ignoring MC info";
   }
+  if (mUseDeadChannelMap && mUpdateDeadChannelMap) {
+    LOG(info) << "Populating reconsturctor object with Dead Channel Map object";
+    auto deadChannelMap = pc.inputs().get("deadChannelMap");
+    mReco.setDeadChannelMap(deadChannelMap.get());
+  }
   int nDig = digitsBC.size();
   mRecPoints.reserve(nDig);
-  mRecChData.resize(digitsCh.size());
+  mRecChData.reserve(digitsCh.size());
   for (int id = 0; id < nDig; id++) {
     const auto& digit = digitsBC[id];
     auto channels = digit.getBunchChannelData(digitsCh);
-    gsl::span out_ch(mRecChData);
-    out_ch = out_ch.subspan(digit.ref.getFirstEntry(), digit.ref.getEntries());
-    mRecPoints.emplace_back(mReco.process(digit, channels, out_ch));
+    mReco.process(digit, channels, mRecPoints, mRecChData);
   }
-
   // do we ignore MC in this task?
+  LOG(debug) << "FDD reconstruction pushes " << mRecPoints.size() << " RecPoints";
+  pc.outputs().snapshot(Output{mOrigin, "RECPOINTS", 0}, mRecPoints);
+  pc.outputs().snapshot(Output{mOrigin, "RECCHDATA", 0}, mRecChData);
+}
 
-  LOG(info) << "FDD reconstruction pushes " << mRecPoints.size() << " RecPoints";
-  pc.outputs().snapshot(Output{mOrigin, "RECPOINTS", 0, Lifetime::Timeframe}, mRecPoints);
-  pc.outputs().snapshot(Output{mOrigin, "RECCHDATA", 0, Lifetime::Timeframe}, mRecChData);
+void FDDReconstructorDPL::finaliseCCDB(ConcreteDataMatcher& matcher, void* obj)
+{
+  if (matcher == ConcreteDataMatcher("FDD", "DeadChannelMap", 0)) {
+    mUpdateDeadChannelMap = false;
+    return;
+  }
 }
 
-DataProcessorSpec getFDDReconstructorSpec(bool useMC)
+DataProcessorSpec getFDDReconstructorSpec(bool useMC, bool useDeadChannelMap)
 {
   std::vector inputSpec;
   std::vector outputSpec;
   inputSpec.emplace_back("digitsBC", o2::header::gDataOriginFDD, "DIGITSBC", 0, Lifetime::Timeframe);
   inputSpec.emplace_back("digitsCh", o2::header::gDataOriginFDD, "DIGITSCH", 0, Lifetime::Timeframe);
+
   if (useMC) {
     LOG(info) << "Currently FDDReconstructor does not consume and provide MC truth";
     // inputSpec.emplace_back("labels", o2::header::gDataOriginFDD, "DIGITSMCTR", 0, Lifetime::Timeframe);
   }
+  if (useDeadChannelMap) {
+    LOG(info) << "Dead channel map will be applied during reconstruction";
+    inputSpec.emplace_back("deadChannelMap", o2::header::gDataOriginFDD, "DeadChannelMap", 0, Lifetime::Condition, ccdbParamSpec("FDD/Calib/DeadChannelMap"));
+  }
   outputSpec.emplace_back(o2::header::gDataOriginFDD, "RECPOINTS", 0, Lifetime::Timeframe);
   outputSpec.emplace_back(o2::header::gDataOriginFDD, "RECCHDATA", 0, Lifetime::Timeframe);
 
@@ -79,7 +94,7 @@ DataProcessorSpec getFDDReconstructorSpec(bool useMC)
     "fdd-reconstructor",
     inputSpec,
     outputSpec,
-    AlgorithmSpec{adaptFromTask(useMC)},
+    AlgorithmSpec{adaptFromTask(useMC, useDeadChannelMap)},
     Options{}};
 }
 
diff --git a/Detectors/FIT/FDD/workflow/src/cluster-integrator-reader.cxx b/Detectors/FIT/FDD/workflow/src/cluster-integrator-reader.cxx
new file mode 100644
index 0000000000000..55c22c162d732
--- /dev/null
+++ b/Detectors/FIT/FDD/workflow/src/cluster-integrator-reader.cxx
@@ -0,0 +1,23 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "FITWorkflow/FITIntegrateClusterReaderSpec.h"
+#include "DataFormatsFDD/RecPoint.h"
+#include "Framework/runDataProcessing.h"
+
+using namespace o2::framework;
+
+WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
+{
+  WorkflowSpec wf;
+  wf.emplace_back(o2::fit::getFITIntegrateClusterReaderSpec());
+  return wf;
+}
diff --git a/Detectors/FIT/FDD/workflow/src/cluster-integrator.cxx b/Detectors/FIT/FDD/workflow/src/cluster-integrator.cxx
new file mode 100644
index 0000000000000..d28b962aa94d8
--- /dev/null
+++ b/Detectors/FIT/FDD/workflow/src/cluster-integrator.cxx
@@ -0,0 +1,44 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "FITWorkflow/FITIntegrateClusterSpec.h"
+#include "DataFormatsFDD/RecPoint.h"
+#include "FITWorkflow/FITIntegrateClusterWriterSpec.h"
+#include "CommonUtils/ConfigurableParam.h"
+#include "Framework/ConfigParamSpec.h"
+
+using namespace o2::framework;
+
+void customize(std::vector& workflowOptions)
+{
+  std::vector options{
+    ConfigParamSpec{"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}},
+    {"min-NChan", VariantType::Int, 0, {"Minimum NChan signal required to avoid noise"}},
+    {"min-Ampl", VariantType::Int, 0, {"Minimum Ampl signal required to avoid noise"}},
+    {"disable-root-output", VariantType::Bool, false, {"disable root-files output writers"}}};
+  std::swap(workflowOptions, options);
+}
+
+#include "Framework/runDataProcessing.h"
+
+WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
+{
+  WorkflowSpec wf;
+  o2::conf::ConfigurableParam::updateFromString(cfgc.options().get("configKeyValues"));
+  const bool disableWriter = cfgc.options().get("disable-root-output");
+  const int minNChan = cfgc.options().get("min-NChan");
+  const int minAmpl = cfgc.options().get("min-Ampl");
+  wf.emplace_back(o2::fit::getFITIntegrateClusterSpec(disableWriter, minNChan, minAmpl));
+  if (!disableWriter) {
+    wf.emplace_back(o2::fit::getFITIntegrateClusterWriterSpec());
+  }
+  return wf;
+}
diff --git a/Detectors/FIT/FDD/workflow/src/cluster-merge-integrator.cxx b/Detectors/FIT/FDD/workflow/src/cluster-merge-integrator.cxx
new file mode 100644
index 0000000000000..c6951cf3639ea
--- /dev/null
+++ b/Detectors/FIT/FDD/workflow/src/cluster-merge-integrator.cxx
@@ -0,0 +1,35 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "FITWorkflow/FITMergeIntegrateClusterSpec.h"
+#include "DataFormatsFDD/RecPoint.h"
+#include "CommonUtils/ConfigurableParam.h"
+#include "Framework/ConfigParamSpec.h"
+
+using namespace o2::framework;
+
+void customize(std::vector& workflowOptions)
+{
+  std::vector options{
+    ConfigParamSpec{"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}},
+  };
+  std::swap(workflowOptions, options);
+}
+
+#include "Framework/runDataProcessing.h"
+
+WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
+{
+  WorkflowSpec wf;
+  o2::conf::ConfigurableParam::updateFromString(cfgc.options().get("configKeyValues"));
+  wf.emplace_back(o2::fit::getFITMergeIntegrateClusterSpec());
+  return wf;
+}
diff --git a/Detectors/FIT/FDD/workflow/src/entropy-encoder-workflow.cxx b/Detectors/FIT/FDD/workflow/src/entropy-encoder-workflow.cxx
index 2adcfe0a1c422..b83e25557e760 100644
--- a/Detectors/FIT/FDD/workflow/src/entropy-encoder-workflow.cxx
+++ b/Detectors/FIT/FDD/workflow/src/entropy-encoder-workflow.cxx
@@ -21,7 +21,10 @@ using namespace o2::framework;
 void customize(std::vector& workflowOptions)
 {
   // option allowing to set parameters
-  std::vector options{ConfigParamSpec{"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}}};
+  std::vector options{
+    ConfigParamSpec{"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}},
+    ConfigParamSpec{"ctf-dict", VariantType::String, "none", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}},
+    ConfigParamSpec{"select-ir-frames", VariantType::Bool, false, {"Subscribe and filter according to external IR Frames"}}};
 
   std::swap(workflowOptions, options);
 }
@@ -35,6 +38,6 @@ WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
   WorkflowSpec wf;
   // Update the (declared) parameters if changed from the command line
   o2::conf::ConfigurableParam::updateFromString(cfgc.options().get("configKeyValues"));
-  wf.emplace_back(o2::fdd::getEntropyEncoderSpec());
+  wf.emplace_back(o2::fdd::getEntropyEncoderSpec(cfgc.options().get("select-ir-frames"), cfgc.options().get("ctf-dict")));
   return wf;
 }
diff --git a/Detectors/FIT/FDD/workflow/src/fdd-flp-workflow.cxx b/Detectors/FIT/FDD/workflow/src/fdd-flp-workflow.cxx
index a3fec9ee3da7c..178c5db7b2561 100644
--- a/Detectors/FIT/FDD/workflow/src/fdd-flp-workflow.cxx
+++ b/Detectors/FIT/FDD/workflow/src/fdd-flp-workflow.cxx
@@ -58,6 +58,16 @@ void customize(std::vector& workflowOptions)
                     o2::framework::VariantType::Bool,
                     false,
                     {"do not subscribe to FLP/DISTSUBTIMEFRAME/0 message (no lost TF recovery)"}});
+  workflowOptions.push_back(
+    ConfigParamSpec{"input-sub-sampled",
+                    o2::framework::VariantType::Bool,
+                    false,
+                    {"SUB_RAWDATA DPL channel will be used as input, in case of dispatcher usage"}});
+  workflowOptions.push_back(
+    ConfigParamSpec{"disable-dpl-ccdb-fetcher",
+                    o2::framework::VariantType::Bool,
+                    false,
+                    {"Disable DPL CCDB fetcher, channel map will be uploaded during initialization by taking last entry in CCDB"}});
 }
 
 // ------------------------------------------------------------------
@@ -71,24 +81,26 @@ WorkflowSpec defineDataProcessing(ConfigContext const& configcontext)
   auto isExtendedMode = configcontext.options().get("tcm-extended-mode");
   auto disableRootOut = configcontext.options().get("disable-root-output");
   auto askSTFDist = !configcontext.options().get("ignore-dist-stf");
+  const auto isSubSampled = configcontext.options().get("input-sub-sampled");
+  const auto disableDplCcdbFetcher = configcontext.options().get("disable-dpl-ccdb-fetcher");
   o2::conf::ConfigurableParam::updateFromString(configcontext.options().get("configKeyValues"));
   LOG(info) << "WorkflowSpec FLPWorkflow";
-  //Type aliases
-  //using RawReaderFDDtrgInput = o2::fit::RawReaderFIT;
+  // Type aliases
+  // using RawReaderFDDtrgInput = o2::fit::RawReaderFIT;
   using RawReaderFDD = o2::fit::RawReaderFIT;
-  //using RawReaderFDDtrgInputExt = o2::fit::RawReaderFIT;
+  // using RawReaderFDDtrgInputExt = o2::fit::RawReaderFIT;
   using RawReaderFDDext = o2::fit::RawReaderFIT;
   using MCLabelCont = o2::dataformats::MCTruthContainer;
   o2::header::DataOrigin dataOrigin = o2::header::gDataOriginFDD;
   //
   WorkflowSpec specs;
   if (isExtendedMode) {
-    specs.emplace_back(o2::fit::getFITDataReaderDPLSpec(RawReaderFDDext{dataOrigin, dumpReader}, askSTFDist));
+    specs.emplace_back(o2::fit::getFITDataReaderDPLSpec(RawReaderFDDext{dataOrigin, dumpReader}, askSTFDist, isSubSampled, disableDplCcdbFetcher));
     if (!disableRootOut) {
       specs.emplace_back(o2::fit::FITDigitWriterSpecHelper::getFITDigitWriterSpec(false, false, dataOrigin));
     }
   } else {
-    specs.emplace_back(o2::fit::getFITDataReaderDPLSpec(RawReaderFDD{dataOrigin, dumpReader}, askSTFDist));
+    specs.emplace_back(o2::fit::getFITDataReaderDPLSpec(RawReaderFDD{dataOrigin, dumpReader}, askSTFDist, isSubSampled, disableDplCcdbFetcher));
     if (!disableRootOut) {
       specs.emplace_back(o2::fit::FITDigitWriterSpecHelper::getFITDigitWriterSpec(false, false, dataOrigin));
     }
diff --git a/Detectors/FIT/FDD/workflow/src/fdd-reco-workflow.cxx b/Detectors/FIT/FDD/workflow/src/fdd-reco-workflow.cxx
index 652ddb8bd2a29..888792425909b 100644
--- a/Detectors/FIT/FDD/workflow/src/fdd-reco-workflow.cxx
+++ b/Detectors/FIT/FDD/workflow/src/fdd-reco-workflow.cxx
@@ -38,6 +38,7 @@ void customize(std::vector& workflowOptions)
     {"disable-mc", o2::framework::VariantType::Bool, false, {"disable MC propagation even if available"}},
     {"disable-root-input", o2::framework::VariantType::Bool, false, {"disable root-files input readers"}},
     {"disable-root-output", o2::framework::VariantType::Bool, false, {"disable root-files output writers"}},
+    {"disable-dead-channel-map", o2::framework::VariantType::Bool, false, {"disable dead channel map"}},
     {"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}}};
   o2::raw::HBFUtilsInitializer::addConfigOption(options);
   std::swap(workflowOptions, options);
@@ -57,8 +58,9 @@ WorkflowSpec defineDataProcessing(ConfigContext const& configcontext)
   auto useMC = !configcontext.options().get("disable-mc");
   auto disableRootInp = configcontext.options().get("disable-root-input");
   auto disableRootOut = configcontext.options().get("disable-root-output");
+  bool useDeadChannelMap = !configcontext.options().get("disable-dead-channel-map");
 
-  auto wf = o2::fdd::getRecoWorkflow(useMC, disableRootInp, disableRootOut);
+  auto wf = o2::fdd::getRecoWorkflow(useMC, disableRootInp, disableRootOut, useDeadChannelMap);
 
   // configure dpl timer to inject correct firstTForbit: start from the 1st orbit of TF containing 1st sampled orbit
   o2::raw::HBFUtilsInitializer hbfIni(configcontext, wf);
diff --git a/Detectors/FIT/FDD/workflow/src/recpoints-reader-workflow.cxx b/Detectors/FIT/FDD/workflow/src/recpoints-reader-workflow.cxx
new file mode 100644
index 0000000000000..fcef4cc46901f
--- /dev/null
+++ b/Detectors/FIT/FDD/workflow/src/recpoints-reader-workflow.cxx
@@ -0,0 +1,57 @@
+// Copyright 2019-2024 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file  recpoints-reader-workflow.cxx
+/// \brief FDD RecPoints reader workflow
+///
+/// \author Andreas Molander andreas.molander@cern.ch
+
+#include "FDDWorkflow/RecPointReaderSpec.h"
+
+#include "CommonUtils/ConfigurableParam.h"
+#include "DetectorsRaw/HBFUtilsInitializer.h"
+#include "Framework/CallbacksPolicy.h"
+#include "Framework/ConfigParamSpec.h"
+#include "Framework/Variant.h"
+
+#include 
+
+using namespace o2::framework;
+
+void customize(std::vector& policies)
+{
+  o2::raw::HBFUtilsInitializer::addNewTimeSliceCallback(policies);
+}
+
+// we need to add workflow options before including Framework/runDataProcessing
+void customize(std::vector& workflowOptions)
+{
+  std::vector options{
+    {"disable-mc", VariantType::Bool, false, {"disable MC propagation even if available"}},
+    {"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}}};
+  o2::raw::HBFUtilsInitializer::addConfigOption(options);
+  std::swap(workflowOptions, options);
+}
+
+#include "Framework/runDataProcessing.h"
+
+WorkflowSpec defineDataProcessing(const ConfigContext& ctx)
+{
+  o2::conf::ConfigurableParam::updateFromString(ctx.options().get("configKeyValues"));
+  bool disableMC = ctx.options().get("disable-mc");
+  WorkflowSpec specs;
+  DataProcessorSpec producer = o2::fdd::getFDDRecPointReaderSpec(!disableMC);
+  specs.push_back(producer);
+
+  // configure dpl timer to inject correct firstTForbit: start from the 1st orbit of TF containing 1st sampled orbit
+  o2::raw::HBFUtilsInitializer hbfIni(ctx, specs);
+  return specs;
+}
diff --git a/Detectors/FIT/FDD/workflow/src/recpoints-writer-workflow.cxx b/Detectors/FIT/FDD/workflow/src/recpoints-writer-workflow.cxx
new file mode 100644
index 0000000000000..e53ccd14c30ab
--- /dev/null
+++ b/Detectors/FIT/FDD/workflow/src/recpoints-writer-workflow.cxx
@@ -0,0 +1,47 @@
+// Copyright 2019-2024 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file  recpoints-writer-workflow.cxx
+/// \brief FDD RecPoints writer workflow
+///
+/// \author Andreas Molander andreas.molander@cern.ch
+
+#include "FDDWorkflow/RecPointWriterSpec.h"
+
+#include "CommonUtils/ConfigurableParam.h"
+#include "Framework/ConfigParamSpec.h"
+#include "Framework/Variant.h"
+
+#include 
+
+using namespace o2::framework;
+
+// we need to add workflow options before including Framework/runDataProcessing
+void customize(std::vector& workflowOptions)
+{
+  std::vector options{
+    {"disable-mc", VariantType::Bool, false, {"disable MC propagation even if available"}},
+    {"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}}};
+  workflowOptions.insert(workflowOptions.end(), options.begin(), options.end());
+}
+
+#include "Framework/runDataProcessing.h"
+
+WorkflowSpec defineDataProcessing(const ConfigContext& ctx)
+{
+  o2::conf::ConfigurableParam::updateFromString(ctx.options().get("configKeyValues"));
+  bool disableMC = ctx.options().get("disable-mc");
+
+  WorkflowSpec specs;
+  DataProcessorSpec producer = o2::fdd::getFDDRecPointWriterSpec(!disableMC);
+  specs.push_back(producer);
+  return specs;
+}
diff --git a/Detectors/FIT/FT0/CMakeLists.txt b/Detectors/FIT/FT0/CMakeLists.txt
index 08477d8d1ea5c..2fbb4dfbbc332 100644
--- a/Detectors/FIT/FT0/CMakeLists.txt
+++ b/Detectors/FIT/FT0/CMakeLists.txt
@@ -10,12 +10,13 @@
 # or submit itself to any jurisdiction.
 
 add_subdirectory(base)
+add_subdirectory(dcsmonitoring)
+if(BUILD_TESTING)
+    add_subdirectory(dcsmonitoring/macros)
+endif()
 add_subdirectory(raw)
 add_subdirectory(reconstruction)
 add_subdirectory(simulation)
 add_subdirectory(workflow)
-add_subdirectory(calibration)
 add_subdirectory(macros)
-if(BUILD_TESTING)
-    add_subdirectory(calibration/macros)
-endif()
\ No newline at end of file
+add_subdirectory(calibration)
\ No newline at end of file
diff --git a/Detectors/FIT/FT0/base/README.md b/Detectors/FIT/FT0/base/README.md
new file mode 100644
index 0000000000000..2ac1732118237
--- /dev/null
+++ b/Detectors/FIT/FT0/base/README.md
@@ -0,0 +1,18 @@
+
+
+# FT0 Base
+
+## Geometry
+
+The `o2::ft0::Geometry` class contains dimensions and information used for constructing the FT0 geometry as used in simulation. It also provides utility methods for retrieving the center locations of the detector cells. See the below example for how to use the `Geometry` class to query the FT0 cell locations. Note that these are the ideal locations, and that any misalignment is not considered.
+
+```cpp
+o2::ft0::Geometry ft0Geometry;
+ft0Geometry.calculateChannelCenter();
+TVector3 cellPosition = ft0Geometry.getChannelCenter(chId);
+float x = cellPosition.X();
+float y = cellPosition.Y();
+float z = cellPosition.Z();
+```
diff --git a/Detectors/FIT/FT0/base/include/FT0Base/FT0DigParam.h b/Detectors/FIT/FT0/base/include/FT0Base/FT0DigParam.h
index fb8b6492354f1..2bf774859aa22 100644
--- a/Detectors/FIT/FT0/base/include/FT0Base/FT0DigParam.h
+++ b/Detectors/FIT/FT0/base/include/FT0Base/FT0DigParam.h
@@ -9,7 +9,7 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 
-/// \file FT0DigParam.h
+/// \file  FT0DigParam.h
 /// \brief Configurable digitization parameters
 
 #ifndef ALICEO2_FT0_DIG_PARAM
@@ -29,25 +29,25 @@ struct FT0DigParam : o2::conf::ConfigurableParamHelper {
   float mAmp_trsh = 100;             // [ph.e]
   float mAmpRecordLow = -4;          // integrate charge from
   float mAmpRecordUp = 15;           // to [ns]
-  float mC_side_cable_cmps = 2.86;   // ns
-  float mA_side_cable_cmps = 11.110; // ns
+  float hitTimeOffsetA = 0;          ///< hit time offset on the A side [ns]
+  float hitTimeOffsetC = 0;          ///< hit time offset on the C side [ns]
   int mtrg_central_trh = 600.;       // channels
   int mtrg_semicentral_trh = 300.;   // channels
 
   float mMip_in_V = 7;       // MIP to mV
   float mPe_in_mip = 0.004;  // invserse Np.e. in MIP 1./250.
   float mCfdShift = 1.66;    // ns
-  float mCFDShiftPos = 1.47; //// shift positive part of CFD signal; distance between 0.3 of max amplitude  to max
+  float mCFDShiftPos = 1.47; // shift positive part of CFD signal; distance between 0.3 of max amplitude  to max
   float mCFDdeadTime = 15.6; // ns
   float mCharge2amp = 0.22;
-  float mNoiseVar = 0.1;                               // noise level
-  float mNoisePeriod = 1 / 0.9;                        // GHz low frequency noise period;
-  static constexpr short mTime_trg_gate = 153;         // #channels as in TCM as in Pilot beams ('OR gate' setting in TCM tab in ControlServer)
-  static constexpr float mAmpThresholdForReco = 5;     // only channels with amplitude higher will participate in calibration and collision time: 0.3 MIP
-  static constexpr short mTimeThresholdForReco = 1000; // only channels with time below will participate in calibration and collision time
-
-  static constexpr float mMV_2_Nchannels = 2.2857143;          // amplitude channel 7 mV ->16channels
-  static constexpr float mMV_2_NchannelsInverse = 0.437499997; // inverse amplitude channel 7 mV ->16channels
+  float mNoiseVar = 0.1;              // noise level
+  float mNoisePeriod = 1 / 0.9;       // GHz low frequency noise period;
+  short mTime_trg_gate = 153;         // #channels as in TCM as in Pilot beams ('OR gate' setting in TCM tab in ControlServer)
+  float mAmpThresholdForReco = 5;     // only channels with amplitude higher will participate in calibration and collision time: 0.3 MIP
+  short mTimeThresholdForReco = 1000; // only channels with time below will participate in calibration and collision time
+
+  float mMV_2_Nchannels = 2.2857143;          // amplitude channel 7 mV ->16channels
+  float mMV_2_NchannelsInverse = 0.437499997; // inverse amplitude channel 7 mV ->16channels
 
   O2ParamDef(FT0DigParam, "FT0DigParam");
 };
diff --git a/Detectors/FIT/FT0/base/include/FT0Base/Geometry.h b/Detectors/FIT/FT0/base/include/FT0Base/Geometry.h
index 0365e730828cb..83be3af4d5b71 100644
--- a/Detectors/FIT/FT0/base/include/FT0Base/Geometry.h
+++ b/Detectors/FIT/FT0/base/include/FT0Base/Geometry.h
@@ -65,11 +65,34 @@ class Geometry
     return o2::base::GeometryManager::getPNEntry(getDetID(), index);
   }
 
+  // NEW:
+  // Calculate the coordinates of all the channels.
+  void calculateChannelCenter();
+  // Get the coordinates of the center of the channel channelId.
+  TVector3 getChannelCenter(UInt_t channelId) const { return mChannelCenter[channelId]; }
+
  private:
   TVector3 mMCP[52];
   TVector3 mAngles[28];
+  std::array mChannelCenter; ///< NEW: Center of each channel in FT0 A (96) and C (112) combined.
+
+  // Convert the local ordering of the channels to the official one and apply it to the channel map.
+  // localChannelOrder[local channel index] = official channel.
+  Int_t localChannelOrder[Nchannels] = {
+    58, 56, 59, 57, 54, 52, 55, 53, 50, 49, 51, 48, 47, 45, 46, 44, 43, 42, 41, 40,
+    61, 60, 63, 62, 14, 12, 15, 13, 10, 9, 11, 8, 7, 6, 5, 4, 39, 38, 37, 36,
+    65, 64, 66, 67, 17, 16, 18, 19, 3, 2, 0, 1, 35, 34, 32, 33,
+    68, 69, 70, 71, 20, 21, 22, 23, 24, 27, 25, 26, 29, 31, 28, 30, 94, 95, 92, 93,
+    72, 73, 74, 75, 76, 78, 77, 79, 80, 83, 81, 82, 85, 87, 84, 86, 89, 91, 88, 90,
+    173, 172, 175, 174, 206, 207, 204, 205, 169, 168, 171, 170, 202, 203, 200, 201,
+    117, 116, 119, 118, 142, 143, 140, 141, 114, 112, 115, 113, 137, 139, 136, 138,
+    166, 164, 167, 165, 197, 199, 196, 198, 110, 108, 111, 109, 133, 135, 132, 134,
+    162, 160, 163, 161, 193, 195, 192, 194, 107, 105, 106, 104, 128, 130, 129, 131,
+    159, 157, 158, 156, 188, 190, 189, 191, 99, 98, 97, 96, 120, 121, 122, 123,
+    103, 102, 101, 100, 124, 125, 126, 127, 155, 153, 154, 152, 184, 186, 185, 187,
+    147, 146, 145, 144, 176, 177, 178, 179, 151, 150, 149, 148, 180, 181, 182, 183};
 
-  ClassDefNV(Geometry, 2);
+  ClassDefNV(Geometry, 3);
 };
 } // namespace ft0
 } // namespace o2
diff --git a/Detectors/FIT/FT0/base/src/Geometry.cxx b/Detectors/FIT/FT0/base/src/Geometry.cxx
index de9b3ddad4c21..4c5fa004f63bb 100644
--- a/Detectors/FIT/FT0/base/src/Geometry.cxx
+++ b/Detectors/FIT/FT0/base/src/Geometry.cxx
@@ -117,3 +117,61 @@ void Geometry::setCsideModules()
     mMCP[i + NCellsA].SetXYZ(xc2[i], yc2[i], zc2[i]);
   }
 }
+void Geometry::calculateChannelCenter()
+{
+  // This method calculates the position of each channel center composing both FT0-A
+  // and FT0-C, based on the position of their corresponding modules given by
+  // "Geometry::setAsideModules()" and "Geometry::setCsideModules()".
+
+  // Ensure the positions of the modules are well defined.
+  setAsideModules();
+  setCsideModules();
+
+  // Calculate first the positions for the channels for FT0-A. These correspond to the
+  // channels 0-95 in the modules mMCP[0-23].
+  Double_t delta = 5.3 / 4.;          // Half-channel width (TODO: ask if it actually corresponds to ChannelWidth)
+  Double_t xLocalChannels[Nchannels]; // x-positions of all channels ordered according to xi and internal numbering.
+  Double_t yLocalChannels[Nchannels]; // y-positions of all channels ordered according to yi and internal numbering.
+  Double_t zLocalChannels[Nchannels]; // z-positions of all channels ordered according to zi and internal numbering.
+                                      // INFO: We assume here the modules are perpendicular to z, so z(channel) = z(module).
+
+  for (int iModA = 0; iModA < NCellsA; iModA++) {
+    xLocalChannels[4 * iModA + 0] = mMCP[iModA].X() - delta;
+    xLocalChannels[4 * iModA + 1] = mMCP[iModA].X() + delta;
+    xLocalChannels[4 * iModA + 2] = mMCP[iModA].X() - delta;
+    xLocalChannels[4 * iModA + 3] = mMCP[iModA].X() + delta;
+
+    yLocalChannels[4 * iModA + 0] = mMCP[iModA].Y() + delta;
+    yLocalChannels[4 * iModA + 1] = mMCP[iModA].Y() + delta;
+    yLocalChannels[4 * iModA + 2] = mMCP[iModA].Y() - delta;
+    yLocalChannels[4 * iModA + 3] = mMCP[iModA].Y() - delta;
+
+    zLocalChannels[4 * iModA + 0] = mMCP[iModA].Z();
+    zLocalChannels[4 * iModA + 1] = mMCP[iModA].Z();
+    zLocalChannels[4 * iModA + 2] = mMCP[iModA].Z();
+    zLocalChannels[4 * iModA + 3] = mMCP[iModA].Z();
+  }
+
+  // Calculate then the positions for the channels for FT0-C, corresponding to the
+  // channels 96-207 in the modules mMCP[24-51].
+  for (int iModC = 0; iModC < NCellsC; iModC++) {
+    xLocalChannels[4 * (iModC + NCellsA) + 0] = mMCP[iModC + NCellsA].X() - delta;
+    xLocalChannels[4 * (iModC + NCellsA) + 1] = mMCP[iModC + NCellsA].X() + delta;
+    xLocalChannels[4 * (iModC + NCellsA) + 2] = mMCP[iModC + NCellsA].X() - delta;
+    xLocalChannels[4 * (iModC + NCellsA) + 3] = mMCP[iModC + NCellsA].X() + delta;
+
+    yLocalChannels[4 * (iModC + NCellsA) + 0] = mMCP[iModC + NCellsA].Y() + delta;
+    yLocalChannels[4 * (iModC + NCellsA) + 1] = mMCP[iModC + NCellsA].Y() + delta;
+    yLocalChannels[4 * (iModC + NCellsA) + 2] = mMCP[iModC + NCellsA].Y() - delta;
+    yLocalChannels[4 * (iModC + NCellsA) + 3] = mMCP[iModC + NCellsA].Y() - delta;
+
+    zLocalChannels[4 * (iModC + NCellsA) + 0] = mMCP[iModC + NCellsA].Z();
+    zLocalChannels[4 * (iModC + NCellsA) + 1] = mMCP[iModC + NCellsA].Z();
+    zLocalChannels[4 * (iModC + NCellsA) + 2] = mMCP[iModC + NCellsA].Z();
+    zLocalChannels[4 * (iModC + NCellsA) + 3] = mMCP[iModC + NCellsA].Z();
+  }
+
+  for (int iChannel = 0; iChannel < Nchannels; iChannel++) {
+    mChannelCenter[localChannelOrder[iChannel]].SetXYZ(xLocalChannels[iChannel], yLocalChannels[iChannel], zLocalChannels[iChannel]);
+  }
+}
diff --git a/Detectors/FIT/FT0/calibration/CMakeLists.txt b/Detectors/FIT/FT0/calibration/CMakeLists.txt
index a36df294e1643..bee0493d300c1 100644
--- a/Detectors/FIT/FT0/calibration/CMakeLists.txt
+++ b/Detectors/FIT/FT0/calibration/CMakeLists.txt
@@ -10,92 +10,50 @@
 # or submit itself to any jurisdiction.
 
 o2_add_library(FT0Calibration
-        SOURCES
-        src/CalibParam.cxx
-        src/FT0ChannelTimeOffsetSlotContainer.cxx
-        src/FT0TimeOffsetSlotContainer.cxx
-        src/GlobalOffsetsContainer.cxx
-        src/FT0CalibTimeSlewing.cxx
-        src/FT0CalibCollector.cxx
-        src/FT0DCSProcessor.cxx
-        PUBLIC_LINK_LIBRARIES
-         O2::CCDB
-         O2::MathUtils
-         O2::DataFormatsFT0
-         O2::CommonDataFormat
-         O2::DetectorsCalibration
-         O2::DetectorsDCS
-         O2::DataFormatsGlobalTracking
-         ROOT::Minuit
-         Microsoft.GSL::GSL
-           )
-       o2_target_root_dictionary(FT0Calibration
-        HEADERS
-        include/FT0Calibration/CalibParam.h
-        include/FT0Calibration/FT0ChannelTimeOffsetSlotContainer.h
-        include/FT0Calibration/FT0TimeOffsetSlotContainer.h
-        include/FT0Calibration/GlobalOffsetsContainer.h
-        include/FT0Calibration/FT0CalibTimeSlewing.h
-        include/FT0Calibration/FT0CalibCollector.h
-        include/FT0Calibration/FT0DCSProcessor.h
-        )
-      o2_add_executable(ft0-channel-offset-calibration
-        COMPONENT_NAME calibration
-        SOURCES testWorkflow/FT0ChannelTimeCalibration-Workflow.cxx
-        PUBLIC_LINK_LIBRARIES
-        O2::FT0Calibration O2::FITCalibration
-        )
-      o2_add_executable(ft0-time-offset-calib
-        COMPONENT_NAME calibration
-        SOURCES testWorkflow/FT0TimeOffsetCalibration-Workflow.cxx
-        PUBLIC_LINK_LIBRARIES
-        O2::FT0Calibration O2::FITCalibration
-        )
-      o2_add_executable(ft0-tf-processor
-        COMPONENT_NAME calibration
-        SOURCES testWorkflow/FT0TFProcessor-Workflow.cxx
-        PUBLIC_LINK_LIBRARIES
-        O2::FT0Calibration
-        )
-      o2_add_executable(ft0-time-spectra-processor
-        COMPONENT_NAME calibration
-        SOURCES testWorkflow/FT0TimeSpectraProcessor-Workflow.cxx
-        PUBLIC_LINK_LIBRARIES
-        O2::FT0Calibration
-        )
-      o2_add_executable(ft0-slew-upload
-        COMPONENT_NAME calibration
-        SOURCES testWorkflow/slew_upload.cxx
-        PUBLIC_LINK_LIBRARIES
-        O2::FT0Calibration
-        )
-      o2_add_executable(ft0-global-offsets
-        COMPONENT_NAME calibration
-        SOURCES testWorkflow/GlobalOffsetsCalibrationWorkflow.cxx
-        PUBLIC_LINK_LIBRARIES
-        O2::FT0Calibration O2::FITCalibration
-        )
-     o2_add_executable(ft0-collect-global-offsets-workflow
-        COMPONENT_NAME calibration
-        SOURCES testWorkflow/GlobalOffsetsCollectWorkflow.cxx
-        PUBLIC_LINK_LIBRARIES
-        O2::FT0Calibration
-        )
-      o2_add_executable(ft0-collect-calib-workflow
-        COMPONENT_NAME calibration
-        SOURCES testWorkflow/ft0-collect-calib-workflow.cxx
-        PUBLIC_LINK_LIBRARIES
-        O2::FT0Calibration
-        )
-      o2_add_executable(ft0-dcs-sim-workflow
-        COMPONENT_NAME calibration
-        SOURCES testWorkflow/ft0-dcs-sim-workflow.cxx
-        PUBLIC_LINK_LIBRARIES O2::DCStestWorkflow)
-      o2_add_executable(ft0-dcs-workflow
-        COMPONENT_NAME calibration
-        SOURCES testWorkflow/ft0-dcs-data-workflow.cxx
-        PUBLIC_LINK_LIBRARIES O2::Framework
-                              O2::FT0Calibration
-                              O2::DetectorsDCS)
+  SOURCES
+    src/FT0TimeOffsetSlotContainer.cxx
+    src/EventsPerBcCalibrator.cxx
+  PUBLIC_LINK_LIBRARIES
+    O2::DetectorsCalibration
+    O2::Framework
+    O2::CommonUtils
+    Microsoft.GSL::GSL
+    O2::DataFormatsFT0
+    O2::CommonDataFormat
+    O2::Steer
+    O2::CCDB
+    ROOT::Minuit
+    ROOT::Hist
+  )
 
-o2_data_file(COPY files DESTINATION Detectors/FT0/)
+o2_target_root_dictionary(FT0Calibration
+  HEADERS
+    include/FT0Calibration/FT0TimeOffsetSlotContainer.h
+    include/FT0Calibration/EventsPerBcCalibrator.h
+  )
+
+o2_add_executable(ft0-time-offset-calib
+  COMPONENT_NAME calibration
+  SOURCES
+    workflow/FT0TimeOffsetCalibration-Workflow.cxx
+  PUBLIC_LINK_LIBRARIES
+    O2::FT0Calibration O2::FITCalibration
+  )
+
+o2_add_executable(ft0-time-spectra-processor
+  COMPONENT_NAME calibration
+  SOURCES
+    workflow/FT0TimeSpectraProcessor-Workflow.cxx
+  PUBLIC_LINK_LIBRARIES
+    O2::FT0Calibration
+  )
+
+o2_add_executable(ft0-events-per-bc-processor
+  COMPONENT_NAME calibration
+  SOURCES
+    workflow/FT0EventsPerBcProcessor-Workflow.cxx
+  PUBLIC_LINK_LIBRARIES
+    O2::FT0Calibration
+    O2::Framework
+    O2::CCDB
+)
\ No newline at end of file
diff --git a/Detectors/FIT/FT0/calibration/README.md b/Detectors/FIT/FT0/calibration/README.md
new file mode 100644
index 0000000000000..78b0f980400d2
--- /dev/null
+++ b/Detectors/FIT/FT0/calibration/README.md
@@ -0,0 +1,62 @@
+# Calibrations
+
+## Events per BC Calibration
+### Description
+Generates histograms of **Events per Bunch Crossing (BC)**. Events can be filtered by applying amplitude thresholds to the **A-side** and **C-side**.
+
+### Command-Line Options
+| Option | Default | Description |
+| :--- | :--- | :--- |
+| `--slot-len-sec` | `3600` | Duration of each slot in seconds. |
+| `--slot-len-tf` | `0` | Slot length in Time Frames (TFs). |
+| `--one-object-per-run` | — | If set, the workflow creates only one calibration object per run. |
+| `--min-entries-number` | `0` | Minimum number of entries required for a slot to be valid. |
+| `--min-ampl-side-a` | `-2147483648` | Amplitude threshold for Side A events. |
+| `--min-ampl-side-c` | `-2147483648` | Amplitude threshold for Side C events. |
+
+---
+
+## How to Run
+
+### Simulation Data
+First, it is important to digitize data with a non-zero run number, orbit, and timestamp. To set these parameters, one can use the `--configKeyValues` option, as shown in the example below.
+```
+o2-sim-digitizer-workflow \
+--onlyDet FT0 \
+--configKeyValues="HBFUtils.nHBFPerTF=128;HBFUtils.orbitFirst=128;HBFUtils.orbitFirstSampled=256;HBFUtils.runNumber=560560;HBFUtils.startTime=1768464099000"
+```
+
+To process simulation data, digits must first be converted to RAW format. The `o2-ft0-digi2raw` tool performs this conversion and generates the required configuration file.
+
+Once converted, you can run the calibration either as a single integrated workflow or by spawning as the sender and receiver components separately.
+
+#### Single Workflow Example
+Execute the following command within the simulation directory:
+```
+o2-raw-file-reader-workflow --input-conf FT0raw.cfg --loop -1 \
+| o2-ft0-flp-dpl-workflow --condition-backend=http://localhost:8080 \
+| o2-calibration-ft0-events-per-bc-processor --FT0EventsPerBcProcessor "--slot-len-sec=10" \
+| o2-calibration-ccdb-populator-workflow --ccdb-path=http://localhost:8080
+```
+
+Sender example (in simulation directory):
+```
+o2-raw-file-reader-workflow --input-conf FT0raw.cfg --loop -1 \
+| o2-ft0-flp-dpl-workflow --condition-backend=http://localhost:8080 \
+| o2-dpl-output-proxy --channel-config "name=downstream,method=connect,address=tcp://localhost:30453,type=push,transport=zeromq" --dataspec "downstream:FT0/DIGITSBC"
+```
+
+Receiver example:
+```
+o2-dpl-raw-proxy --channel-config "name=readout-proxy,type=pull,method=bind,address=tcp://localhost:30453,rateLogging=1,transport=zeromq" --dataspec "A:FT0/DIGITSBC/0" \
+| o2-calibration-ft0-events-per-bc-processor --FT0EventsPerBcProcessor "--slot-len-sec=10 --min-ampl-side-a=0" \
+| o2-calibration-ccdb-populator-workflow --ccdb-path=http://localhost:8080/
+```
+
+### CTF Data
+Example:
+```
+o2-ctf-reader-workflow --ctf-input ctf.root --onlyDet FT0 \
+| o2-calibration-ft0-events-per-bc-processor --FT0EventsPerBcProcessor "--slot-len-sec=10" \
+| o2-calibration-ccdb-populator-workflow --ccdb-path=http://localhost:8080/
+```
\ No newline at end of file
diff --git a/Detectors/FIT/FT0/calibration/files/dispatcherCalib.json b/Detectors/FIT/FT0/calibration/files/dispatcherCalib.json
deleted file mode 100644
index edaf486043790..0000000000000
--- a/Detectors/FIT/FT0/calibration/files/dispatcherCalib.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
-  "dataSamplingPolicies": [
-    {
-      "id": "sampling1",
-      "active": "true",
-      "machines": [
-        "localhost"
-      ],
-      "query": "digits:FT0/DIGITSBC/0;channels:FT0/DIGITSCH/0",
-      "outputs": "digits_sampled:FT0/SUB_DIGITSBC/0;channels_sampled:FT0/SUB_DIGITSCH/0",
-      "samplingConditions": [
-        {
-          "condition": "random",
-          "fraction": "0.99",
-          "seed": "1234"
-        }
-      ],
-      "blocking": "false"
-    } 
-  ]
-}
diff --git a/Detectors/FIT/FT0/calibration/include/FT0Calibration/CalibParam.h b/Detectors/FIT/FT0/calibration/include/FT0Calibration/CalibParam.h
deleted file mode 100644
index a3584b441052f..0000000000000
--- a/Detectors/FIT/FT0/calibration/include/FT0Calibration/CalibParam.h
+++ /dev/null
@@ -1,44 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-#ifndef ALICEO2_CALIB_PARAM_H_
-#define ALICEO2_CALIB_PARAM_H_
-
-#include "CommonUtils/ConfigurableParamHelper.h"
-
-namespace o2
-{
-namespace ft0
-{
-
-struct CalibParam : o2::conf::ConfigurableParamHelper {
-  // Logic for obtaining bad channels and for making decision concerning slot finalization
-  std::size_t mMinEntriesThreshold = 500;
-  std::size_t mMaxEntriesThreshold = 1000;
-  uint8_t mNExtraSlots = 1;
-  // Fitting ranges
-  double mMinFitRange = -200.;
-  double mMaxFitRange = 200.;
-  // Conditions for checking fit quality, otherwise hist mean will be taken as offset
-  double mMaxDiffMean = 20;
-  double mMinRMS = 3;
-  double mMaxSigma = 30;
-  //
-  bool mUseDynamicRange = false; // use dynamic ranges [mean-RMS*mRangeInRMS,mean+RMS*mRangeInRMS] for fitting
-  double mRangeInRMS = 3;
-
-  O2ParamDef(CalibParam, "FT0CalibParam");
-};
-
-} // end namespace ft0
-} // end namespace o2
-
-#endif /* FT0_CALIB_PARAM_H_ */
\ No newline at end of file
diff --git a/Detectors/FIT/FT0/calibration/include/FT0Calibration/EventsPerBcCalibrator.h b/Detectors/FIT/FT0/calibration/include/FT0Calibration/EventsPerBcCalibrator.h
new file mode 100644
index 0000000000000..d831cc36201ab
--- /dev/null
+++ b/Detectors/FIT/FT0/calibration/include/FT0Calibration/EventsPerBcCalibrator.h
@@ -0,0 +1,83 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef O2_FT0TVXPERBCID
+#define O2_FT0TVXPERBCID
+
+#include 
+#include 
+#include 
+#include 
+
+#include "CommonDataFormat/FlatHisto2D.h"
+#include "CommonConstants/LHCConstants.h"
+#include "DataFormatsFT0/SpectraInfoObject.h"
+#include "DataFormatsFT0/Digit.h"
+#include "DataFormatsFT0/EventsPerBc.h"
+#include "DetectorsCalibration/TimeSlotCalibration.h"
+#include "DetectorsCalibration/TimeSlot.h"
+#include "CommonDataFormat/TFIDInfo.h"
+#include "TH1F.h"
+#include "Rtypes.h"
+
+namespace o2::ft0
+{
+struct EventsPerBcContainer {
+  EventsPerBcContainer(int32_t minAmplitudeSideA, int32_t minAmplitudeSideC, int32_t minSumOfAmplitude) : mMinAmplitudeSideA(minAmplitudeSideA), mMinAmplitudeSideC(minAmplitudeSideC), mMinSumOfAmplitude(minSumOfAmplitude) {}
+
+  size_t getEntries() const { return entries; }
+  void print() const;
+  void fill(const o2::dataformats::TFIDInfo& ti, const gsl::span data);
+  void merge(const EventsPerBcContainer* prev);
+
+  const int32_t mMinAmplitudeSideA;
+  const int32_t mMinAmplitudeSideC;
+  const int32_t mMinSumOfAmplitude;
+
+  std::array mTvx{0.0};
+  size_t entries{0};
+  long startTimeStamp{0};
+  long stopTimeStamp{0};
+
+  ClassDefNV(EventsPerBcContainer, 1);
+};
+
+class EventsPerBcCalibrator final : public o2::calibration::TimeSlotCalibration
+{
+  using Slot = o2::calibration::TimeSlot;
+  using TFType = o2::calibration::TFType;
+  using EventsHistogram = std::array;
+
+ public:
+  EventsPerBcCalibrator(uint32_t minNumberOfEntries, int32_t minAmplitudeSideA, int32_t minAmplitudeSideC, int32_t minSumOfAmplitude);
+
+  bool hasEnoughData(const Slot& slot) const override;
+  void initOutput() override;
+  void finalizeSlot(Slot& slot) override;
+  Slot& emplaceNewSlot(bool front, TFType tstart, TFType tend) override;
+
+  const std::vector& getTvxPerBc() { return mTvxPerBcs; }
+  std::vector>& getTvxPerBcCcdbInfo() { return mTvxPerBcInfos; }
+
+ private:
+  const uint32_t mMinNumberOfEntries;
+  const int32_t mMinAmplitudeSideA;
+  const int32_t mMinAmplitudeSideC;
+  const int32_t mMinSumOfAmplitude;
+
+  std::vector mTvxPerBcs;
+  std::vector> mTvxPerBcInfos;
+
+  ClassDefOverride(EventsPerBcCalibrator, 1);
+};
+} // namespace o2::ft0
+
+#endif
diff --git a/Detectors/FIT/FT0/calibration/include/FT0Calibration/FT0CalibCollector.h b/Detectors/FIT/FT0/calibration/include/FT0Calibration/FT0CalibCollector.h
deleted file mode 100644
index 4994867b5a675..0000000000000
--- a/Detectors/FIT/FT0/calibration/include/FT0Calibration/FT0CalibCollector.h
+++ /dev/null
@@ -1,95 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-#ifndef FT0_CALIB_COLLECTOR_H_
-#define FT0_CALIB_COLLECTOR_H_
-
-#include "DetectorsCalibration/TimeSlotCalibration.h"
-#include "DetectorsCalibration/TimeSlot.h"
-#include "DataFormatsFT0/FT0CalibrationInfoObject.h"
-#include "FT0Base/Geometry.h"
-
-#include 
-
-namespace o2
-{
-namespace ft0
-{
-
-class FT0CalibInfoSlot
-{
-
-  using Slot = o2::calibration::TimeSlot;
-  using Geo = o2::ft0::Geometry;
-
- public:
-  static constexpr int NCHANNELS = Geo::Nchannels;
-  static constexpr int HISTO_RANGE = 200;
-
-  FT0CalibInfoSlot()
-  {
-    for (int ch = 0; ch < NCHANNELS; ch++) {
-      mEntriesSlot[ch] = 0;
-    }
-  }
-
-  ~FT0CalibInfoSlot() = default;
-
-  void print() const;
-  void printEntries() const;
-  void fill(const gsl::span data);
-  void merge(const FT0CalibInfoSlot* prev);
-
-  auto& getEntriesPerChannel() const { return mEntriesSlot; }
-  auto& getEntriesPerChannel() { return mEntriesSlot; }
-  auto& getCollectedCalibInfoSlot() { return mFT0CollectedCalibInfoSlot; }
-  auto& getCollectedCalibInfoSlot() const { return mFT0CollectedCalibInfoSlot; }
-
- private:
-  std::array mEntriesSlot;                                   // vector containing number of entries per channel
-  std::vector mFT0CollectedCalibInfoSlot; ///< output FT0 calibration info
-
-  ClassDefNV(FT0CalibInfoSlot, 1);
-};
-
-class FT0CalibCollector final : public o2::calibration::TimeSlotCalibration
-{
-  using TFType = o2::calibration::TFType;
-  using Slot = o2::calibration::TimeSlot;
-  static constexpr int NCHANNELS = o2::ft0::Geometry::Nchannels;
-
- public:
-  FT0CalibCollector(bool TFsendingPolicy, int maxNumOfHits) : mTFsendingPolicy(TFsendingPolicy), mMaxNumOfHits(maxNumOfHits){};
-
-  ~FT0CalibCollector() final = default;
-
-  bool hasEnoughData(const Slot& slot) const final;
-  void initOutput() final;
-  void finalizeSlot(Slot& slot) final;
-  Slot& emplaceNewSlot(bool front, TFType tstart, TFType tend) final;
-  auto& getCollectedCalibInfo() const { return mFT0CollectedCalibInfo; }
-  auto& getEntriesPerChannel() const { return mEntries; }
-  void setIsMaxNumberOfHitsAbsolute(bool absNumber) { mAbsMaxNumOfHits = absNumber; }
-
- private:
-  bool mTFsendingPolicy = false;                                         // whether we will send information at every TF or only when we have a certain statistics
-  int mMaxNumOfHits = 1000000;                                           // maximum number of hits for one single channel to trigger the sending of the information (if mTFsendingPolicy = false)
-  bool mAbsMaxNumOfHits = true;                                          // to decide if the mMaxNumOfHits should be multiplied by the number of FT0 channels
-  std::array mEntries;                                   // vector containing number of entries per channel
-  std::vector mFT0CollectedCalibInfo; ///< output FT0 calibration info
-
-  ClassDefOverride(FT0CalibCollector, 1);
-};
-
-} // end namespace ft0
-} // end namespace o2
-
-#endif /* FT0 CALIB COLLECTOR */
diff --git a/Detectors/FIT/FT0/calibration/include/FT0Calibration/FT0CalibTimeSlewing.h b/Detectors/FIT/FT0/calibration/include/FT0Calibration/FT0CalibTimeSlewing.h
deleted file mode 100644
index adb37a74280ff..0000000000000
--- a/Detectors/FIT/FT0/calibration/include/FT0Calibration/FT0CalibTimeSlewing.h
+++ /dev/null
@@ -1,78 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-/// \file CalibTimeSlewingParamFT0.h
-/// \brief Class to store the output of the matching to FT0 for calibration
-
-#ifndef ALICEO2_FT0CALIBTIMESLEWING_H
-#define ALICEO2_FT0CALIBTIMESLEWING_H
-
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include "Rtypes.h"
-#include "FairLogger.h"
-#include "DataFormatsFT0/FT0CalibrationInfoObject.h"
-#include "FT0Base/Geometry.h"
-namespace o2::ft0
-{
-static constexpr int NCHANNELS = o2::ft0::Geometry::Nchannels;
-class FT0CalibTimeSlewing
-{
- public:
-  static constexpr int HISTOGRAM_RANGE_X = 4000;
-  static constexpr unsigned int NUMBER_OF_HISTOGRAM_BINS_X = HISTOGRAM_RANGE_X / 4;
-  static constexpr int HISTOGRAM_RANGE_Y = 200;
-  static constexpr unsigned int NUMBER_OF_HISTOGRAM_BINS_Y = HISTOGRAM_RANGE_Y;
-  FT0CalibTimeSlewing();
-
-  FT0CalibTimeSlewing(const FT0CalibTimeSlewing& source) = default;
-
-  FT0CalibTimeSlewing& operator=(const FT0CalibTimeSlewing& source) = default;
-
-  float getChannelOffset(int channel, int amplitude) const;
-
-  const TGraph getGraph(int channel) const { return mTimeSlewing[channel]; }
-  std::array getGraphs() const { return mTimeSlewing; }
-  void fillGraph(int channel, TH2F* histo);
-
-  float getSigmaPeak(int channel) const { return mSigmaPeak[channel]; }
-  void setSigmaPeak(int channel, float value) { mSigmaPeak[channel] = value; }
-
-  ///< perform all initializations
-  void init();
-  void mergeFilesWithTree();
-  void fillHistos(TTree* tr);
-  TH2F* getTimeAmpHist(int channel) { return mTimeAmpHist[channel]; };
-  void setSingleFileName(std::string name) { mSingleFileName = name; }
-  void setMergedFileName(std::string name) { mMergedFileName = name; }
-  void setNfiles(int nfiles) { mNfiles = nfiles; };
-
-  FT0CalibTimeSlewing& operator+=(const FT0CalibTimeSlewing& other);
-  static constexpr const char* getObjectPath() { return "FT0/Calib/SlewingCorrection"; }
-
- private:
-  // FT0 channel calibrations
-  std::array mTimeSlewing; ///< array of TGraph wirh time -amplitude for each channel
-  std::array mSigmaPeak;    ///< array with the sigma of the peak
-  TFile* mMergedFile;                         // file  with merged tree
-  TH2F* mTimeAmpHist[NCHANNELS];              // historgams time vs amplitude
-  int mNfiles;                                // number of files with stored Tree with CalibrationInfoObject
-  std::string mSingleFileName;
-  std::string mMergedFileName;
-
-  ClassDefNV(FT0CalibTimeSlewing, 1); // class for FT0 time slewing params
-};
-} // namespace o2::ft0
-#endif
diff --git a/Detectors/FIT/FT0/calibration/include/FT0Calibration/FT0ChannelTimeOffsetSlotContainer.h b/Detectors/FIT/FT0/calibration/include/FT0Calibration/FT0ChannelTimeOffsetSlotContainer.h
deleted file mode 100644
index b8b2880b8d408..0000000000000
--- a/Detectors/FIT/FT0/calibration/include/FT0Calibration/FT0ChannelTimeOffsetSlotContainer.h
+++ /dev/null
@@ -1,80 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-#ifndef O2_FT0CHANNELTIMEOFFSETSLOTCONTAINER_H
-#define O2_FT0CHANNELTIMEOFFSETSLOTCONTAINER_H
-
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include "DataFormatsFT0/FT0CalibrationInfoObject.h"
-#include "DataFormatsFT0/FT0ChannelTimeCalibrationObject.h"
-#include "FT0Base/Geometry.h"
-#include "Rtypes.h"
-#include 
-
-namespace o2::ft0
-{
-
-class FT0ChannelTimeOffsetSlotContainer final
-{
-
-  // ranges to be discussed
-  static constexpr int HISTOGRAM_RANGE = 500;
-  static constexpr unsigned int NUMBER_OF_HISTOGRAM_BINS = HISTOGRAM_RANGE;
-  static constexpr int NCHANNELS = o2::ft0::Geometry::Nchannels;
-
- public:
-  explicit FT0ChannelTimeOffsetSlotContainer(std::size_t minEntries);
-  FT0ChannelTimeOffsetSlotContainer(FT0ChannelTimeOffsetSlotContainer const&);
-  FT0ChannelTimeOffsetSlotContainer(FT0ChannelTimeOffsetSlotContainer&&) = default;
-  FT0ChannelTimeOffsetSlotContainer& operator=(FT0ChannelTimeOffsetSlotContainer const&);
-  FT0ChannelTimeOffsetSlotContainer& operator=(FT0ChannelTimeOffsetSlotContainer&&) = default;
-  [[nodiscard]] bool hasEnoughEntries() const;
-  void fill(const gsl::span& data);
-  [[nodiscard]] int16_t getMeanGaussianFitValue(std::size_t channelID) const;
-  void merge(FT0ChannelTimeOffsetSlotContainer* prev);
-  void print() const;
-  FT0ChannelTimeCalibrationObject generateCalibrationObject() const;
-  void updateFirstCreation(std::uint64_t creation)
-  {
-    if (creation < mFirstCreation) {
-      mFirstCreation = creation;
-      for (int iCh = 0; iCh < NCHANNELS; iCh++) {
-        std::string histName = std::string{mHistogram[iCh]->GetName()} + "_" + std::to_string(mFirstCreation);
-        mHistogram[iCh]->SetName(histName.c_str());
-      }
-    }
-  }
-  void resetFirstCreation()
-  {
-    mFirstCreation = std::numeric_limits::max();
-  }
-  std::uint64_t getFirstCreation() const
-  {
-    return mFirstCreation;
-  }
-  static int sGausFitBins;
-
- private:
-  std::size_t mMinEntries = 1000;
-  std::array mEntriesPerChannel{};
-  std::array, NCHANNELS> mHistogram;
-  std::uint64_t mFirstCreation = std::numeric_limits::max();
-  ClassDefNV(FT0ChannelTimeOffsetSlotContainer, 2);
-};
-
-} // namespace o2::ft0
-
-#endif // O2_FT0CHANNELTIMEOFFSETSLOTCONTAINER_H
diff --git a/Detectors/FIT/FT0/calibration/include/FT0Calibration/FT0DCSProcessor.h b/Detectors/FIT/FT0/calibration/include/FT0Calibration/FT0DCSProcessor.h
deleted file mode 100644
index 0c7f229c5edce..0000000000000
--- a/Detectors/FIT/FT0/calibration/include/FT0Calibration/FT0DCSProcessor.h
+++ /dev/null
@@ -1,98 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-#ifndef DETECTOR_FT0DCSPROCESSOR_H_
-#define DETECTOR_FT0DCSPROCESSOR_H_
-
-#include 
-#include 
-#include 
-#include 
-#include 
-#include "Framework/Logger.h"
-#include "DataFormatsFIT/DCSDPValues.h"
-#include "DetectorsDCS/DataPointCompositeObject.h"
-#include "DetectorsDCS/DataPointIdentifier.h"
-#include "DetectorsDCS/DataPointValue.h"
-#include "DetectorsDCS/DeliveryType.h"
-#include "CCDB/CcdbObjectInfo.h"
-#include "CommonUtils/MemFileHelper.h"
-#include "CCDB/CcdbApi.h"
-#include 
-
-/// @brief Class to process DCS data points
-
-namespace o2
-{
-namespace ft0
-{
-
-using DPID = o2::dcs::DataPointIdentifier;
-using DPVAL = o2::dcs::DataPointValue;
-using DPCOM = o2::dcs::DataPointCompositeObject;
-
-class FT0DCSProcessor
-{
-
- public:
-  using CcdbObjectInfo = o2::ccdb::CcdbObjectInfo;
-
-  FT0DCSProcessor() = default;
-  ~FT0DCSProcessor() = default;
-
-  void init(const std::vector& pids);
-
-  int process(const gsl::span dps);
-  int processDP(const DPCOM& dpcom);
-  uint64_t processFlags(uint64_t flag, const char* pid);
-
-  void updateDPsCCDB();
-
-  const CcdbObjectInfo& getccdbDPsInfo() const { return mccdbDPsInfo; }
-  CcdbObjectInfo& getccdbDPsInfo() { return mccdbDPsInfo; }
-  const std::unordered_map& getFT0DPsInfo() const { return mFT0DCS; }
-
-  long getStartValidity() { return mStartValidity; }
-  void setStartValidity(long t) { mStartValidity = t; }
-  void resetStartValidity() { mStartValidity = o2::ccdb::CcdbObjectInfo::INFINITE_TIMESTAMP; }
-
-  void clearDPsinfo()
-  {
-    mDpsMap.clear();
-    mFT0DCS.clear();
-  }
-
-  bool getVerboseMode() { return mVerbose; }
-  void useVerboseMode() { mVerbose = true; }
-
- private:
-  std::unordered_map mFT0DCS; // the object that will go to the CCDB
-  std::unordered_map mPids;                   // contains all PIDs for the processor, the bool
-                                                          // will be true if the DP was processed at least once
-  std::unordered_map mDpsMap;                // this is the map that will hold the DPs
-
-  CcdbObjectInfo mccdbDPsInfo;
-  long mStartValidity = o2::ccdb::CcdbObjectInfo::INFINITE_TIMESTAMP; // TF index for processing, used to store CCDB object for DPs
-
-  union Converter {
-    uint64_t raw_data;
-    double double_value;
-    uint uint_value;
-  } converter;
-
-  bool mVerbose = false;
-
-  ClassDefNV(FT0DCSProcessor, 0);
-};
-} // namespace ft0
-} // namespace o2
-
-#endif
diff --git a/Detectors/FIT/FT0/calibration/include/FT0Calibration/FT0TimeOffsetSlotContainer.h b/Detectors/FIT/FT0/calibration/include/FT0Calibration/FT0TimeOffsetSlotContainer.h
index 08f7e18bcfce3..2d7470d94e9e8 100644
--- a/Detectors/FIT/FT0/calibration/include/FT0Calibration/FT0TimeOffsetSlotContainer.h
+++ b/Detectors/FIT/FT0/calibration/include/FT0Calibration/FT0TimeOffsetSlotContainer.h
@@ -16,7 +16,11 @@
 #include 
 
 #include "CommonDataFormat/FlatHisto2D.h"
-#include "DataFormatsFT0/FT0ChannelTimeCalibrationObject.h"
+#include "DataFormatsFT0/SpectraInfoObject.h"
+#include "DetectorsCalibration/TimeSlotCalibration.h"
+#include "DetectorsCalibration/TimeSlot.h"
+
+#include "TList.h"
 
 #include "Rtypes.h"
 namespace o2::ft0
@@ -25,21 +29,25 @@ namespace o2::ft0
 class FT0TimeOffsetSlotContainer final
 {
   static constexpr int sNCHANNELS = o2::ft0::Geometry::Nchannels;
+  using TimeSlot = o2::calibration::TimeSlot;
+  using TimeSlotCalibration = o2::calibration::TimeSlotCalibration;
 
  public:
   FT0TimeOffsetSlotContainer(std::size_t minEntries); // constructor is needed due to current version of FITCalibration library, should be removed
   FT0TimeOffsetSlotContainer(FT0TimeOffsetSlotContainer const&) = default;
   FT0TimeOffsetSlotContainer(FT0TimeOffsetSlotContainer&&) = default;
-  FT0TimeOffsetSlotContainer& operator=(FT0TimeOffsetSlotContainer const&) = default;
+  FT0TimeOffsetSlotContainer& operator=(FT0TimeOffsetSlotContainer&) = default;
   FT0TimeOffsetSlotContainer& operator=(FT0TimeOffsetSlotContainer&&) = default;
   bool hasEnoughEntries() const;
   void fill(const gsl::span& data);
-  int16_t getMeanGaussianFitValue(std::size_t channelID) const;
+  SpectraInfoObject getSpectraInfoObject(std::size_t channelID, TList* listHists) const;
   void merge(FT0TimeOffsetSlotContainer* prev);
   void print() const;
-  FT0ChannelTimeCalibrationObject generateCalibrationObject() const;
+  TimeSpectraInfoObject generateCalibrationObject(long tsStartMS, long tsEndMS, const std::string& pathToHists) const;
   typedef float FlatHistoValue_t;
   typedef o2::dataformats::FlatHisto2D FlatHisto2D_t;
+  auto getHistogram() const { return mHistogram; }
+  auto isFirstTF() const { return mIsFirstTF; }
 
  private:
   // Slot number
@@ -53,6 +61,8 @@ class FT0TimeOffsetSlotContainer final
   bool mIsReady{false};
   // Once it is upper than max entry threshold it stops increasing
   std::array mArrEntries{};
+  // Total number of events
+  uint64_t mTotalNevents{0};
   // Contains all information about time spectra
   FlatHisto2D_t mHistogram;
   ClassDefNV(FT0TimeOffsetSlotContainer, 1);
diff --git a/Detectors/FIT/FT0/calibration/include/FT0Calibration/GlobalOffsetsContainer.h b/Detectors/FIT/FT0/calibration/include/FT0Calibration/GlobalOffsetsContainer.h
deleted file mode 100644
index c0cf9b049233e..0000000000000
--- a/Detectors/FIT/FT0/calibration/include/FT0Calibration/GlobalOffsetsContainer.h
+++ /dev/null
@@ -1,69 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-#ifndef O2_GLOBALOFFSETSCONTAINER_H
-#define O2_GLOBALOFFSETSCONTAINER_H
-
-#include "DataFormatsFT0/GlobalOffsetsInfoObject.h"
-#include "DataFormatsFT0/GlobalOffsetsCalibrationObject.h"
-#include "Rtypes.h"
-#include 
-#include 
-#include 
-#include 
-
-namespace o2::ft0
-{
-class GlobalOffsetsContainer final
-{
-  static constexpr int RANGE = 1000;
-  static constexpr unsigned int NBINS = 2 * RANGE;
-
- public:
-  explicit GlobalOffsetsContainer(std::size_t minEntries)
-  {
-    //    mHisto.resize(NBINS, 0.);
-  }
-
-  bool hasEnoughEntries() const;
-  void fill(const gsl::span& data);
-  int getMeanGaussianFitValue() const;
-  void merge(GlobalOffsetsContainer* prev);
-  void print() const;
-  GlobalOffsetsCalibrationObject generateCalibrationObject() const;
-  void updateFirstCreation(std::uint64_t creation)
-  {
-    if (creation < mFirstCreation) {
-      mFirstCreation = creation;
-    }
-  }
-  void resetFirstCreation()
-  {
-
-    mFirstCreation = std::numeric_limits::max();
-  }
-  std::uint64_t getFirstCreation() const
-  {
-    return mFirstCreation;
-  }
-
- private:
-  std::uint64_t mFirstCreation = std::numeric_limits::max();
-  std::size_t mMinEntries;
-  std::array mHisto{};
-  int mEntries = 0;
-
-  ClassDefNV(GlobalOffsetsContainer, 2);
-};
-
-} // namespace o2::ft0
-
-#endif // O2_GLOBALOFFSETCONTAINER_H
diff --git a/Detectors/FIT/FT0/calibration/include/FT0Calibration/LHCClockCalibrator.h b/Detectors/FIT/FT0/calibration/include/FT0Calibration/LHCClockCalibrator.h
deleted file mode 100644
index 42e69c199b946..0000000000000
--- a/Detectors/FIT/FT0/calibration/include/FT0Calibration/LHCClockCalibrator.h
+++ /dev/null
@@ -1,89 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-#ifndef FT0_LHCPHASE_CALIBRATION_H_
-#define FT0_LHCPHASE_CALIBRATION_H_
-
-#include "DetectorsCalibration/TimeSlotCalibration.h"
-#include "DetectorsCalibration/TimeSlot.h"
-#include "DataFormatsFT0/FT0CalibrationInfoObject.h"
-#include "DataFormatsFT0/LHCphaseCalibrationObject.h"
-#include "CommonConstants/LHCConstants.h"
-#include "CommonUtils/NameConf.h"
-#include "CCDB/CcdbObjectInfo.h"
-#include 
-
-namespace o2
-{
-namespace ft0
-{
-
-struct LHCClockDataHisto {
-  float range = 1000. * o2::constants::lhc::LHCBunchSpacingNS * 0.5; // BC in PS
-  int nbins = 1000;
-  float v2Bin = nbins / (2 * range);
-  int entries = 0;
-  std::vector histo{0};
-
-  LHCClockDataHisto();
-
-  LHCClockDataHisto(int nb, float r) : nbins(nb), range(r), v2Bin(0)
-  {
-    if (r <= 0. || nb < 1) {
-      throw std::runtime_error("Wrong initialization of the histogram");
-    }
-    v2Bin = nbins / (2 * range);
-    histo.resize(nbins, 0.);
-  }
-
-  size_t getEntries() const { return entries; }
-  void print() const;
-  void fill(const gsl::span& data);
-  void merge(const LHCClockDataHisto* prev);
-
-  ClassDefNV(LHCClockDataHisto, 1);
-};
-
-class LHCClockCalibrator
-{
-  using TFType = o2::calibration::TFType;
-  using Slot = o2::calibration::TimeSlot;
-  using LHCphase = o2::ft0::LHCphaseCalibrationObject;
-  using CcdbObjectInfo = o2::ccdb::CcdbObjectInfo;
-  using CcdbObjectInfoVector = std::vector;
-  using LHCphaseVector = std::vector;
-
- public:
-  LHCClockCalibrator(int minEnt = 500, int nb = 1000, float r = 24400, const std::string path = o2::base::NameConf::getCCDBServer()) : mMinEntries(minEnt), mNBins(nb), mRange(r) {}
-  ~LHCClockCalibrator() = default;
-  bool hasEnoughData(const Slot& slot) const { return slot.getContainer()->entries >= mMinEntries; }
-  void initOutput();
-  void finalizeSlot(Slot& slot);
-  Slot& emplaceNewSlot(bool front, TFType tstart, TFType tend);
-
-  const LHCphaseVector& getLHCphaseVector() const { return mLHCphaseVector; }
-  const CcdbObjectInfoVector& getLHCphaseInfoVector() const { return mInfoVector; }
-  CcdbObjectInfoVector& getLHCphaseInfoVector() { return mInfoVector; }
-
- private:
-  int mMinEntries = 0;
-  int mNBins = 0;
-  float mRange = 0.;
-  CcdbObjectInfoVector mInfoVector; // vector of CCDB Infos , each element is filled with the CCDB description of the accompanying LHCPhase
-  LHCphaseVector mLHCphaseVector;   // vector of LhcPhase, each element is filled in "process" when we finalize one slot (multiple can be finalized during the same "process", which is why we have a vector. Each element is to be considered the output of the device, and will go to the CCDB
-
-  ClassDef(LHCClockCalibrator, 1);
-};
-
-} // end namespace ft0
-} // end namespace o2
-
-#endif /* FT0_LHCPHASE_CALIBRATION_H_ */
diff --git a/Detectors/FIT/FT0/calibration/include/FT0Calibration/RecoCalibInfoWorkflow.h b/Detectors/FIT/FT0/calibration/include/FT0Calibration/RecoCalibInfoWorkflow.h
deleted file mode 100644
index c9c0bb6210ce6..0000000000000
--- a/Detectors/FIT/FT0/calibration/include/FT0Calibration/RecoCalibInfoWorkflow.h
+++ /dev/null
@@ -1,61 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-/// \file   RecoCalibInfoWorkflow.h
-///\ brief  Collect data for global offsets calibration
-/// \author Alla.Maevskaya@cern.ch
-
-#ifndef O2_RECOCALIBINFO_WORKFLOW
-#define O2_RECOCALIBINFO_WORKFLOW
-
-#include 
-#include 
-#include 
-#include "Framework/DeviceSpec.h"
-#include "Framework/WorkflowSpec.h"
-#include "Framework/Task.h"
-#include "DataFormatsGlobalTracking/RecoContainer.h"
-#include "DataFormatsFT0/RecPoints.h"
-#include "CommonDataFormat/InteractionRecord.h"
-#include "CommonDataFormat/TimeStamp.h"
-#include "ReconstructionDataFormats/GlobalTrackID.h"
-#include "ReconstructionDataFormats/PrimaryVertex.h"
-#include "DataFormatsFT0/RecoCalibInfoObject.h"
-#include "TStopwatch.h"
-#include 
-#include 
-
-using namespace o2::framework;
-using DataRequest = o2::globaltracking::DataRequest;
-using GID = o2::dataformats::GlobalTrackID;
-
-namespace o2::ft0
-{
-
-class RecoCalibInfoWorkflow final : public o2::framework::Task
-{
- public:
-  RecoCalibInfoWorkflow() {}
-  void run(o2::framework::ProcessingContext& pc) final;
-  void init(InitContext& ic) final;
-  void endOfStream(framework::EndOfStreamContext& ec) final;
-
- private:
-  std::shared_ptr mDataRequest;
-  const float cSpeed = 0.029979246f; // speed of light in TOF units
-  GID::mask_t mInputSources;
-  TStopwatch mTimer;
-};
-framework::DataProcessorSpec getRecoCalibInfoWorkflow(bool useMC);
-
-} // namespace o2::ft0
-
-#endif /* O2_RECOCALIBINFO_WORKFLOW */
diff --git a/Detectors/FIT/FT0/calibration/macros/CMakeLists.txt b/Detectors/FIT/FT0/calibration/macros/CMakeLists.txt
deleted file mode 100644
index 8531528e86c50..0000000000000
--- a/Detectors/FIT/FT0/calibration/macros/CMakeLists.txt
+++ /dev/null
@@ -1,13 +0,0 @@
-o2_add_test_root_macro(
-        makeChannelOffsetCalibObjectInCCDB.C
-        PUBLIC_LINK_LIBRARIES O2::DataFormatsFT0 O2::Framework O2::CCDB O2::DetectorsCalibration
-        LABELS ft0)
-
-o2_add_test_root_macro(
-        makeFT0CCDBEntryForDCS.C
-        PUBLIC_LINK_LIBRARIES O2::DetectorsDCS O2::CCDB
-        LABELS ft0)
-
-o2_add_test_root_macro(
-        readFT0DCSentries.C
-        PUBLIC_LINK_LIBRARIES O2::DetectorsDCS O2::CCDB)
\ No newline at end of file
diff --git a/Detectors/FIT/FT0/calibration/macros/makeChannelOffsetCalibObjectInCCDB.C b/Detectors/FIT/FT0/calibration/macros/makeChannelOffsetCalibObjectInCCDB.C
deleted file mode 100644
index f184989c75bc3..0000000000000
--- a/Detectors/FIT/FT0/calibration/macros/makeChannelOffsetCalibObjectInCCDB.C
+++ /dev/null
@@ -1,49 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-#if !defined(__CLING__) || defined(__ROOTCLING__)
-#include 
-#include "TFile.h"
-#include "CCDB/CcdbApi.h"
-#include "CCDB/CCDBTimeStampUtils.h"
-#include "CCDB/CcdbObjectInfo.h"
-#include "CommonUtils/MemFileHelper.h"
-#include "CCDB/BasicCCDBManager.h"
-#include 
-#include 
-#include "DataFormatsFT0/FT0ChannelTimeCalibrationObject.h"
-#endif
-
-int makeChannelOffsetCalibObjectInCCDB(const std::string url = "http://ccdb-test.cern.ch:8080")
-{
-  using CalibObjWithInfoType = std::pair>>;
-  std::array offsets;
-  for (int i = 0; i < 208; i++) {
-    offsets[i] = 0;
-  }
-  o2::ccdb::CcdbApi api;
-  api.init(url);
-  //  std::map md;
-  CalibObjWithInfoType result;
-  o2::ft0::FT0ChannelTimeCalibrationObject calibrationObject;
-  static std::map metaData;
-  auto clName = o2::utils::MemFileHelper::getClassName(calibrationObject);
-  auto flName = o2::ccdb::CcdbApi::generateFileName(clName);
-  uint64_t starting = 1546300800;                                   // 01.01.2019
-  uint64_t stopping = o2::ccdb::CcdbObjectInfo::INFINITE_TIMESTAMP; // 1633046400; // 01.10.2021
-  LOG(info) << " clName " << clName << " flName " << flName;
-  result.first = o2::ccdb::CcdbObjectInfo("FT0/Calib/ChannelTimeOffset", clName, flName, metaData, starting, stopping);
-  result.second = o2::ccdb::CcdbApi::createObjectImage(&offsets, &result.first);
-  LOG(info) << " start " << starting << " end " << stopping;
-  api.storeAsTFileAny(&calibrationObject, "FT0/Calib/ChannelTimeOffset", metaData, starting, stopping);
-
-  return 0;
-}
diff --git a/Detectors/FIT/FT0/calibration/macros/makeFT0CCDBEntryForDCS.C b/Detectors/FIT/FT0/calibration/macros/makeFT0CCDBEntryForDCS.C
deleted file mode 100644
index e845a1d3d78ec..0000000000000
--- a/Detectors/FIT/FT0/calibration/macros/makeFT0CCDBEntryForDCS.C
+++ /dev/null
@@ -1,69 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-#include 
-#include 
-#include "TFile.h"
-#include "CCDB/CcdbApi.h"
-#include "DetectorsDCS/AliasExpander.h"
-#include "DetectorsDCS/DeliveryType.h"
-#include "DetectorsDCS/DataPointIdentifier.h"
-
-#include 
-#include 
-
-using DPID = o2::dcs::DataPointIdentifier;
-
-int makeFT0CCDBEntryForDCS(const std::string url = "http://localhost:8080")
-{
-  // Macro to populate CCDB for FT0 with the configuration for DCS
-
-  std::unordered_map dpid2DataDesc;
-  std::vector aliasesHV = {"FT0/HV/FT0_A/MCP_A[1..5]/actual/iMon",
-                                        "FT0/HV/FT0_A/MCP_B[1..5]/actual/iMon",
-                                        "FT0/HV/FT0_A/MCP_C[1..2]/actual/iMon",
-                                        "FT0/HV/FT0_A/MCP_C[4..5]/actual/iMon",
-                                        "FT0/HV/FT0_A/MCP_D[1..5]/actual/iMon",
-                                        "FT0/HV/FT0_A/MCP_E[1..5]/actual/iMon",
-                                        "FT0/HV/FT0_C/MCP_A[2..5]/actual/iMon",
-                                        "FT0/HV/FT0_C/MCP_B[1..6]/actual/iMon",
-                                        "FT0/HV/FT0_C/MCP_C[1..2]/actual/iMon",
-                                        "FT0/HV/FT0_C/MCP_C[5..6]/actual/iMon",
-                                        "FT0/HV/FT0_C/MCP_D[1..2]/actual/iMon",
-                                        "FT0/HV/FT0_C/MCP_D[5..6]/actual/iMon",
-                                        "FT0/HV/FT0_C/MCP_E[1..6]/actual/iMon",
-                                        "FT0/HV/FT0_C/MCP_F[2..5]/actual/iMon",
-                                        "FT0/HV/MCP_LC/actual/iMon"};
-  std::string aliasesADCZERO = "FT0/PM/channel[000..211]/actual/ADC[0..1]_BASELINE";
-  std::vector expAliasesHV = o2::dcs::expandAliases(aliasesHV);
-  std::vector expAliasesADCZERO = o2::dcs::expandAlias(aliasesADCZERO);
-
-  std::cout << "DP aliases:" << std::endl;
-  DPID dpidtmp;
-  for (size_t i = 0; i < expAliasesHV.size(); ++i) {
-    std::cout << expAliasesHV[i] << std::endl;
-    DPID::FILL(dpidtmp, expAliasesHV[i], o2::dcs::DeliveryType::DPVAL_DOUBLE);
-    dpid2DataDesc[dpidtmp] = "FT0DATAPOINTS";
-  }
-  for (size_t i = 0; i < expAliasesADCZERO.size(); ++i) {
-    std::cout << expAliasesADCZERO[i] << std::endl;
-    DPID::FILL(dpidtmp, expAliasesADCZERO[i], o2::dcs::DeliveryType::DPVAL_UINT);
-    dpid2DataDesc[dpidtmp] = "FT0DATAPOINTS";
-  }
-
-  o2::ccdb::CcdbApi api;
-  api.init(url); // or http://localhost:8080 for a local installation
-  std::map md;
-  long ts = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count();
-  api.storeAsTFileAny(&dpid2DataDesc, "FT0/Config/DCSDPconfig", md, ts, 99999999999999);
-
-  return 0;
-}
diff --git a/Detectors/FIT/FT0/calibration/macros/readFT0DCSentries.C b/Detectors/FIT/FT0/calibration/macros/readFT0DCSentries.C
deleted file mode 100644
index 08232957a3d5f..0000000000000
--- a/Detectors/FIT/FT0/calibration/macros/readFT0DCSentries.C
+++ /dev/null
@@ -1,51 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-// macro to read the FT0 DCS information from CCDB
-// default ts is very big: Saturday, November 20, 2286 5:46:39 PM
-
-#if !defined(__CLING__) || defined(__ROOTCLING__)
-#include "CCDB/CcdbApi.h"
-#include "DataFormatsFIT/DCSDPValues.h"
-#include "DetectorsDCS/DataPointIdentifier.h"
-#include "FT0Calibration/FT0DCSProcessor.h"
-
-#include 
-#include 
-#include 
-#endif
-
-void readFT0DCSentries(long ts = 9999999999000, const char* ccdb = "http://localhost:8080", const bool printEmpty = false)
-{
-
-  o2::ccdb::CcdbApi api;
-  api.init(ccdb); // or http://ccdb-test.cern.ch:8080
-  std::map metadata;
-  if (ts == 9999999999000) {
-    ts = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count();
-  }
-
-  std::unordered_map* m = api.retrieveFromTFileAny>("FT0/Calib/DCSDPs", metadata, ts);
-  std::cout << "size of map = " << m->size() << std::endl;
-  if (!printEmpty) {
-    std::cout << "Not printing DPs with no values" << std::endl;
-  }
-
-  for (auto& i : *m) {
-    if (i.second.values.empty() && !printEmpty) {
-      continue;
-    }
-    LOG(info) << "DPID = " << i.first;
-    i.second.print();
-  }
-
-  return;
-}
diff --git a/Detectors/FIT/FT0/calibration/src/EventsPerBcCalibrator.cxx b/Detectors/FIT/FT0/calibration/src/EventsPerBcCalibrator.cxx
new file mode 100644
index 0000000000000..b17c81213cd08
--- /dev/null
+++ b/Detectors/FIT/FT0/calibration/src/EventsPerBcCalibrator.cxx
@@ -0,0 +1,81 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "FT0Calibration/EventsPerBcCalibrator.h"
+#include "CommonUtils/MemFileHelper.h"
+
+namespace o2::ft0
+{
+void EventsPerBcContainer::print() const
+{
+  LOG(info) << entries << " entries";
+}
+
+void EventsPerBcContainer::fill(const o2::dataformats::TFIDInfo& ti, const gsl::span data)
+{
+  size_t oldEntries = entries;
+  for (const auto& digit : data) {
+    if (digit.mTriggers.getVertex() && digit.mTriggers.getAmplA() >= mMinAmplitudeSideA && digit.mTriggers.getAmplC() >= mMinAmplitudeSideC && (digit.mTriggers.getAmplA() + digit.mTriggers.getAmplC()) >= mMinSumOfAmplitude) {
+      mTvx[digit.mIntRecord.bc]++;
+      entries++;
+    }
+  }
+  LOG(debug) << "Container is filled with " << entries - oldEntries << " new events";
+}
+
+void EventsPerBcContainer::merge(const EventsPerBcContainer* prev)
+{
+  for (int bc = 0; bc < o2::constants::lhc::LHCMaxBunches; bc++) {
+    mTvx[bc] += prev->mTvx[bc];
+  }
+  entries += prev->entries;
+}
+
+void EventsPerBcCalibrator::initOutput()
+{
+  mTvxPerBcs.clear();
+  mTvxPerBcInfos.clear();
+}
+
+EventsPerBcCalibrator::EventsPerBcCalibrator(uint32_t minNumberOfEntries, int32_t minAmplitudeSideA, int32_t minAmplitudeSideC, int32_t minSumOfAmplitude) : mMinNumberOfEntries(minNumberOfEntries), mMinAmplitudeSideA(minAmplitudeSideA), mMinAmplitudeSideC(minAmplitudeSideC), mMinSumOfAmplitude(minSumOfAmplitude)
+{
+  LOG(info) << "Defined threshold for number of entires per slot: " << mMinNumberOfEntries;
+  LOG(info) << "Defined threshold for side A amplitude for event: " << mMinAmplitudeSideA;
+  LOG(info) << "Defined threshold for side C amplitude for event: " << mMinAmplitudeSideC;
+}
+
+bool EventsPerBcCalibrator::hasEnoughData(const EventsPerBcCalibrator::Slot& slot) const
+{
+  return slot.getContainer()->entries > mMinNumberOfEntries;
+}
+
+void EventsPerBcCalibrator::finalizeSlot(EventsPerBcCalibrator::Slot& slot)
+{
+  LOG(info) << "Finalizing slot from " << slot.getStartTimeMS() << " to " << slot.getEndTimeMS();
+  o2::ft0::EventsPerBcContainer* data = slot.getContainer();
+  mTvxPerBcs.emplace_back(data->mTvx);
+
+  auto clName = o2::utils::MemFileHelper::getClassName(mTvxPerBcs.back());
+  auto flName = o2::ccdb::CcdbApi::generateFileName(clName);
+
+  std::map metaData;
+  mTvxPerBcInfos.emplace_back(std::make_unique("FT0/Calib/EventsPerBc", clName, flName, metaData, slot.getStartTimeMS(), slot.getEndTimeMS()));
+  LOG(info) << "Created object valid from " << mTvxPerBcInfos.back()->getStartValidityTimestamp() << " to " << mTvxPerBcInfos.back()->getEndValidityTimestamp();
+}
+
+EventsPerBcCalibrator::Slot& EventsPerBcCalibrator::emplaceNewSlot(bool front, TFType tstart, TFType tend)
+{
+  auto& cont = getSlots();
+  auto& slot = front ? cont.emplace_front(tstart, tend) : cont.emplace_back(tstart, tend);
+  slot.setContainer(std::make_unique(mMinAmplitudeSideA, mMinAmplitudeSideC, mMinSumOfAmplitude));
+  return slot;
+}
+} // namespace o2::ft0
\ No newline at end of file
diff --git a/Detectors/FIT/FT0/calibration/src/FT0CalibCollector.cxx b/Detectors/FIT/FT0/calibration/src/FT0CalibCollector.cxx
deleted file mode 100644
index acff7a13ceabe..0000000000000
--- a/Detectors/FIT/FT0/calibration/src/FT0CalibCollector.cxx
+++ /dev/null
@@ -1,164 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-#include "FT0Calibration/FT0CalibCollector.h"
-#include "Framework/Logger.h"
-#include 
-#include 
-#include 
-#include 
-#include 
-
-namespace o2
-{
-namespace ft0
-{
-
-using Slot = o2::calibration::TimeSlot;
-
-//_____________________________________________
-void FT0CalibInfoSlot::fill(const gsl::span data)
-{
-  // fill container
-  // we first order the data that arrived, to improve speed when filling
-  int nd = data.size();
-  LOG(info) << "FT0CalibInfoSlot::fill entries in incoming data = " << nd;
-  std::vector ord(nd);
-  std::iota(ord.begin(), ord.end(), 0);
-  std::sort(ord.begin(), ord.end(), [&data](int i, int j) { return data[i].getChannelIndex() < data[j].getChannelIndex(); });
-  int chPrev = 0, offsPrev = 0;
-  for (int i = 0; i < nd; i++) {
-    if (std::abs(data[ord[i]].getTime()) > HISTO_RANGE) {
-      continue;
-    }
-    const auto& dti = data[ord[i]];
-    auto ch = dti.getChannelIndex();
-    auto offset = offsPrev;
-    if (ch > chPrev) {
-      offset += std::accumulate(mEntriesSlot.begin() + chPrev, mEntriesSlot.begin() + ch, 0);
-    }
-    offsPrev = offset;
-    chPrev = ch;
-    auto it = mFT0CollectedCalibInfoSlot.emplace(mFT0CollectedCalibInfoSlot.begin() + offset, data[ord[i]].getChannelIndex(), data[ord[i]].getTime(), data[ord[i]].getAmp(), data[ord[i]].getTimeStamp());
-    mEntriesSlot[ch]++;
-  }
-}
-//_____________________________________________
-void FT0CalibInfoSlot::merge(const FT0CalibInfoSlot* prev)
-{
-  // merge data of 2 slots
-
-  LOG(info) << "Merging two slots with entries: current slot -> " << mFT0CollectedCalibInfoSlot.size() << " , previous slot -> " << prev->mFT0CollectedCalibInfoSlot.size();
-
-  int offset = 0, offsetPrev = 0;
-  std::vector tmpVector;
-  for (int ch = 0; ch < NCHANNELS; ch++) {
-    if (mEntriesSlot[ch] != 0) {
-      for (int i = offset; i < offset + mEntriesSlot[ch]; i++) {
-        tmpVector.emplace_back(mFT0CollectedCalibInfoSlot[i]);
-      }
-      offset += mEntriesSlot[ch];
-    }
-    if (prev->mEntriesSlot[ch] != 0) {
-      for (int i = offsetPrev; i < offsetPrev + prev->mEntriesSlot[ch]; i++) {
-        tmpVector.emplace_back(prev->mFT0CollectedCalibInfoSlot[i]);
-      }
-      offsetPrev += prev->mEntriesSlot[ch];
-      mEntriesSlot[ch] += prev->mEntriesSlot[ch];
-    }
-  }
-  mFT0CollectedCalibInfoSlot.swap(tmpVector);
-  LOG(debug) << "After merging the size is " << mFT0CollectedCalibInfoSlot.size();
-  return;
-}
-//_____________________________________________
-void FT0CalibInfoSlot::print() const
-{
-  // to print number of entries in the tree and the channel with the max number of entries
-
-  LOG(info) << "Total number of entries " << mFT0CollectedCalibInfoSlot.size();
-  auto maxElementIndex = std::max_element(mEntriesSlot.begin(), mEntriesSlot.end());
-  auto channelIndex = std::distance(mEntriesSlot.begin(), maxElementIndex);
-  LOG(info) << "The maximum number of entries per channel in the current mFT0CollectedCalibInfo is " << *maxElementIndex << " for channel " << channelIndex;
-  return;
-}
-
-//_____________________________________________
-void FT0CalibInfoSlot::printEntries() const
-{
-  // to print number of entries in the tree and per channel
-
-  LOG(debug) << "Total number of entries " << mFT0CollectedCalibInfoSlot.size();
-  for (int i = 0; i < mEntriesSlot.size(); ++i) {
-    if (mEntriesSlot[i] != 0) {
-      LOG(info) << "channel " << i << " has " << mEntriesSlot[i] << " entries";
-    }
-  }
-  return;
-}
-
-//===================================================================
-
-//_____________________________________________
-void FT0CalibCollector::initOutput()
-{
-  // emptying the vectors
-
-  mFT0CollectedCalibInfo.clear();
-  for (int ch = 0; ch < NCHANNELS; ch++) {
-    mEntries[ch] = 0;
-  }
-
-  return;
-}
-
-//_____________________________________________
-bool FT0CalibCollector::hasEnoughData(const Slot& slot) const
-{
-  // We define that we have enough data if the tree is big enough.
-  // each FT0CalibrationInfoObject is composed of two int8 and one int16 --> 32 bytes
-  // E.g. supposing that we have 500000 entries per channel  --> 500 eneries per one amplitude bin
-  // we can check if we have  500000*o2::ft0::Geometry::NCHANNELS entries in the vector
-
-  const o2::ft0::FT0CalibInfoSlot* c = slot.getContainer();
-  LOG(info) << "we have " << c->getCollectedCalibInfoSlot().size() << " entries";
-  int maxNumberOfHits = mAbsMaxNumOfHits ? mMaxNumOfHits : mMaxNumOfHits * NCHANNELS;
-  if (mTFsendingPolicy || c->getCollectedCalibInfoSlot().size() > maxNumberOfHits) {
-    return true;
-  }
-  return false;
-}
-
-//_____________________________________________
-void FT0CalibCollector::finalizeSlot(Slot& slot)
-{
-
-  o2::ft0::FT0CalibInfoSlot* c = slot.getContainer();
-  mFT0CollectedCalibInfo = c->getCollectedCalibInfoSlot();
-  LOG(info) << "vector of  received with size = " << mFT0CollectedCalibInfo.size();
-  mEntries = c->getEntriesPerChannel();
-  return;
-}
-
-//_____________________________________________
-Slot& FT0CalibCollector::emplaceNewSlot(bool front, TFType tstart, TFType tend)
-{
-
-  LOG(info) << "FIT_CALIBRATOR_TYPE::emplaceNewSlot "
-            << " start " << tstart << " end " << tend;
-  auto& cont = getSlots();
-  auto& slot = front ? cont.emplace_front(tstart, tend) : cont.emplace_back(tstart, tend);
-  slot.setContainer(std::make_unique());
-  return slot;
-}
-
-} // end namespace ft0
-} // end namespace o2
diff --git a/Detectors/FIT/FT0/calibration/src/FT0CalibTimeSlewing.cxx b/Detectors/FIT/FT0/calibration/src/FT0CalibTimeSlewing.cxx
deleted file mode 100644
index 3ea4fcb788a99..0000000000000
--- a/Detectors/FIT/FT0/calibration/src/FT0CalibTimeSlewing.cxx
+++ /dev/null
@@ -1,122 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-/// \file FT0CalibTimeSlewing.cxx
-/// \brief Class for  slewing calibration object
-///
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include "FT0Calibration/FT0CalibTimeSlewing.h"
-
-using namespace o2::ft0;
-
-FT0CalibTimeSlewing::FT0CalibTimeSlewing()
-{
-  for (int iCh = 0; iCh < NCHANNELS; iCh++) {
-    mSigmaPeak[iCh] = -1.;
-    mTimeAmpHist[iCh] = new TH2F(Form("hTimeAmpHist%d", iCh), Form("TimeAmp%d", iCh),
-                                 NUMBER_OF_HISTOGRAM_BINS_X, 0, HISTOGRAM_RANGE_X,
-                                 NUMBER_OF_HISTOGRAM_BINS_Y, -HISTOGRAM_RANGE_Y, HISTOGRAM_RANGE_Y);
-  }
-}
-
-//______________________________________________
-float FT0CalibTimeSlewing::getChannelOffset(int channel, int amplitude) const
-{
-  return mTimeSlewing[channel].Eval(amplitude);
-}
-
-//______________________________________________
-void FT0CalibTimeSlewing::fillGraph(int channel, TH2F* histo)
-{
-  LOG(info) << "FT0CalibTimeSlewing::fillGraph " << channel << " entries " << int(histo->GetEntries());
-  double shiftchannel = 0;
-  TH1D* hist_Proj = histo->ProjectionY();
-  TFitResultPtr res = hist_Proj->Fit("gaus", "SQ");
-  if ((Int_t)res == 0) {
-    shiftchannel = res->Parameter(1);
-  }
-  Double_t xgr[NUMBER_OF_HISTOGRAM_BINS_X] = {};
-  Double_t ygr[NUMBER_OF_HISTOGRAM_BINS_X] = {};
-  TH1D* proj = nullptr;
-  int nbins = 0;
-  for (int ibin = 1; ibin < NUMBER_OF_HISTOGRAM_BINS_X; ibin++) {
-    xgr[ibin] = histo->GetXaxis()->GetBinCenter(ibin);
-    proj = histo->ProjectionY(Form("proj_px%i", ibin), ibin, ibin + 1);
-    if (proj->GetEntries() < 500) {
-      ygr[ibin] = 0;
-      continue;
-    }
-    TFitResultPtr r = proj->Fit("gaus", "SQ");
-    if ((Int_t)r == 0) {
-      ygr[ibin] = r->Parameter(1) - shiftchannel;
-    }
-    nbins++;
-    LOG(info) << "channel " << channel << " bin " << ibin << " x " << xgr[ibin] << " y " << ygr[ibin] << " ent " << proj->GetEntries() << " sigma " << r->Parameter(2) << " shiftchannel " << shiftchannel;
-  }
-  TGraph* grTimeAmp = new TGraph(nbins + 5, xgr, ygr);
-  mTimeSlewing[channel] = *grTimeAmp;
-}
-//______________________________________________
-FT0CalibTimeSlewing& FT0CalibTimeSlewing::operator+=(const FT0CalibTimeSlewing& other)
-{
-  for (int i = 0; i < NCHANNELS; i++) {
-    mTimeSlewing[i] = other.mTimeSlewing[i];
-    mSigmaPeak[i] = other.mSigmaPeak[i];
-  }
-  return *this;
-}
-
-//______________________________________________
-void FT0CalibTimeSlewing::mergeFilesWithTree()
-{
-  TFileMerger merger;
-  merger.OutputFile(mMergedFileName.c_str());
-  for (Int_t i = 0; i < mNfiles; i++) {
-    TFile* file =
-      TFile::Open(Form("%s_%d.root", mSingleFileName.c_str(), i));
-    if (file) {
-      merger.AddAdoptFile(file);
-    }
-  }
-  if (!merger.Merge()) {
-    LOG(fatal) << "Could not merge files";
-  }
-  TFile mMergedFile{merger.GetOutputFileName()};
-  TTree* tr = (TTree*)mMergedFile.Get("treeCollectedCalibInfo");
-  if (!tr) {
-    LOG(fatal) << "Could not get tree with calib info";
-  }
-  fillHistos(tr);
-}
-
-//______________________________________________
-void FT0CalibTimeSlewing::fillHistos(TTree* tr)
-{
-  std::vector* localCalibInfoFT0 = nullptr;
-  if (!tr->GetBranch("FT0CollectedCalibInfo")) {
-    LOG(fatal) << "Did not find collected FT0 calib info branch  in the input tree";
-  }
-  tr->SetBranchAddress("FT0CollectedCalibInfo", &localCalibInfoFT0);
-  for (Int_t ievent = 0; ievent < tr->GetEntries(); ievent++) {
-    tr->GetEvent(ievent);
-    for (auto& info : *localCalibInfoFT0) {
-      LOG(debug) << " ch " << int(info.getChannelIndex()) << " time " << info.getTime() << " amp " << info.getAmp();
-      int iCh = info.getChannelIndex();
-      mTimeAmpHist[iCh]->Fill(info.getAmp(), info.getTime());
-    }
-  }
-  delete localCalibInfoFT0;
-}
diff --git a/Detectors/FIT/FT0/calibration/src/FT0CalibrationLinkDef.h b/Detectors/FIT/FT0/calibration/src/FT0CalibrationLinkDef.h
index 1e34f6b86db76..11b1ce25e9353 100644
--- a/Detectors/FIT/FT0/calibration/src/FT0CalibrationLinkDef.h
+++ b/Detectors/FIT/FT0/calibration/src/FT0CalibrationLinkDef.h
@@ -15,19 +15,10 @@
 #pragma link off all classes;
 #pragma link off all functions;
 
-#pragma link C++ class o2::ft0::CalibParam + ;
-#pragma link C++ class o2::ft0::FT0ChannelTimeOffsetSlotContainer + ;
 #pragma link C++ class o2::ft0::FT0TimeOffsetSlotContainer + ;
-#pragma link C++ class o2::ft0::GlobalOffsetsContainer + ;
-#pragma link C++ class o2::ft0::FT0CalibTimeSlewing + ;
-#pragma link C++ class o2::ft0::FT0CalibCollector + ;
-#pragma link C++ class o2::calibration::TimeSlotCalibration < o2::ft0::FT0CalibrationInfoObject, o2::ft0::FT0CalibInfoSlot>;
-#pragma link C++ class o2::calibration::TimeSlotCalibration < float, o2::ft0::FT0TimeOffsetSlotContainer>;
-#pragma link C++ class o2::calibration::TimeSlotCalibration < o2::ft0::FT0CalibrationInfoObject, o2::ft0::FT0ChannelTimeOffsetSlotContainer>;
-#pragma link C++ class o2::calibration::TimeSlotCalibration < o2::ft0::GlobalOffsetsInfoObject, o2::ft0::GlobalOffsetsContainer>;
-
-#pragma link C++ std::vector < std::pair < uint64_t, int>> + ;
-#pragma link C++ struct o2::fit::DCSDPValues + ;
-#pragma link C++ class std::unordered_map < o2::dcs::DataPointIdentifier, o2::fit::DCSDPValues> + ;
-
+#pragma link C++ class o2::ft0::EventsPerBcCalibrator + ;
+#pragma link C++ class o2::calibration::TimeSlot < o2::ft0::FT0TimeOffsetSlotContainer>;
+#pragma link C++ class o2::calibration::TimeSlotCalibration < o2::ft0::FT0TimeOffsetSlotContainer>;
+#pragma link C++ class o2::calibration::TimeSlot < o2::ft0::EventsPerBcContainer> + ;
+#pragma link C++ class o2::calibration::TimeSlotCalibration < o2::ft0::EventsPerBcContainer> + ;
 #endif
diff --git a/Detectors/FIT/FT0/calibration/src/FT0ChannelTimeOffsetSlotContainer.cxx b/Detectors/FIT/FT0/calibration/src/FT0ChannelTimeOffsetSlotContainer.cxx
deleted file mode 100644
index 379e25eeb9453..0000000000000
--- a/Detectors/FIT/FT0/calibration/src/FT0ChannelTimeOffsetSlotContainer.cxx
+++ /dev/null
@@ -1,115 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-#include "FT0Calibration/FT0ChannelTimeOffsetSlotContainer.h"
-#include "FT0Base/Geometry.h"
-#include "FT0Base/FT0DigParam.h"
-#include 
-#include 
-#include 
-#include 
-#include "MathUtils/fit.h"
-#include "TH1.h"
-#include "TFitResult.h"
-
-using namespace o2::ft0;
-
-int FT0ChannelTimeOffsetSlotContainer::sGausFitBins = 999; // NOT USED
-
-FT0ChannelTimeOffsetSlotContainer::FT0ChannelTimeOffsetSlotContainer(std::size_t minEntries)
-  : mMinEntries(minEntries)
-{
-  for (int ich = 0; ich < NCHANNELS; ++ich) {
-    mHistogram[ich].reset(new TH1F(Form("hTime%i", ich), "time", NUMBER_OF_HISTOGRAM_BINS, -HISTOGRAM_RANGE, HISTOGRAM_RANGE));
-  }
-}
-
-FT0ChannelTimeOffsetSlotContainer::FT0ChannelTimeOffsetSlotContainer(FT0ChannelTimeOffsetSlotContainer const& other)
-  : mMinEntries(other.mMinEntries)
-{
-  for (int ich = 0; ich < NCHANNELS; ++ich) {
-    mHistogram[ich].reset(new TH1F(*other.mHistogram[ich]));
-  }
-}
-
-FT0ChannelTimeOffsetSlotContainer& FT0ChannelTimeOffsetSlotContainer::operator=(FT0ChannelTimeOffsetSlotContainer const& other)
-{
-  mMinEntries = other.mMinEntries;
-  for (int ich = 0; ich < NCHANNELS; ++ich) {
-    mHistogram[ich].reset(new TH1F(*other.mHistogram[ich]));
-  }
-  return *this;
-}
-
-bool FT0ChannelTimeOffsetSlotContainer::hasEnoughEntries() const
-{
-  return *std::min_element(mEntriesPerChannel.begin(), mEntriesPerChannel.end()) > mMinEntries;
-}
-void FT0ChannelTimeOffsetSlotContainer::fill(const gsl::span& data)
-{
-  for (auto& entry : data) {
-    updateFirstCreation(entry.getTimeStamp());
-    const auto chID = entry.getChannelIndex();
-    const auto chTime = entry.getTime();
-    if (chID < NCHANNELS && std::abs(chTime) < o2::ft0::FT0DigParam::mTime_trg_gate && entry.getAmp() > o2::ft0::FT0DigParam::mAmpThresholdForReco) {
-      mHistogram[chID]->Fill(chTime);
-      ++mEntriesPerChannel[chID];
-      LOG(debug) << "@@@@entries " << mEntriesPerChannel[chID] << " chID " << int(chID) << " time " << chTime << " tiestamp " << uint64_t(entry.getTimeStamp());
-    }
-  }
-}
-
-void FT0ChannelTimeOffsetSlotContainer::merge(FT0ChannelTimeOffsetSlotContainer* prev)
-{
-  for (unsigned int iCh = 0; iCh < NCHANNELS; ++iCh) {
-    mHistogram[iCh]->Add(prev->mHistogram[iCh].get(), 1);
-    mEntriesPerChannel[iCh] += prev->mEntriesPerChannel[iCh];
-    LOG(debug) << " entries " << mEntriesPerChannel[iCh] << " " << prev->mEntriesPerChannel[iCh];
-  }
-  mFirstCreation = std::min(mFirstCreation, prev->mFirstCreation);
-}
-
-int16_t FT0ChannelTimeOffsetSlotContainer::getMeanGaussianFitValue(std::size_t channelID) const
-{
-
-  if (0 == mEntriesPerChannel[channelID]) {
-    return 0;
-  }
-  LOG(debug) << " for channel " << int(channelID) << " entries " << mEntriesPerChannel[channelID] << " hist entries " << mHistogram[channelID]->GetEntries() << " mean " << mHistogram[channelID]->GetMean() << " RMS " << mHistogram[channelID]->GetRMS();
-
-  int outputGaussianFitValues = -99999;
-  int sigma;
-  TFitResultPtr r = mHistogram[channelID]->Fit("gaus", "0SQ", "", -sGausFitBins, sGausFitBins);
-  if ((Int_t)r == 0) {
-    outputGaussianFitValues = int(r->Parameters()[1]);
-    sigma = int(r->Parameters()[2]);
-  };
-  //  if (r != 0 || std::abs(outputGaussianFitValues - mHistogram[channelID]->GetMean()) > 20 || mHistogram[channelID]->GetRMS() < 3 || sigma > 30) {
-  if (r != 0 || std::abs(outputGaussianFitValues - mHistogram[channelID]->GetMean()) > 20 || mHistogram[channelID]->GetRMS() < 1 || sigma > 30) { // to be used fot test with laser
-    LOG(info) << "!!! Bad gauss fit " << outputGaussianFitValues << " sigma " << sigma << " mean " << mHistogram[channelID]->GetMean() << " RMS " << mHistogram[channelID]->GetRMS();
-    outputGaussianFitValues = mHistogram[channelID]->GetMean();
-  }
-
-  return static_cast(outputGaussianFitValues);
-}
-FT0ChannelTimeCalibrationObject FT0ChannelTimeOffsetSlotContainer::generateCalibrationObject() const
-{
-  FT0ChannelTimeCalibrationObject calibrationObject;
-  for (unsigned int iCh = 0; iCh < o2::ft0::Geometry::Nchannels; ++iCh) {
-    calibrationObject.mTimeOffsets[iCh] = getMeanGaussianFitValue(iCh);
-  }
-  return calibrationObject;
-}
-
-void FT0ChannelTimeOffsetSlotContainer::print() const
-{
-  // QC will do that part
-}
diff --git a/Detectors/FIT/FT0/calibration/src/FT0DCSProcessor.cxx b/Detectors/FIT/FT0/calibration/src/FT0DCSProcessor.cxx
deleted file mode 100644
index 677e3cf8c0493..0000000000000
--- a/Detectors/FIT/FT0/calibration/src/FT0DCSProcessor.cxx
+++ /dev/null
@@ -1,184 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-#include 
-#include "DetectorsCalibration/Utils.h"
-#include "Rtypes.h"
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-
-using namespace o2::ft0;
-using namespace o2::dcs;
-
-using DeliveryType = o2::dcs::DeliveryType;
-using DPID = o2::dcs::DataPointIdentifier;
-using DPVAL = o2::dcs::DataPointValue;
-
-void FT0DCSProcessor::init(const std::vector& pids)
-{
-  // fill the array of the DPIDs that will be used by FT0
-  // pids should be provided by CCDB
-
-  for (const auto& it : pids) {
-    mPids[it] = false;
-    mFT0DCS[it].makeEmpty();
-  }
-}
-
-int FT0DCSProcessor::process(const gsl::span dps)
-{
-  // first we check which DPs are missing - if some are, it means that
-  // the delta map was sent
-
-  if (mVerbose) {
-    LOG(info) << "\n\nProcessing new DCS DP map\n-------------------------";
-  }
-
-  if (false) {
-    std::unordered_map mapin;
-    for (auto& it : dps) {
-      mapin[it.id] = it.data;
-    }
-    for (auto& it : mPids) {
-      const auto& el = mapin.find(it.first);
-      if (el == mapin.end()) {
-        LOG(debug) << "DP " << it.first << " not found in map";
-      } else {
-        LOG(debug) << "DP " << it.first << " found in map";
-      }
-    }
-  }
-
-  // now we process all DPs, one by one
-  for (const auto& it : dps) {
-    // we process only the DPs defined in the configuration
-    const auto& el = mPids.find(it.id);
-    if (el == mPids.end()) {
-      LOG(info) << "DP " << it.id << " not found in FT0DCSProcessor, we will not process it";
-      continue;
-    }
-    processDP(it);
-    mPids[it.id] = true;
-  }
-
-  return 0;
-}
-
-int FT0DCSProcessor::processDP(const DPCOM& dpcom)
-{
-  // processing a single DP
-
-  const auto& dpid = dpcom.id;
-  const auto& type = dpid.get_type();
-  const auto& val = dpcom.data;
-  if (mVerbose) {
-    if (type == DPVAL_DOUBLE) {
-      LOG(info) << "Processing DP = " << dpcom << " (epoch " << val.get_epoch_time() << "), with value = " << o2::dcs::getValue(dpcom);
-    } else if (type == DPVAL_UINT) {
-      LOG(info) << "Processing DP = " << dpcom << " (epoch " << val.get_epoch_time() << "), with value = " << o2::dcs::getValue(dpcom);
-    }
-  }
-  auto flags = val.get_flags();
-  if (processFlags(flags, dpid.get_alias()) == 0) {
-    // Store all DP values
-    if (mFT0DCS[dpid].values.empty() || val.get_epoch_time() > mFT0DCS[dpid].values.back().first) {
-      converter.raw_data = val.payload_pt1;
-      if (type == DPVAL_DOUBLE) {
-        mFT0DCS[dpid].add(val.get_epoch_time(), lround(converter.double_value * 1000)); // store as nA
-      } else if (type == DPVAL_UINT) {
-        mFT0DCS[dpid].add(val.get_epoch_time(), converter.uint_value);
-      }
-    }
-  }
-  return 0;
-}
-
-uint64_t FT0DCSProcessor::processFlags(const uint64_t flags, const char* pid)
-{
-  // function to process the flag. the return code zero means that all is fine.
-  // anything else means that there was an issue
-
-  // for now, I don't know how to use the flags, so I do nothing
-
-  if (flags & DataPointValue::KEEP_ALIVE_FLAG) {
-    LOG(debug) << "KEEP_ALIVE_FLAG active for DP " << pid;
-  }
-  if (flags & DataPointValue::END_FLAG) {
-    LOG(debug) << "END_FLAG active for DP " << pid;
-  }
-  if (flags & DataPointValue::FBI_FLAG) {
-    LOG(debug) << "FBI_FLAG active for DP " << pid;
-  }
-  if (flags & DataPointValue::NEW_FLAG) {
-    LOG(debug) << "NEW_FLAG active for DP " << pid;
-  }
-  if (flags & DataPointValue::DIRTY_FLAG) {
-    LOG(debug) << "DIRTY_FLAG active for DP " << pid;
-  }
-  if (flags & DataPointValue::TURN_FLAG) {
-    LOG(debug) << "TURN_FLAG active for DP " << pid;
-  }
-  if (flags & DataPointValue::WRITE_FLAG) {
-    LOG(debug) << "WRITE_FLAG active for DP " << pid;
-  }
-  if (flags & DataPointValue::READ_FLAG) {
-    LOG(debug) << "READ_FLAG active for DP " << pid;
-  }
-  if (flags & DataPointValue::OVERWRITE_FLAG) {
-    LOG(debug) << "OVERWRITE_FLAG active for DP " << pid;
-  }
-  if (flags & DataPointValue::VICTIM_FLAG) {
-    LOG(debug) << "VICTIM_FLAG active for DP " << pid;
-  }
-  if (flags & DataPointValue::DIM_ERROR_FLAG) {
-    LOG(debug) << "DIM_ERROR_FLAG active for DP " << pid;
-  }
-  if (flags & DataPointValue::BAD_DPID_FLAG) {
-    LOG(debug) << "BAD_DPID_FLAG active for DP " << pid;
-  }
-  if (flags & DataPointValue::BAD_FLAGS_FLAG) {
-    LOG(debug) << "BAD_FLAGS_FLAG active for DP " << pid;
-  }
-  if (flags & DataPointValue::BAD_TIMESTAMP_FLAG) {
-    LOG(debug) << "BAD_TIMESTAMP_FLAG active for DP " << pid;
-  }
-  if (flags & DataPointValue::BAD_PAYLOAD_FLAG) {
-    LOG(debug) << "BAD_PAYLOAD_FLAG active for DP " << pid;
-  }
-  if (flags & DataPointValue::BAD_FBI_FLAG) {
-    LOG(debug) << "BAD_FBI_FLAG active for DP " << pid;
-  }
-
-  return 0;
-}
-
-void FT0DCSProcessor::updateDPsCCDB()
-{
-  // Prepare the object to be sent to CCDB
-  if (mVerbose) {
-    for (auto& dp : mFT0DCS) {
-      // if (dp.second.values.empty()) {
-      //   continue;
-      // }
-      LOG(info) << "PID = " << dp.first.get_alias();
-      dp.second.print();
-    }
-  }
-
-  std::map md;
-  o2::calibration::Utils::prepareCCDBobjectInfo(mFT0DCS, mccdbDPsInfo, "FT0/Calib/DCSDPs", md, mStartValidity, mStartValidity + 3 * o2::ccdb::CcdbObjectInfo::DAY);
-
-  return;
-}
\ No newline at end of file
diff --git a/Detectors/FIT/FT0/calibration/src/FT0TimeOffsetSlotContainer.cxx b/Detectors/FIT/FT0/calibration/src/FT0TimeOffsetSlotContainer.cxx
index 69440c1e8c10f..1e31d24f8c20d 100644
--- a/Detectors/FIT/FT0/calibration/src/FT0TimeOffsetSlotContainer.cxx
+++ b/Detectors/FIT/FT0/calibration/src/FT0TimeOffsetSlotContainer.cxx
@@ -10,12 +10,13 @@
 // or submit itself to any jurisdiction.
 
 #include "FT0Calibration/FT0TimeOffsetSlotContainer.h"
-#include "FT0Calibration/CalibParam.h"
+#include "DataFormatsFT0/CalibParam.h"
 #include "CommonDataFormat/FlatHisto1D.h"
 
 #include 
 
 #include "TH1.h"
+#include "TFile.h"
 #include "TFitResult.h"
 
 using namespace o2::ft0;
@@ -24,21 +25,26 @@ FT0TimeOffsetSlotContainer::FT0TimeOffsetSlotContainer(std::size_t minEntries) {
 
 bool FT0TimeOffsetSlotContainer::hasEnoughEntries() const
 {
-  if (mIsReady) {
+  if (mTotalNevents == 0) {
+    // Dummy slot, should be ignored for protection
+    LOG(warning) << "RESULT: Empty slot, ignoring";
+    return false;
+  } else if (mIsReady) {
     // ready : bad+good == NChannels (i.e. no pending channel)
     LOG(info) << "RESULT: ready";
     print();
     return true;
-  } else if (mCurrentSlot > CalibParam::Instance().mNExtraSlots) {
-    LOG(info) << "RESULT: Extra slots are used";
+  } else if (mCurrentSlot >= CalibParam::Instance().mNExtraSlots) {
+    LOG(info) << "RESULT: Extra slots(" << CalibParam::Instance().mNExtraSlots << ") are used";
     print();
     return true;
-  } else if (mCurrentSlot == 0) {
+  } else if (mCurrentSlot < CalibParam::Instance().mNExtraSlots) {
     for (int iCh = 0; iCh < sNCHANNELS; iCh++) {
       const auto nEntries = mArrEntries[iCh];
       if (nEntries >= CalibParam::Instance().mMinEntriesThreshold && nEntries < CalibParam::Instance().mMaxEntriesThreshold) {
         // Check if there are any pending channel in first slot
         LOG(info) << "RESULT: pending channels";
+        print();
         return false;
       }
     }
@@ -47,10 +53,10 @@ bool FT0TimeOffsetSlotContainer::hasEnoughEntries() const
     print();
     return true;
   } else {
-    // Probably will never happen, all other conditions are already checked
+    // Probably will be never happen, all other conditions are already checked
     LOG(info) << "RESULT: should be never happen";
     print();
-    return false;
+    return true;
   }
 }
 
@@ -64,33 +70,40 @@ void FT0TimeOffsetSlotContainer::fill(const gsl::span& data)
     mIsFirstTF = false;
   }
   mHistogram.add(histView);
-  if (!mIsReady) {
-    // This part should at the stage `hasEnoughData()` but it is const method
-    for (int iCh = 0; iCh < sNCHANNELS; iCh++) {
-      if (mBitsetGoodChIDs.test(iCh) || mBitsetBadChIDs.test(iCh)) {
-        // No need in checking entries at channels with enough data or at channels which marked as bad in first slot
-        continue;
-      }
-      auto sliceChID = mHistogram.getSliceY(iCh);
-      FlatHistoValue_t nEntries{};
-      for (auto& en : sliceChID) {
-        nEntries += en;
-      }
-      mArrEntries[iCh] = nEntries;
-      if (nEntries >= CalibParam::Instance().mMaxEntriesThreshold) {
-        mBitsetGoodChIDs.set(iCh);
-      }
+  // This part should at the stage `hasEnoughData()` but it is const method
+  for (int iCh = 0; iCh < sNCHANNELS; iCh++) {
+    if (mBitsetGoodChIDs.test(iCh) || mBitsetBadChIDs.test(iCh)) {
+      // No need in checking entries at channels with enough data or at channels which marked as bad in first slot
+      continue;
     }
-    const auto totalNCheckedChIDs = mBitsetGoodChIDs.count() + mBitsetBadChIDs.count();
-    if (totalNCheckedChIDs == sNCHANNELS) {
-      mIsReady = true;
+    auto sliceChID = mHistogram.getSliceY(iCh);
+    FlatHistoValue_t nEntries{};
+    for (auto& en : sliceChID) {
+      nEntries += en;
     }
+    mArrEntries[iCh] = nEntries;
+    mTotalNevents += nEntries;
+    if (nEntries >= CalibParam::Instance().mMaxEntriesThreshold) {
+      mBitsetGoodChIDs.set(iCh);
+    }
+  }
+  const auto totalNCheckedChIDs = mBitsetGoodChIDs.count() + mBitsetBadChIDs.count();
+  if (totalNCheckedChIDs == sNCHANNELS) {
+    mIsReady = true;
   }
 }
 
 void FT0TimeOffsetSlotContainer::merge(FT0TimeOffsetSlotContainer* prev)
 {
   LOG(info) << "MERGING";
+  if (mIsFirstTF && prev->isFirstTF()) {
+    // nothing to be done
+    return;
+  } else if (mIsFirstTF && !prev->isFirstTF()) {
+    // need to make mHistogram operational first
+    mHistogram.init(prev->getHistogram().getNBinsX(), prev->getHistogram().getXMin(), prev->getHistogram().getXMax(), prev->getHistogram().getNBinsY(), prev->getHistogram().getYMin(), prev->getHistogram().getYMax());
+    mIsFirstTF = false;
+  }
   *this = *prev;
   if (mCurrentSlot == 0) {
     // This part should at the stage `hasEnoughData()` but it is const method
@@ -105,15 +118,20 @@ void FT0TimeOffsetSlotContainer::merge(FT0TimeOffsetSlotContainer* prev)
   mCurrentSlot++;
 }
 
-int16_t FT0TimeOffsetSlotContainer::getMeanGaussianFitValue(std::size_t channelID) const
+SpectraInfoObject FT0TimeOffsetSlotContainer::getSpectraInfoObject(std::size_t channelID, TList* listHists) const
 {
-  double meanGaus{0};
-  double sigmaGaus{0};
+  uint32_t statusBits{};
   double minFitRange{0};
   double maxFitRange{0};
   auto hist = mHistogram.createSliceYTH1F(channelID);
-  const auto meanHist = hist->GetMean();
-  const auto rmsHist = hist->GetRMS();
+  if (channelID < sNCHANNELS) {
+    if (CalibParam::Instance().mRebinFactorPerChID[channelID] > 0) {
+      hist->Rebin(CalibParam::Instance().mRebinFactorPerChID[channelID]);
+    }
+  }
+  const float meanHist = hist->GetMean();
+  const float rmsHist = hist->GetRMS();
+  const float stat = hist->Integral();
   if (CalibParam::Instance().mUseDynamicRange) {
     minFitRange = meanHist - CalibParam::Instance().mRangeInRMS * rmsHist;
     maxFitRange = meanHist + CalibParam::Instance().mRangeInRMS * rmsHist;
@@ -121,28 +139,57 @@ int16_t FT0TimeOffsetSlotContainer::getMeanGaussianFitValue(std::size_t channelI
     minFitRange = CalibParam::Instance().mMinFitRange;
     maxFitRange = CalibParam::Instance().mMaxFitRange;
   }
-  TFitResultPtr resultFit = hist->Fit("gaus", "0SQ", "", minFitRange, maxFitRange);
-  if (((int)resultFit) == 0) {
-    meanGaus = resultFit->Parameters()[1];
-    sigmaGaus = resultFit->Parameters()[2];
+  float constantGaus{};
+  float meanGaus{};
+  float sigmaGaus{};
+  float fitChi2{};
+  if (stat > 0) {
+    TFitResultPtr resultFit = hist->Fit("gaus", "0SQ", "", minFitRange, maxFitRange);
+    if (((int)resultFit) == 0) {
+      constantGaus = resultFit->Parameters()[0];
+      meanGaus = resultFit->Parameters()[1];
+      sigmaGaus = resultFit->Parameters()[2];
+      fitChi2 = resultFit->Chi2();
+      statusBits |= (1 << 0);
+    }
+    if (((int)resultFit) != 0 || std::abs(meanGaus - meanHist) > CalibParam::Instance().mMaxDiffMean || rmsHist < CalibParam::Instance().mMinRMS || sigmaGaus > CalibParam::Instance().mMaxSigma) {
+      statusBits |= (2 << 0);
+      LOG(debug) << "Bad gaus fit: meanGaus " << meanGaus << " sigmaGaus " << sigmaGaus << " meanHist " << meanHist << " rmsHist " << rmsHist << "resultFit " << ((int)resultFit);
+    }
   }
-  if (((int)resultFit) != 0 || std::abs(meanGaus - meanHist) > CalibParam::Instance().mMaxDiffMean || rmsHist < CalibParam::Instance().mMinRMS || sigmaGaus > CalibParam::Instance().mMaxSigma) {
-    LOG(debug) << "Bad gaus fit: meanGaus " << meanGaus << " sigmaGaus " << sigmaGaus << " meanHist " << meanHist << " rmsHist " << rmsHist << "resultFit " << ((int)resultFit);
-    meanGaus = meanHist;
+  if (listHists != nullptr) {
+    auto histPtr = hist.release();
+    const std::string histName = "histCh" + std::to_string(channelID);
+    histPtr->SetName(histName.c_str());
+    listHists->Add(histPtr);
   }
-  return static_cast(meanGaus);
+  return SpectraInfoObject{meanGaus, sigmaGaus, constantGaus, fitChi2, meanHist, rmsHist, stat, statusBits};
 }
 
-FT0ChannelTimeCalibrationObject FT0TimeOffsetSlotContainer::generateCalibrationObject() const
+TimeSpectraInfoObject FT0TimeOffsetSlotContainer::generateCalibrationObject(long tsStartMS, long tsEndMS, const std::string& extraInfo) const
 {
-  FT0ChannelTimeCalibrationObject calibrationObject;
+  TList* listHists = nullptr;
+  bool storeHists{false};
+  if (extraInfo.size() > 0) {
+    storeHists = true;
+    listHists = new TList();
+    listHists->SetOwner(true);
+    listHists->SetName("output");
+  }
+  TimeSpectraInfoObject calibrationObject;
   for (unsigned int iCh = 0; iCh < sNCHANNELS; ++iCh) {
-    if (mBitsetGoodChIDs.test(iCh)) {
-      calibrationObject.mTimeOffsets[iCh] = getMeanGaussianFitValue(iCh);
-    } else {
-      // If channel is bad, set zero offset(or use histogram mean?). Later will be hidden value for tagging as bad channel
-      calibrationObject.mTimeOffsets[iCh] = 0;
-    }
+    calibrationObject.mTime[iCh] = getSpectraInfoObject(iCh, listHists);
+  }
+  calibrationObject.mTimeA = getSpectraInfoObject(sNCHANNELS, listHists);
+  calibrationObject.mTimeC = getSpectraInfoObject(sNCHANNELS + 1, listHists);
+  calibrationObject.mSumTimeAC = getSpectraInfoObject(sNCHANNELS + 2, listHists);
+  calibrationObject.mDiffTimeCA = getSpectraInfoObject(sNCHANNELS + 3, listHists);
+  if (storeHists) {
+    const std::string filename = extraInfo + "/histsTimeSpectra" + std::to_string(tsStartMS) + "_" + std::to_string(tsEndMS) + ".root";
+    TFile fileHists(filename.c_str(), "RECREATE");
+    fileHists.WriteObject(listHists, listHists->GetName(), "SingleKey");
+    fileHists.Close();
+    delete listHists;
   }
   return calibrationObject;
 }
@@ -157,4 +204,4 @@ void FT0TimeOffsetSlotContainer::print() const
   LOG(info) << "mIsFirstTF " << mIsFirstTF;
   LOG(info) << "mIsReady " << mIsReady;
   // QC will do that part
-}
\ No newline at end of file
+}
diff --git a/Detectors/FIT/FT0/calibration/src/GlobalOffsetsContainer.cxx b/Detectors/FIT/FT0/calibration/src/GlobalOffsetsContainer.cxx
deleted file mode 100644
index 538f1088d109a..0000000000000
--- a/Detectors/FIT/FT0/calibration/src/GlobalOffsetsContainer.cxx
+++ /dev/null
@@ -1,91 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-#include "Framework/Logger.h"
-#include "MathUtils/fit.h"
-#include "CommonUtils/MemFileHelper.h"
-#include "CCDB/CcdbApi.h"
-#include "DetectorsCalibration/Utils.h"
-#include "FT0Calibration//GlobalOffsetsContainer.h"
-#include "DataFormatsFT0/GlobalOffsetsInfoObject.h"
-#include 
-#include 
-#include "MathUtils/fit.h"
-#include 
-#include 
-
-using namespace o2::ft0;
-using o2::math_utils::fitGaus;
-
-bool GlobalOffsetsContainer::hasEnoughEntries() const
-{
-  return mEntries < mMinEntries;
-}
-void GlobalOffsetsContainer::fill(const gsl::span& data)
-{
-  // fill container
-  for (auto& entry : data) {
-    if (std::abs(entry.getT0AC()) < RANGE) {
-      updateFirstCreation(entry.getTimeStamp());
-      auto time = entry.getT0AC();
-      time += RANGE;
-      mHisto[(time)]++;
-      mEntries++;
-    }
-  }
-}
-void GlobalOffsetsContainer::merge(GlobalOffsetsContainer* prev)
-{
-  // merge data of 2 slots
-  for (int i = mHisto.size(); i--;) {
-    mHisto[i] += prev->mHisto[i];
-  }
-  mEntries += prev->mEntries;
-  mFirstCreation = std::min(mFirstCreation, prev->mFirstCreation);
-}
-
-int GlobalOffsetsContainer::getMeanGaussianFitValue() const
-{
-  if (!hasEnoughEntries()) {
-    return 0;
-  }
-  if (mEntries == 0) {
-    return 0;
-  }
-  float sum = 0;
-  for (int ic = 0; ic < NBINS; ic++) {
-    sum += float(ic - RANGE) * float(mHisto[ic]);
-    //    LOG(info)<<" histo "< fitValues;
-  double fitres = fitGaus(NBINS, mHisto.data(), -RANGE, RANGE, fitValues);
-  if (fitres >= 0) {
-    LOG(info) << "Fit result " << fitres << " Mean = " << fitValues[1] << " Sigma = " << fitValues[2] << " size " << mEntries << " mean " << sum / mEntries;
-    return (static_cast(std::round(fitValues[1])));
-  } else {
-    LOG(warning) << "Fit failed with result = " << fitres;
-    return 0;
-  }
-}
-GlobalOffsetsCalibrationObject GlobalOffsetsContainer::generateCalibrationObject() const
-{
-  GlobalOffsetsCalibrationObject calibrationObject;
-  calibrationObject.mCollisionTimeOffsets = getMeanGaussianFitValue();
-  LOG(info) << "GlobalOffsetsCalibrationObjectAlgorithm generate CalibrationObject for T0"
-            << " = " << calibrationObject.mCollisionTimeOffsets;
-  return calibrationObject;
-}
-
-void GlobalOffsetsContainer::print() const
-{
-  LOG(info) << "Container keep data for LHC phase calibration:";
-  LOG(info) << "Gaussian mean time AC side " << getMeanGaussianFitValue() << " based on" << mEntries << " entries";
-}
diff --git a/Detectors/FIT/FT0/calibration/src/RecoCalibInfoWorkflow.cxx b/Detectors/FIT/FT0/calibration/src/RecoCalibInfoWorkflow.cxx
deleted file mode 100644
index e0fa8b13a1bf9..0000000000000
--- a/Detectors/FIT/FT0/calibration/src/RecoCalibInfoWorkflow.cxx
+++ /dev/null
@@ -1,110 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-/// \file   RecoCalibInfoWorkflow.cxx
-///\ brief  Collect data for global offsets calibration
-/// \author Alla.Maevskaya@cern.ch
-
-#include 
-#include 
-#include "Framework/DeviceSpec.h"
-#include "Framework/WorkflowSpec.h"
-#include "Framework/Task.h"
-#include "DataFormatsGlobalTracking/RecoContainer.h"
-#include "DataFormatsFT0/RecPoints.h"
-#include "FT0Calibration/RecoCalibInfoWorkflow.h"
-#include "CommonDataFormat/InteractionRecord.h"
-#include "CommonDataFormat/TimeStamp.h"
-#include "ReconstructionDataFormats/GlobalTrackID.h"
-#include "ReconstructionDataFormats/PrimaryVertex.h"
-#include "DataFormatsFT0/RecoCalibInfoObject.h"
-#include 
-#include 
-#include 
-
-using namespace o2::framework;
-using namespace o2::math_utils::detail;
-using PVertex = o2::dataformats::PrimaryVertex;
-using GIndex = o2::dataformats::VtxTrackIndex;
-using DataRequest = o2::globaltracking::DataRequest;
-
-namespace o2::ft0
-{
-void RecoCalibInfoWorkflow::init(InitContext& ic)
-{
-  mTimer.Stop();
-  mTimer.Reset();
-}
-void RecoCalibInfoWorkflow::run(o2::framework::ProcessingContext& pc)
-{
-  o2::globaltracking::RecoContainer recoData;
-  recoData.collectData(pc, *mDataRequest);
-  auto primVertices = recoData.getPrimaryVertices();
-  auto ft0RecPoints = recoData.getFT0RecPoints();
-  std::map bcsMap;
-  auto& calib_data = pc.outputs().make>(o2::framework::OutputRef{"calib", 0});
-  calib_data.reserve(ft0RecPoints.size());
-  for (auto& vertex : primVertices) {
-    auto& timeStamp = vertex.getTimeStamp();
-    double tsTimeStamp = timeStamp.getTimeStamp() * 1E3; // mus to ns
-    uint64_t globalBC = std::round(tsTimeStamp / o2::constants::lhc::LHCBunchSpacingNS);
-    LOG(debug) << "PrimVertices " << globalBC;
-    auto [iter, inserted] = bcsMap.try_emplace(globalBC, &vertex);
-    if (!inserted) {
-      iter->second = nullptr;
-    }
-  }
-  for (auto& ft0RecPoint : ft0RecPoints) {
-    uint64_t bc = ft0RecPoint.getInteractionRecord().toLong();
-    auto item = bcsMap.find(bc);
-    LOG(debug) << " <second == nullptr) {
-      LOG(debug) << "Error: could not find a corresponding BC ID for a FT0 rec. point; BC = " << bc;
-      continue;
-    }
-    auto& vertex = *item->second;
-    auto currentVertex = vertex.getZ();
-    ushort ncont = vertex.getNContributors();
-    LOG(debug) << "CurrentVertex " << currentVertex << " ncont " << int(ncont);
-    if (ncont < 3) {
-      continue;
-    }
-    auto shift = currentVertex / cSpeed;
-    short t0A = ft0RecPoint.getCollisionTimeA() + shift;
-    short t0C = ft0RecPoint.getCollisionTimeC() - shift;
-    short t0AC = ft0RecPoint.getCollisionTimeMean();
-    LOG(debug) << " BC  t0  " << bc << " shift " << shift << " A " << t0A << " C " << t0C << " AC " << t0AC;
-    calib_data.emplace_back(t0A, t0C, t0AC);
-  }
-  mTimer.Stop();
-}
-void RecoCalibInfoWorkflow::endOfStream(EndOfStreamContext& ec)
-{
-  LOGF(info, "Reco calib info workflow  dpl total timing: Cpu: %.3e Real: %.3e s in %d slots",
-       mTimer.CpuTime(), mTimer.RealTime(), mTimer.Counter() - 1);
-}
-
-DataProcessorSpec getRecoCalibInfoWorkflow(GID::mask_t src, bool useMC)
-{
-  auto dataRequest = std::make_shared();
-  dataRequest->requestPrimaryVertertices(false);
-  dataRequest->requestFT0RecPoints(false);
-
-  return DataProcessorSpec{
-    "ft0-calib-reco",
-    dataRequest->inputs,
-    Outputs{
-      {{"calib"}, "FT0", "CALIB_INFO"}},
-    AlgorithmSpec{adaptFromTask(src, dataRequest)},
-    Options{}};
-}
-
-}; // namespace o2::ft0
diff --git a/Detectors/FIT/FT0/calibration/testWorkflow/FT0CalibCollectorWriterSpec.h b/Detectors/FIT/FT0/calibration/testWorkflow/FT0CalibCollectorWriterSpec.h
deleted file mode 100644
index 14ce96ce1adbd..0000000000000
--- a/Detectors/FIT/FT0/calibration/testWorkflow/FT0CalibCollectorWriterSpec.h
+++ /dev/null
@@ -1,132 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-#ifndef O2_CALIBRATION_FT0CALIB_COLLECTOR_WRITER_H
-#define O2_CALIBRATION_FT0CALIB_COLLECTOR_WRITER_H
-
-/// @file   FT0CalibCollectorWriterSpec.h
-/// @brief  Device to write to tree the information for FT0 time slewing calibration.
-
-#include "DataFormatsFT0/FT0CalibrationInfoObject.h"
-#include 
-#include 
-#include "FairLogger.h"
-
-using namespace o2::framework;
-
-namespace o2
-{
-namespace calibration
-{
-
-class FT0CalibCollectorWriter : public o2::framework::Task
-{
-
-  using Geo = o2::ft0::Geometry;
-
- public:
-  void createAndOpenFileAndTree()
-  {
-    TString filename = TString::Format("collFT0%d.root", mCount);
-    LOG(debug) << "opening file " << filename.Data();
-    mfileOut.reset(TFile::Open(TString::Format("%s", filename.Data()), "RECREATE"));
-    mOutputTree = std::make_unique("treeCollectedCalibInfo", "Tree with FT0 calib info for Time Slewing");
-    mOutputTree->Branch(mOutputBranchName.data(), &mPFT0CalibInfoOut);
-    LOG(info) << " @@@@@ createAndOpenFileAndTree tree  set";
-  }
-
-  void init(o2::framework::InitContext& ic) final
-  {
-    mCount = 0;
-    createAndOpenFileAndTree();
-    mFT0CalibInfoOut.reserve(1000000 * Geo::Nchannels); // tree size  216ch * 10^6 entries * 12 byte
-    LOG(info) << " @@@@@ init";
-  }
-
-  void run(o2::framework::ProcessingContext& pc) final
-  {
-    auto collectedInfo = pc.inputs().get>("collectedInfo");
-    auto entriesPerChannel = pc.inputs().get>("entriesCh");
-    int offsetStart = 0;
-    for (int ich = 0; ich < Geo::Nchannels; ich++) {
-      mFT0CalibInfoOut.clear();
-      if (entriesPerChannel[ich] > 0) {
-        mFT0CalibInfoOut.resize(entriesPerChannel[ich]);
-        auto subSpanVect = collectedInfo.subspan(offsetStart, entriesPerChannel[ich]);
-        memcpy(&mFT0CalibInfoOut[0], subSpanVect.data(), sizeof(o2::ft0::FT0CalibrationInfoObject) * subSpanVect.size());
-        const o2::ft0::FT0CalibrationInfoObject* tmp = subSpanVect.data();
-        LOG(debug) << "@@@@@ run ich " << ich << " entries " << entriesPerChannel[ich];
-      }
-      mOutputTree->Fill();
-      offsetStart += entriesPerChannel[ich];
-    }
-    sendOutput(pc.outputs());
-  }
-
-  void endOfStream(o2::framework::EndOfStreamContext& ec) final
-  {
-    mIsEndOfStream = true;
-    sendOutput(ec.outputs());
-  }
-
- private:
-  int mCount = 0; // how many times we filled the tree
-  bool mIsEndOfStream = false;
-  std::vector mFT0CalibInfoOut, *mPFT0CalibInfoOut = &mFT0CalibInfoOut; ///< these are the object and pointer to the CalibInfo of a specific channel that we need to fill the output tree
-  std::unique_ptr mOutputTree;                                                                      ///< tree for the collected calib FT0 info
-  std::string mFT0CalibInfoBranchName = "FT0CalibInfo";                                                    ///< name of branch containing input FT0 calib infos
-  std::string mOutputBranchName = "FT0CollectedCalibInfo";                                                 ///< name of branch containing output
-  std::unique_ptr mfileOut = nullptr;                                                               // file in which to write the output
-
-  //________________________________________________________________
-  void sendOutput(DataAllocator& output)
-  {
-    // This is to fill the tree.
-    // One file with an empty tree will be created at the end, because we have to have a
-    // tree opened before processing, since we do not know a priori if something else
-    // will still come. The size of this extra file is ~6.5 kB
-
-    mfileOut->cd();
-    mOutputTree->Write();
-    mOutputTree->Reset();
-    mCount++;
-    if (!mIsEndOfStream) {
-      createAndOpenFileAndTree();
-    }
-  }
-};
-} // namespace calibration
-
-namespace framework
-{
-
-DataProcessorSpec getFT0CalibCollectorWriterSpec()
-{
-  LOG(debug) << " @@@@ getFT0CalibCollectorWriterSpec ";
-  using device = o2::calibration::FT0CalibCollectorWriter;
-  std::vector inputs;
-  inputs.emplace_back("collectedInfo", o2::header::gDataOriginFT0, "COLLECTEDINFO");
-  inputs.emplace_back("entriesCh", o2::header::gDataOriginFT0, "ENTRIESCH");
-
-  std::vector outputs; // empty
-
-  return DataProcessorSpec{
-    "ft0-calib-collector-writer",
-    inputs,
-    outputs,
-    AlgorithmSpec{adaptFromTask()},
-    Options{}};
-}
-
-} // namespace framework
-} // namespace o2
-
-#endif
diff --git a/Detectors/FIT/FT0/calibration/testWorkflow/FT0CalibSlewingCollectorSpec.h b/Detectors/FIT/FT0/calibration/testWorkflow/FT0CalibSlewingCollectorSpec.h
deleted file mode 100644
index c85b3778b91b2..0000000000000
--- a/Detectors/FIT/FT0/calibration/testWorkflow/FT0CalibSlewingCollectorSpec.h
+++ /dev/null
@@ -1,137 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-#ifndef O2_CALIBRATION_FT0CALIB_SLEWING_COLLECTOR_H
-#define O2_CALIBRATION_FT0CALIB_SLEWING_COLLECTOR_H
-
-/// @file   FT0CalibSlewingCollectorSpec.h
-/// @brief  Device to collect information for FT0 time slewing calibration.
-
-//#include "FT0Calibration/FT0CollectCalibInfo.h"
-#include "FT0Calibration/FT0CalibCollector.h"
-#include "DetectorsCalibration/Utils.h"
-#include "DataFormatsFT0/FT0CalibrationInfoObject.h"
-#include "Framework/Task.h"
-#include "Framework/ConfigParamRegistry.h"
-#include "Framework/ControlService.h"
-#include "Framework/WorkflowSpec.h"
-#include "FairLogger.h"
-#include "DetectorsBase/GRPGeomHelper.h"
-
-using namespace o2::framework;
-
-namespace o2
-{
-namespace ft0
-{
-
-class FT0CalibCollectorDevice : public o2::framework::Task
-{
-
- public:
-  FT0CalibCollectorDevice(std::shared_ptr req) : mCCDBRequest(req) {}
-  void init(o2::framework::InitContext& ic) final
-  {
-    o2::base::GRPGeomHelper::instance().setRequest(mCCDBRequest);
-    bool isTFsendingPolicy = ic.options().get("tf-sending-policy");
-    int maxEnt = ic.options().get("max-number-hits-to-fill-tree");
-    auto slotL = ic.options().get("tf-per-slot");
-    bool absMaxEnt = ic.options().get("is-max-number-hits-to-fill-tree-absolute");
-    mCollector = std::make_unique(isTFsendingPolicy, maxEnt);
-    mCollector->setSlotLength(slotL);
-  }
-
-  void finaliseCCDB(o2::framework::ConcreteDataMatcher& matcher, void* obj) final
-  {
-    o2::base::GRPGeomHelper::instance().finaliseCCDB(matcher, obj);
-  }
-
-  void run(o2::framework::ProcessingContext& pc) final
-  {
-    o2::base::GRPGeomHelper::instance().checkUpdates(pc);
-    o2::base::TFIDInfoHelper::fillTFIDInfo(pc, mCollector->getCurrentTFInfo());
-    auto data = pc.inputs().get>("input");
-    mCollector->process(data);
-    sendOutput(pc.outputs());
-  }
-
-  void endOfStream(o2::framework::EndOfStreamContext& ec) final
-  {
-    mCollector->checkSlotsToFinalize(o2::calibration::INFINITE_TF);
-    // we force finalizing slot zero (unless everything was already finalized), no matter how many entries we had
-    if (mCollector->getNSlots() != 0) {
-      mCollector->finalizeSlot(mCollector->getSlot(0));
-    }
-    sendOutput(ec.outputs());
-  }
-
- private:
-  std::unique_ptr mCollector;
-  std::shared_ptr mCCDBRequest;
-  int mMaxNumOfHits = 0;
-
-  //________________________________________________________________
-  void sendOutput(DataAllocator& output)
-  {
-    // in output we send the calibration tree
-    auto& collectedInfo = mCollector->getCollectedCalibInfo();
-    LOG(debug) << "In CollectorSpec sendOutput: size = " << collectedInfo.size();
-    if (collectedInfo.size()) {
-      auto entries = collectedInfo.size();
-      // this means that we are ready to send the output
-      auto entriesPerChannel = mCollector->getEntriesPerChannel();
-      output.snapshot(Output{o2::header::gDataOriginFT0, "COLLECTEDINFO", 0, Lifetime::Timeframe}, collectedInfo);
-      output.snapshot(Output{o2::header::gDataOriginFT0, "ENTRIESCH", 0, Lifetime::Timeframe}, entriesPerChannel);
-      mCollector->initOutput(); // reset the output for the next round
-    }
-  }
-};
-
-} // namespace ft0
-
-namespace framework
-{
-
-DataProcessorSpec getFT0CalibCollectorDeviceSpec()
-{
-  using device = o2::ft0::FT0CalibCollectorDevice;
-  using clbUtils = o2::calibration::Utils;
-
-  std::vector outputs;
-  outputs.emplace_back(o2::header::gDataOriginFT0, "COLLECTEDINFO", 0, Lifetime::Timeframe);
-  outputs.emplace_back(o2::header::gDataOriginFT0, "ENTRIESCH", 0, Lifetime::Timeframe);
-
-  std::vector inputs;
-  inputs.emplace_back("input", "FT0", "CALIB_INFO");
-  auto ccdbRequest = std::make_shared(true,                           // orbitResetTime
-                                                                true,                           // GRPECS=true
-                                                                false,                          // GRPLHCIF
-                                                                false,                          // GRPMagField
-                                                                false,                          // askMatLUT
-                                                                o2::base::GRPGeomRequest::None, // geometry
-                                                                inputs);
-  return DataProcessorSpec{
-    "calib-ft0calib-collector",
-    inputs,
-    outputs,
-    AlgorithmSpec{adaptFromTask(ccdbRequest)},
-    Options{
-      {"max-number-hits-to-fill-tree", VariantType::Int, 1000, {"maximum number of entries in one channel to trigger teh filling of the tree"}},
-      {"is-max-number-hits-to-fill-tree-absolute", VariantType::Bool, false, {"to decide if we want to multiply the max-number-hits-to-fill-tree by the number of channels (when set to true), or not (when set to false) for fast checks"}},
-      {"tf-sending-policy", VariantType::Bool, false, {"if we are sending output at every TF; otherwise, we use the max-number-hits-to-fill-tree"}},
-      {"tf-per-slot", o2::framework::VariantType::UInt32, 200000u, {"TF per slot "}}}};
-}
-
-} // namespace framework
-
-} // namespace o2
-
-#endif
diff --git a/Detectors/FIT/FT0/calibration/testWorkflow/FT0ChannelTimeCalibration-Workflow.cxx b/Detectors/FIT/FT0/calibration/testWorkflow/FT0ChannelTimeCalibration-Workflow.cxx
deleted file mode 100644
index 05d3ab818e991..0000000000000
--- a/Detectors/FIT/FT0/calibration/testWorkflow/FT0ChannelTimeCalibration-Workflow.cxx
+++ /dev/null
@@ -1,34 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-#include "FT0ChannelTimeCalibrationSpec.h"
-#include "FT0Calibration/FT0ChannelTimeOffsetSlotContainer.h"
-
-using namespace o2::framework;
-
-void customize(std::vector& workflowOptions)
-{
-  // probably some option will be added
-  std::vector options;
-  options.push_back(ConfigParamSpec{"time-calib-fitting-nbins", VariantType::Int, 100, {""}});
-  std::swap(workflowOptions, options);
-}
-
-#include "Framework/runDataProcessing.h"
-
-using namespace o2::framework;
-WorkflowSpec defineDataProcessing(ConfigContext const& config)
-{
-  WorkflowSpec workflow;
-  o2::ft0::FT0ChannelTimeOffsetSlotContainer::sGausFitBins = config.options().get("time-calib-fitting-nbins");
-  workflow.emplace_back(o2::ft0::getFT0ChannelTimeCalibrationSpec());
-  return workflow;
-}
diff --git a/Detectors/FIT/FT0/calibration/testWorkflow/FT0ChannelTimeCalibrationSpec.h b/Detectors/FIT/FT0/calibration/testWorkflow/FT0ChannelTimeCalibrationSpec.h
deleted file mode 100644
index 0daa8eebe5841..0000000000000
--- a/Detectors/FIT/FT0/calibration/testWorkflow/FT0ChannelTimeCalibrationSpec.h
+++ /dev/null
@@ -1,54 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-#ifndef O2_FT0CHANNELTIMECALIBRATIONSPEC_H
-#define O2_FT0CHANNELTIMECALIBRATIONSPEC_H
-
-#include "Framework/DataProcessorSpec.h"
-#include "DataFormatsFT0/FT0CalibrationInfoObject.h"
-#include "DataFormatsFT0/FT0ChannelTimeCalibrationObject.h"
-#include "FITCalibration/FITCalibrationDevice.h"
-#include "FT0Calibration/FT0ChannelTimeOffsetSlotContainer.h"
-
-namespace o2::ft0
-{
-o2::framework::DataProcessorSpec getFT0ChannelTimeCalibrationSpec()
-{
-  using CalibrationDeviceType = o2::fit::FITCalibrationDevice;
-
-  std::vector outputs;
-  outputs.emplace_back(o2::framework::ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBPayload, "FIT_CALIB"}, o2::framework::Lifetime::Sporadic);
-  outputs.emplace_back(o2::framework::ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBWrapper, "FIT_CALIB"}, o2::framework::Lifetime::Sporadic);
-
-  constexpr const char* DEFAULT_INPUT_LABEL = "calib";
-
-  std::vector inputs;
-  inputs.emplace_back(DEFAULT_INPUT_LABEL, "FT0", "CALIB_INFO");
-  auto ccdbRequest = std::make_shared(true,                           // orbitResetTime
-                                                                true,                           // GRPECS=true
-                                                                false,                          // GRPLHCIF
-                                                                false,                          // GRPMagField
-                                                                false,                          // askMatLUT
-                                                                o2::base::GRPGeomRequest::None, // geometry
-                                                                inputs);
-  return o2::framework::DataProcessorSpec{
-    "calib-ft0-channel-time",
-    inputs,
-    outputs,
-    o2::framework::AlgorithmSpec{o2::framework::adaptFromTask(DEFAULT_INPUT_LABEL, ccdbRequest)},
-    o2::framework::Options{
-      {"tf-per-slot", o2::framework::VariantType::UInt32, 56000u, {""}},
-      {"max-delay", o2::framework::VariantType::UInt32, 3u, {""}}}};
-}
-} // namespace o2::ft0
-
-#endif // O2_FT0CHANNELTIMECALIBRATIONSPEC_H
diff --git a/Detectors/FIT/FT0/calibration/testWorkflow/FT0DCSDataProcessorSpec.h b/Detectors/FIT/FT0/calibration/testWorkflow/FT0DCSDataProcessorSpec.h
deleted file mode 100644
index cd0ac4aa67238..0000000000000
--- a/Detectors/FIT/FT0/calibration/testWorkflow/FT0DCSDataProcessorSpec.h
+++ /dev/null
@@ -1,198 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-#ifndef O2_FT0_DATAPROCESSOR_H
-#define O2_FT0_DATAPROCESSOR_H
-
-/// @file   DCSFT0DataProcessorSpec.h
-/// @brief  FT0 Processor for DCS Data Points
-
-#include 
-#include 
-#include 
-#include "DetectorsDCS/DataPointIdentifier.h"
-#include "DetectorsDCS/DataPointValue.h"
-#include "DetectorsDCS/DataPointCompositeObject.h"
-#include "DetectorsDCS/DeliveryType.h"
-#include "DetectorsDCS/AliasExpander.h"
-#include "FT0Calibration/FT0DCSProcessor.h"
-#include "DetectorsCalibration/Utils.h"
-#include "CCDB/CcdbApi.h"
-#include "CCDB/BasicCCDBManager.h"
-#include "Framework/DeviceSpec.h"
-#include "Framework/ConfigParamRegistry.h"
-#include "Framework/ControlService.h"
-#include "Framework/WorkflowSpec.h"
-#include "Framework/Task.h"
-#include "Framework/Logger.h"
-
-using namespace o2::framework;
-
-namespace o2
-{
-namespace ft0
-{
-
-using DPID = o2::dcs::DataPointIdentifier;
-using DPVAL = o2::dcs::DataPointValue;
-using DPCOM = o2::dcs::DataPointCompositeObject;
-using namespace o2::ccdb;
-using CcdbManager = o2::ccdb::BasicCCDBManager;
-using clbUtils = o2::calibration::Utils;
-using HighResClock = std::chrono::high_resolution_clock;
-using Duration = std::chrono::duration>;
-
-class FT0DCSDataProcessor : public o2::framework::Task
-{
- public:
-  void init(o2::framework::InitContext& ic) final
-  {
-    std::vector vect;
-    mDPsUpdateInterval = ic.options().get("DPs-update-interval");
-    if (mDPsUpdateInterval == 0) {
-      LOG(error) << "FT0 DPs update interval set to zero seconds --> changed to 10 min";
-      mDPsUpdateInterval = 600;
-    }
-    bool useCCDBtoConfigure = ic.options().get("use-ccdb-to-configure");
-    if (useCCDBtoConfigure) {
-      LOG(info) << "Configuring via CCDB";
-      std::string ccdbpath = ic.options().get("ccdb-path");
-      auto& mgr = CcdbManager::instance();
-      mgr.setURL(ccdbpath);
-      long ts = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count();
-      std::unordered_map* dpid2DataDesc = mgr.getForTimeStamp>("FT0/Config/DCSDPconfig", ts);
-      for (auto& i : *dpid2DataDesc) {
-        vect.push_back(i.first);
-      }
-    } else {
-      LOG(info) << "Configuring via hardcoded strings";
-      std::vector aliasesHV = {"FT0/HV/FT0_A/MCP_A[1..5]/actual/iMon",
-                                            "FT0/HV/FT0_A/MCP_B[1..5]/actual/iMon",
-                                            "FT0/HV/FT0_A/MCP_C[1..2]/actual/iMon",
-                                            "FT0/HV/FT0_A/MCP_C[4..5]/actual/iMon",
-                                            "FT0/HV/FT0_A/MCP_D[1..5]/actual/iMon",
-                                            "FT0/HV/FT0_A/MCP_E[1..5]/actual/iMon",
-                                            "FT0/HV/FT0_C/MCP_A[2..5]/actual/iMon",
-                                            "FT0/HV/FT0_C/MCP_B[1..6]/actual/iMon",
-                                            "FT0/HV/FT0_C/MCP_C[1..2]/actual/iMon",
-                                            "FT0/HV/FT0_C/MCP_C[5..6]/actual/iMon",
-                                            "FT0/HV/FT0_C/MCP_D[1..2]/actual/iMon",
-                                            "FT0/HV/FT0_C/MCP_D[5..6]/actual/iMon",
-                                            "FT0/HV/FT0_C/MCP_E[1..6]/actual/iMon",
-                                            "FT0/HV/FT0_C/MCP_F[2..5]/actual/iMon",
-                                            "FT0/HV/MCP_LC/actual/iMon"};
-      std::string aliasesADCZERO = "FT0/PM/channel[000..211]/actual/ADC[0..1]_BASELINE";
-      std::vector expAliasesHV = o2::dcs::expandAliases(aliasesHV);
-      std::vector expAliasesADCZERO = o2::dcs::expandAlias(aliasesADCZERO);
-      for (const auto& i : expAliasesHV) {
-        vect.emplace_back(i, o2::dcs::DPVAL_DOUBLE);
-      }
-      for (const auto& i : expAliasesADCZERO) {
-        vect.emplace_back(i, o2::dcs::DPVAL_UINT);
-      }
-    }
-
-    const bool useVerboseMode = ic.options().get("use-verbose-mode");
-    LOG(info) << "Verbose mode: " << useVerboseMode;
-
-    if (useVerboseMode) {
-      LOG(info) << "Listing Data Points for FT0:";
-      for (auto& i : vect) {
-        LOG(info) << i;
-      }
-    }
-
-    mProcessor = std::make_unique();
-
-    if (useVerboseMode) {
-      mProcessor->useVerboseMode();
-    }
-    mProcessor->init(vect);
-    mTimer = HighResClock::now();
-  }
-
-  void run(o2::framework::ProcessingContext& pc) final
-  {
-    auto timeNow = HighResClock::now();
-    long dataTime = (long)(pc.services().get().creation);
-    if (dataTime == 0xffffffffffffffff) {                                                                   // it means it is not set
-      dataTime = std::chrono::duration_cast(timeNow.time_since_epoch()).count(); // in ms
-    }
-    if (mProcessor->getStartValidity() == o2::ccdb::CcdbObjectInfo::INFINITE_TIMESTAMP) {
-      if (mProcessor->getVerboseMode()) {
-        LOG(info) << "startValidity for DPs changed to = " << dataTime;
-      }
-      mProcessor->setStartValidity(dataTime);
-    }
-    auto dps = pc.inputs().get>("input");
-    mProcessor->process(dps);
-    Duration elapsedTime = timeNow - mTimer; // in seconds
-    if (elapsedTime.count() >= mDPsUpdateInterval) {
-      sendDPsoutput(pc.outputs());
-      mTimer = timeNow;
-    }
-  }
-
-  void endOfStream(o2::framework::EndOfStreamContext& ec) final
-  {
-    sendDPsoutput(ec.outputs());
-  }
-
- private:
-  std::unique_ptr mProcessor;
-  HighResClock::time_point mTimer;
-  int64_t mDPsUpdateInterval;
-
-  void sendDPsoutput(DataAllocator& output)
-  {
-    // extract CCDB infos and calibration object for DPs
-    mProcessor->updateDPsCCDB();
-    const auto& payload = mProcessor->getFT0DPsInfo();
-    auto& info = mProcessor->getccdbDPsInfo();
-    auto image = o2::ccdb::CcdbApi::createObjectImage(&payload, &info);
-    LOG(info) << "Sending object " << info.getPath() << "/" << info.getFileName() << " of size " << image->size()
-              << " bytes, valid for " << info.getStartValidityTimestamp() << " : " << info.getEndValidityTimestamp();
-    output.snapshot(Output{o2::calibration::Utils::gDataOriginCDBPayload, "FT0_DCSDPs", 0}, *image.get());
-    output.snapshot(Output{o2::calibration::Utils::gDataOriginCDBWrapper, "FT0_DCSDPs", 0}, info);
-    mProcessor->clearDPsinfo();
-    mProcessor->resetStartValidity();
-  }
-
-}; // end class
-} // namespace ft0
-
-namespace framework
-{
-
-DataProcessorSpec getFT0DCSDataProcessorSpec()
-{
-
-  using clbUtils = o2::calibration::Utils;
-
-  std::vector outputs;
-  outputs.emplace_back(ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBPayload, "FT0_DCSDPs"}, Lifetime::Sporadic);
-  outputs.emplace_back(ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBWrapper, "FT0_DCSDPs"}, Lifetime::Sporadic);
-
-  return DataProcessorSpec{
-    "ft0-dcs-data-processor",
-    Inputs{{"input", "DCS", "FT0DATAPOINTS"}},
-    outputs,
-    AlgorithmSpec{adaptFromTask()},
-    Options{{"ccdb-path", VariantType::String, "http://localhost:8080", {"Path to CCDB"}},
-            {"use-ccdb-to-configure", VariantType::Bool, false, {"Use CCDB to configure"}},
-            {"use-verbose-mode", VariantType::Bool, false, {"Use verbose mode"}},
-            {"DPs-update-interval", VariantType::Int64, 600ll, {"Interval (in s) after which to update the DPs CCDB entry"}}}};
-}
-
-} // namespace framework
-} // namespace o2
-
-#endif
diff --git a/Detectors/FIT/FT0/calibration/testWorkflow/FT0SlewingCalibrationWorkflow.cxx b/Detectors/FIT/FT0/calibration/testWorkflow/FT0SlewingCalibrationWorkflow.cxx
deleted file mode 100644
index 9a9c8e078322a..0000000000000
--- a/Detectors/FIT/FT0/calibration/testWorkflow/FT0SlewingCalibrationWorkflow.cxx
+++ /dev/null
@@ -1,28 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-#include "FT0SlewingCalibrationSpec.h"
-
-void customize(std::vector& workflowOptions)
-{
-  //probably some option will be added
-}
-
-#include "Framework/runDataProcessing.h"
-
-using namespace o2::framework;
-WorkflowSpec defineDataProcessing(ConfigContext const& config)
-{
-  WorkflowSpec workflow;
-  workflow.emplace_back(o2::ft0::getFT0SlewingCalibrationSpec());
-  //add calib spec here...
-  return workflow;
-}
diff --git a/Detectors/FIT/FT0/calibration/testWorkflow/FT0TFProcessor-Workflow.cxx b/Detectors/FIT/FT0/calibration/testWorkflow/FT0TFProcessor-Workflow.cxx
deleted file mode 100644
index 0a1da60d90858..0000000000000
--- a/Detectors/FIT/FT0/calibration/testWorkflow/FT0TFProcessor-Workflow.cxx
+++ /dev/null
@@ -1,84 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-#include "Framework/ConfigParamSpec.h"
-#include 
-#include "Framework/DeviceSpec.h"
-#include "Framework/WorkflowSpec.h"
-#include "Framework/Task.h"
-#include "DataFormatsFT0/ChannelData.h"
-#include "DataFormatsFT0/Digit.h"
-#include "DataFormatsFT0/FT0CalibrationInfoObject.h"
-
-using namespace o2::framework;
-
-namespace o2::ft0
-{
-
-class FT0TFProcessor final : public o2::framework::Task
-{
-
- public:
-  void run(o2::framework::ProcessingContext& pc) final
-  {
-    auto creationTime = pc.services().get().creation; // approximate time in ms
-    //    LOG(info)<<" FT0TFProcessor run "<>("digits");
-    auto channels = pc.inputs().get>("channels");
-    auto& calib_data = pc.outputs().make>(o2::framework::OutputRef{"calib", 0});
-    calib_data.reserve(channels.size());
-    int nDig = digits.size();
-    LOG(debug) << " nDig " << nDig;
-    for (const auto& digit : digits) {
-      const auto& chan = digit.getBunchChannelData(channels);
-      for (const auto& channel : chan) {
-        if (channel.QTCAmpl > 14 && std::abs(channel.CFDTime) < 100) {
-          calib_data.emplace_back(channel.ChId, channel.CFDTime, channel.QTCAmpl, uint64_t(creationTime));
-        }
-      }
-    }
-  }
-};
-
-} // namespace o2::ft0
-
-void customize(std::vector& workflowOptions)
-{
-  std::vector options;
-  options.push_back(ConfigParamSpec{"dispatcher-mode", VariantType::Bool, false, {"Dispatcher mode (FT0/SUB_DIGITSCH and FT0/SUB_DIGITSBC DPL channels should be applied as dispatcher output)."}});
-  std::swap(workflowOptions, options);
-}
-
-#include "Framework/runDataProcessing.h"
-
-WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
-{
-  Inputs inputs{};
-  if (cfgc.options().get("dispatcher-mode")) {
-    inputs.push_back(InputSpec{{"channels"}, "FT0", "SUB_DIGITSCH"});
-    inputs.push_back(InputSpec{{"digits"}, "FT0", "SUB_DIGITSBC"});
-  } else {
-    inputs.push_back(InputSpec{{"channels"}, "FT0", "DIGITSCH"});
-    inputs.push_back(InputSpec{{"digits"}, "FT0", "DIGITSBC"});
-  }
-  DataProcessorSpec dataProcessorSpec{
-    "FT0TFProcessor",
-    inputs,
-    Outputs{
-      {{"calib"}, "FT0", "CALIB_INFO"}},
-    AlgorithmSpec{adaptFromTask()},
-    Options{}};
-
-  WorkflowSpec workflow;
-  workflow.emplace_back(dataProcessorSpec);
-
-  return workflow;
-}
diff --git a/Detectors/FIT/FT0/calibration/testWorkflow/FT0TimeSpectraProcessor-Workflow.cxx b/Detectors/FIT/FT0/calibration/testWorkflow/FT0TimeSpectraProcessor-Workflow.cxx
deleted file mode 100644
index 0bbbcf8ed4f71..0000000000000
--- a/Detectors/FIT/FT0/calibration/testWorkflow/FT0TimeSpectraProcessor-Workflow.cxx
+++ /dev/null
@@ -1,115 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-#include "CommonUtils/ConfigurableParam.h"
-#include "Framework/ConfigParamSpec.h"
-#include 
-#include "Framework/DeviceSpec.h"
-#include "Framework/WorkflowSpec.h"
-#include "Framework/Task.h"
-#include "FT0Base/Geometry.h"
-#include "DataFormatsFT0/ChannelData.h"
-#include "DataFormatsFT0/Digit.h"
-#include "DataFormatsFT0/DigitFilterParam.h"
-#include "CommonDataFormat/FlatHisto2D.h"
-
-using namespace o2::framework;
-
-namespace o2::ft0
-{
-
-class FT0TimeSpectraProcessor final : public o2::framework::Task
-{
-
- public:
-  static constexpr int sNCHANNELS = o2::ft0::Geometry::Nchannels;
-  int mNbinsY{400};
-  float mMinY{-200.};
-  float mMaxY{200.};
-  int mAmpThreshold{10};
-  int mTimeWindow{153};
-  uint8_t mPMbitsToCheck{0b11111110};
-  uint8_t mPMbitsGood{0b01001000};
-  uint8_t mTrgBitsToCheck{0b11110000};
-  uint8_t mTrgBitsGood{0b10010000};
-  void init(o2::framework::InitContext& ic) final
-  {
-    mNbinsY = ic.options().get("number-bins");
-    mMinY = ic.options().get("low-edge");
-    mMaxY = ic.options().get("upper-edge");
-    LOG(info) << "Histogram parameters: " << mNbinsY << " " << mMinY << " " << mMaxY;
-    const auto& param = o2::ft0::DigitFilterParam::Instance();
-    param.printKeyValues();
-    mAmpThreshold = param.mAmpThreshold;
-    mTimeWindow = param.mTimeWindow;
-    mPMbitsGood = param.mPMbitsGood;
-    mPMbitsToCheck = param.mPMbitsToCheck;
-    mTrgBitsGood = param.mTrgBitsGood;
-    mTrgBitsToCheck = param.mTrgBitsToCheck;
-  }
-  void run(o2::framework::ProcessingContext& pc) final
-  {
-    auto digits = pc.inputs().get>("digits");
-    auto channels = pc.inputs().get>("channels");
-    o2::dataformats::FlatHisto2D timeSpectraInfoObject(sNCHANNELS, 0, sNCHANNELS, mNbinsY, mMinY, mMaxY);
-    for (const auto& digit : digits) {
-      if ((digit.mTriggers.triggersignals & mTrgBitsToCheck) != mTrgBitsGood) {
-        continue;
-      }
-      const auto& chan = digit.getBunchChannelData(channels);
-      for (const auto& channel : chan) {
-        if (channel.QTCAmpl > mAmpThreshold && std::abs(channel.CFDTime) < mTimeWindow && ((channel.ChainQTC & mPMbitsToCheck) == mPMbitsGood)) {
-          const auto result = timeSpectraInfoObject.fill(channel.ChId, channel.CFDTime);
-        }
-      }
-    }
-    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "CALIB_INFO", 0, o2::framework::Lifetime::Timeframe}, timeSpectraInfoObject.getBase());
-  }
-};
-
-} // namespace o2::ft0
-
-void customize(std::vector& workflowOptions)
-{
-  std::vector options;
-  options.push_back(ConfigParamSpec{"dispatcher-mode", VariantType::Bool, false, {"Dispatcher mode (FT0/SUB_DIGITSCH and FT0/SUB_DIGITSBC DPL channels should be applied as dispatcher output)."}});
-  options.push_back(ConfigParamSpec{"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}});
-  std::swap(workflowOptions, options);
-}
-
-#include "Framework/runDataProcessing.h"
-
-WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
-{
-  Inputs inputs{};
-  if (cfgc.options().get("dispatcher-mode")) {
-    inputs.push_back(InputSpec{{"channels"}, "FT0", "SUB_DIGITSCH"});
-    inputs.push_back(InputSpec{{"digits"}, "FT0", "SUB_DIGITSBC"});
-  } else {
-    inputs.push_back(InputSpec{{"channels"}, "FT0", "DIGITSCH"});
-    inputs.push_back(InputSpec{{"digits"}, "FT0", "DIGITSBC"});
-  }
-  o2::conf::ConfigurableParam::updateFromString(cfgc.options().get("configKeyValues"));
-  DataProcessorSpec dataProcessorSpec{
-    "FT0TimeSpectraProcessor",
-    inputs,
-    Outputs{
-      {{"calib"}, "FT0", "CALIB_INFO"}},
-    AlgorithmSpec{adaptFromTask()},
-    Options{{"number-bins", VariantType::Int, 400, {"Number of bins along Y-axis"}},
-            {"low-edge", VariantType::Float, -200.0f, {"Lower edge of first bin along Y-axis"}},
-            {"upper-edge", VariantType::Float, 200.0f, {"Upper edge of last bin along Y-axis"}}}};
-
-  WorkflowSpec workflow;
-  workflow.emplace_back(dataProcessorSpec);
-
-  return workflow;
-}
diff --git a/Detectors/FIT/FT0/calibration/testWorkflow/GlobalOffsetsCalibrationSpec.h b/Detectors/FIT/FT0/calibration/testWorkflow/GlobalOffsetsCalibrationSpec.h
deleted file mode 100644
index eb870e361cc08..0000000000000
--- a/Detectors/FIT/FT0/calibration/testWorkflow/GlobalOffsetsCalibrationSpec.h
+++ /dev/null
@@ -1,58 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-#ifndef O2_CALIBRATION_LHCCLOCK_CALIBRATOR_H
-#define O2_CALIBRATION_LHCCLOCK_CALIBRATOR_H
-
-/// @file   LHCClockCalibratorSpec.h
-/// @brief  Device to calibrate LHC clock phase using FT0 data
-
-#include "Framework/DataProcessorSpec.h"
-#include "DataFormatsFT0/GlobalOffsetsCalibrationObject.h"
-#include "DataFormatsFT0/GlobalOffsetsInfoObject.h"
-#include "FT0Calibration/GlobalOffsetsContainer.h"
-#include "FITCalibration/FITCalibrationDevice.h"
-
-using namespace o2::framework;
-namespace o2::ft0
-{
-o2::framework::DataProcessorSpec getGlobalOffsetsCalibrationSpec()
-{
-  using CalibrationDeviceType = o2::fit::FITCalibrationDevice;
-  LOG(info) << " getGlobalOffsetsCalibrationSpec()";
-  constexpr const char* DEFAULT_INPUT_LABEL = "calib";
-
-  std::vector inputs;
-  inputs.emplace_back(DEFAULT_INPUT_LABEL, "FT0", "CALIB_INFO");
-  auto ccdbRequest = std::make_shared(true,                           // orbitResetTime
-                                                                true,                           // GRPECS=true
-                                                                false,                          // GRPLHCIF
-                                                                false,                          // GRPMagField
-                                                                false,                          // askMatLUT
-                                                                o2::base::GRPGeomRequest::None, // geometry
-                                                                inputs);
-  std::vector outputs;
-  outputs.emplace_back(o2::framework::ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBPayload, "FIT_CALIB"}, o2::framework::Lifetime::Sporadic);
-  outputs.emplace_back(o2::framework::ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBWrapper, "FIT_CALIB"}, o2::framework::Lifetime::Sporadic);
-  return o2::framework::DataProcessorSpec{
-    "ft0-global-offsets",
-    inputs, // o2::framework::Inputs{{"input", "FT0", "CALIBDATA"}},
-    outputs,
-    o2::framework::AlgorithmSpec{o2::framework::adaptFromTask(DEFAULT_INPUT_LABEL, ccdbRequest)},
-    Options{
-      {"tf-per-slot", VariantType::UInt32, 55000u, {"number of TFs per calibration time slot"}},
-      {"max-delay", VariantType::UInt32, 3u, {"number of slots in past to consider"}},
-      {"min-entries", VariantType::Int, 500, {"minimum number of entries to fit single time slot"}}}};
-}
-
-} // namespace o2::ft0
-
-#endif
diff --git a/Detectors/FIT/FT0/calibration/testWorkflow/GlobalOffsetsCalibrationWorkflow.cxx b/Detectors/FIT/FT0/calibration/testWorkflow/GlobalOffsetsCalibrationWorkflow.cxx
deleted file mode 100644
index f355a96a4a792..0000000000000
--- a/Detectors/FIT/FT0/calibration/testWorkflow/GlobalOffsetsCalibrationWorkflow.cxx
+++ /dev/null
@@ -1,28 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-#include "GlobalOffsetsCalibrationSpec.h"
-
-void customize(std::vector& workflowOptions)
-{
-  //probably some option will be added
-}
-
-#include "Framework/runDataProcessing.h"
-
-using namespace o2::framework;
-WorkflowSpec defineDataProcessing(ConfigContext const& config)
-{
-
-  WorkflowSpec workflow;
-  workflow.emplace_back(o2::ft0::getGlobalOffsetsCalibrationSpec());
-  return workflow;
-}
diff --git a/Detectors/FIT/FT0/calibration/testWorkflow/GlobalOffsetsCollectWorkflow.cxx b/Detectors/FIT/FT0/calibration/testWorkflow/GlobalOffsetsCollectWorkflow.cxx
deleted file mode 100644
index 3a90816f7d5c2..0000000000000
--- a/Detectors/FIT/FT0/calibration/testWorkflow/GlobalOffsetsCollectWorkflow.cxx
+++ /dev/null
@@ -1,64 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-#include "Framework/ConfigParamSpec.h"
-#include 
-#include "Framework/DeviceSpec.h"
-#include "Framework/WorkflowSpec.h"
-#include "Framework/Task.h"
-#include "DataFormatsFT0/GlobalOffsetsInfoObject.h"
-#include "DataFormatsFT0/RecPoints.h"
-#include "Framework/Logger.h"
-using namespace o2::framework;
-
-namespace o2::ft0
-{
-
-class GlobalOffsetsCollectWorkflow final : public o2::framework::Task
-{
-
- public:
-  void run(o2::framework::ProcessingContext& pc) final
-  {
-    auto creationTime = pc.services().get().creation; // approximate time in ms
-    auto recpoints = pc.inputs().get>("recpoints");
-    auto& calib_data = pc.outputs().make>(o2::framework::OutputRef{"calib", 0});
-    for (const auto& recpoint : recpoints) {
-      short t0AC = recpoint.getCollisionTimeMean();
-      if (std::abs(t0AC) < 1000) {
-        calib_data.emplace_back(t0AC, uint64_t(creationTime));
-      }
-    }
-  }
-};
-
-} // namespace o2::ft0
-
-#include "Framework/runDataProcessing.h"
-
-WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
-{
-  Inputs inputs{};
-  inputs.push_back(InputSpec{{"recpoints"}, "FT0", "RECPOINTS"});
-
-  DataProcessorSpec dataProcessorSpec{
-    "collect-global-offsets",
-    inputs,
-    Outputs{
-      {{"calib"}, "FT0", "CALIB_INFO"}},
-    AlgorithmSpec{adaptFromTask()},
-    Options{}};
-
-  WorkflowSpec workflow;
-  workflow.emplace_back(dataProcessorSpec);
-
-  return workflow;
-}
diff --git a/Detectors/FIT/FT0/calibration/testWorkflow/LHCClockCalibratorSpec.h b/Detectors/FIT/FT0/calibration/testWorkflow/LHCClockCalibratorSpec.h
deleted file mode 100644
index b680ece7169a4..0000000000000
--- a/Detectors/FIT/FT0/calibration/testWorkflow/LHCClockCalibratorSpec.h
+++ /dev/null
@@ -1,58 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-#ifndef O2_CALIBRATION_LHCCLOCK_CALIBRATOR_H
-#define O2_CALIBRATION_LHCCLOCK_CALIBRATOR_H
-
-/// @file   LHCClockCalibratorSpec.h
-/// @brief  Device to calibrate LHC clock phase using FT0 data
-
-#include "Framework/DataProcessorSpec.h"
-#include "FT0Calibration/LHCClockDataHisto.h"
-#include "FT0Calibration/LHCphaseCalibrationObject.h"
-#include "FT0Calibration/FT0CalibrationInfoObject.h"
-#include "FITCalibration/FITCalibrationDevice.h"
-
-using namespace o2::framework;
-namespace o2::ft0
-{
-o2::framework::DataProcessorSpec getLHCClockCalibDeviceSpec()
-{
-  using CalibrationDeviceType = o2::fit::FITCalibrationDevice;
-
-  constexpr const char* DEFAULT_INPUT_LABEL = "calib";
-
-  std::vector inputs;
-  inputs.emplace_back(DEFAULT_INPUT_LABEL, "FT0", "CALIB_INFO");
-  auto ccdbRequest = std::make_shared(true,                           // orbitResetTime
-                                                                true,                           // GRPECS=true
-                                                                false,                          // GRPLHCIF
-                                                                false,                          // GRPMagField
-                                                                false,                          // askMatLUT
-                                                                o2::base::GRPGeomRequest::None, // geometry
-                                                                inputs);
-  std::vector outputs;
-  outputs.emplace_back(o2::framework::ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBPayload, "FIT_CALIB"}, o2::framework::Lifetime::Sporadic);
-  outputs.emplace_back(o2::framework::ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBWrapper, "FIT_CALIB"}, o2::framework::Lifetime::Sporadic);
-  return o2::framework::DataProcessorSpec{
-    "ft0-lhcphase-calibration",
-    inputs, // o2::framework::Inputs{{"input", "FT0", "CALIBDATA"}},
-    outputs,
-    o2::framework::AlgorithmSpec{o2::framework::adaptFromTask(DEFAULT_INPUT_LABEL, ccdbRequest)},
-    Options{
-      {"tf-per-slot", VariantType::UInt32, 26000u, {"number of TFs per calibration time slot"}},
-      {"max-delay", VariantType::UInt32, 3u, {"number of slots in past to consider"}},
-      {"min-entries", VariantType::Int, 500, {"minimum number of entries to fit single time slot"}}}};
-}
-
-} // namespace o2::ft0
-
-#endif
diff --git a/Detectors/FIT/FT0/calibration/testWorkflow/RecoCalibInfoWorkflow.cxx b/Detectors/FIT/FT0/calibration/testWorkflow/RecoCalibInfoWorkflow.cxx
deleted file mode 100644
index 86b89c9787228..0000000000000
--- a/Detectors/FIT/FT0/calibration/testWorkflow/RecoCalibInfoWorkflow.cxx
+++ /dev/null
@@ -1,116 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-#include 
-#include 
-#include 
-#include "Framework/DeviceSpec.h"
-#include "Framework/WorkflowSpec.h"
-#include "Framework/Task.h"
-#include "DataFormatsFT0/ChannelData.h"
-#include "DataFormatsFT0/Digit.h"
-#include "DataFormatsGlobalTracking/RecoContainer.h"
-#include "DataFormatsFT0/RecPoints.h"
-#include "CommonDataFormat/InteractionRecord.h"
-#include "CommonDataFormat/TimeStamp.h"
-#include "ReconstructionDataFormats/GlobalTrackID.h"
-#include "ReconstructionDataFormats/PrimaryVertex.h"
-#include "DataFormatsFT0/RecoCalibInfoObject.h"
-
-using namespace o2::framework;
-
-namespace o2::ft0
-{
-
-class RecoCalibInfoWorkflow final : public o2::framework::Task
-{
-  using DataRequest = o2::globaltracking::DataRequest;
-
- public:
-  /* void collectBCs(gsl::span& ft0RecPoints, */
-  /*                 gsl::span& primVertices, */
-  /*                 std::map& bcsMap); */
-  void run(o2::framework::ProcessingContext& pc) final
-  {
-
-    o2::globaltracking::RecoContainer recoData;
-    recoData.collectData(pc, *mDataRequest);
-    LOG(info) << " @@@read RecoContainer";
-    auto primVertices = recoData.getPrimaryVertices();
-    LOG(info) << "@@@ primVertices  ";
-    auto ft0RecPoints = recoData.getFT0RecPoints();
-    LOG(info) << "@@@@ read T0 recpoints ";
-    std::map bcsMap;
-    for (auto& vertex : primVertices) {
-      auto& timeStamp = vertex.getTimeStamp();
-      double tsTimeStamp = timeStamp.getTimeStamp() * 1E3; // mus to ns
-      uint64_t globalBC = std::round(tsTimeStamp / o2::constants::lhc::LHCBunchSpacingNS);
-      auto [iter, inserted] = bcsMap.try_emplace(globalBC, &vertex);
-      if (!inserted)
-        iter->second = nullptr;
-    }
-    /* collectBCs(ft0RecPoints, primVertices, bcsMap); */
-
-    for (auto& ft0RecPoint : ft0RecPoints) {
-      uint64_t bc = ft0RecPoint.getInteractionRecord().toLong();
-      /* uint64_t bc = globalBC; */
-      auto item = bcsMap.find(bc);
-      if (item == bcsMap.end() || item->second == nullptr) {
-        LOG(fatal) << "Error: could not find a corresponding BC ID for a FT0 rec. point; BC = " << bc;
-        continue;
-      }
-      auto& vertex = *item->second;
-      /* int bcID = -1; */
-      /* if (item != bcsMap.end()) { */
-      /*   bcID = item->second; */
-      auto currentVertex = vertex.getZ();
-      ushort ncont = vertex.getNContributors();
-      LOG(info) << "@@@ currentVertex " << currentVertex << " ncont " << int(ncont);
-      if (ncont == 0)
-        continue;
-      auto shift = currentVertex / TMath::C();
-      LOG(info) << " BC  t0 " << bc;
-      short t0A = ft0RecPoint.getCollisionTimeA() + shift;
-      short t0C = ft0RecPoint.getCollisionTimeC() - shift;
-      short t0AC = ft0RecPoint.getCollisionTimeMean();
-
-      /* auto recpoints = */
-      /*     pc.inputs().get>("recpoints"); */
-      auto& calib_data = pc.outputs().make>(o2::framework::OutputRef{"calib", 0});
-      calib_data.emplace_back(t0A, t0C, t0AC);
-    }
-  }
-
- private:
-  std::shared_ptr mDataRequest;
-};
-
-} // namespace o2::ft0
-
-#include "Framework/runDataProcessing.h"
-
-WorkflowSpec defineDataProcessing(ConfigContext const&)
-{
-
-  DataProcessorSpec dataProcessorSpec{
-    "RecoCalibInfoWorkflow",
-    Inputs{
-      {{"recpoints"}, "FT0", "FT0CLUSTER"},
-    },
-    Outputs{
-      {{"calib"}, "FT0", "CALIB_INFO"}},
-    AlgorithmSpec{adaptFromTask()},
-    Options{}};
-
-  WorkflowSpec workflow;
-  workflow.emplace_back(dataProcessorSpec);
-  return workflow;
-}
diff --git a/Detectors/FIT/FT0/calibration/testWorkflow/RecoQAInfoWorkflow.cxx b/Detectors/FIT/FT0/calibration/testWorkflow/RecoQAInfoWorkflow.cxx
deleted file mode 100644
index 86b89c9787228..0000000000000
--- a/Detectors/FIT/FT0/calibration/testWorkflow/RecoQAInfoWorkflow.cxx
+++ /dev/null
@@ -1,116 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-#include 
-#include 
-#include 
-#include "Framework/DeviceSpec.h"
-#include "Framework/WorkflowSpec.h"
-#include "Framework/Task.h"
-#include "DataFormatsFT0/ChannelData.h"
-#include "DataFormatsFT0/Digit.h"
-#include "DataFormatsGlobalTracking/RecoContainer.h"
-#include "DataFormatsFT0/RecPoints.h"
-#include "CommonDataFormat/InteractionRecord.h"
-#include "CommonDataFormat/TimeStamp.h"
-#include "ReconstructionDataFormats/GlobalTrackID.h"
-#include "ReconstructionDataFormats/PrimaryVertex.h"
-#include "DataFormatsFT0/RecoCalibInfoObject.h"
-
-using namespace o2::framework;
-
-namespace o2::ft0
-{
-
-class RecoCalibInfoWorkflow final : public o2::framework::Task
-{
-  using DataRequest = o2::globaltracking::DataRequest;
-
- public:
-  /* void collectBCs(gsl::span& ft0RecPoints, */
-  /*                 gsl::span& primVertices, */
-  /*                 std::map& bcsMap); */
-  void run(o2::framework::ProcessingContext& pc) final
-  {
-
-    o2::globaltracking::RecoContainer recoData;
-    recoData.collectData(pc, *mDataRequest);
-    LOG(info) << " @@@read RecoContainer";
-    auto primVertices = recoData.getPrimaryVertices();
-    LOG(info) << "@@@ primVertices  ";
-    auto ft0RecPoints = recoData.getFT0RecPoints();
-    LOG(info) << "@@@@ read T0 recpoints ";
-    std::map bcsMap;
-    for (auto& vertex : primVertices) {
-      auto& timeStamp = vertex.getTimeStamp();
-      double tsTimeStamp = timeStamp.getTimeStamp() * 1E3; // mus to ns
-      uint64_t globalBC = std::round(tsTimeStamp / o2::constants::lhc::LHCBunchSpacingNS);
-      auto [iter, inserted] = bcsMap.try_emplace(globalBC, &vertex);
-      if (!inserted)
-        iter->second = nullptr;
-    }
-    /* collectBCs(ft0RecPoints, primVertices, bcsMap); */
-
-    for (auto& ft0RecPoint : ft0RecPoints) {
-      uint64_t bc = ft0RecPoint.getInteractionRecord().toLong();
-      /* uint64_t bc = globalBC; */
-      auto item = bcsMap.find(bc);
-      if (item == bcsMap.end() || item->second == nullptr) {
-        LOG(fatal) << "Error: could not find a corresponding BC ID for a FT0 rec. point; BC = " << bc;
-        continue;
-      }
-      auto& vertex = *item->second;
-      /* int bcID = -1; */
-      /* if (item != bcsMap.end()) { */
-      /*   bcID = item->second; */
-      auto currentVertex = vertex.getZ();
-      ushort ncont = vertex.getNContributors();
-      LOG(info) << "@@@ currentVertex " << currentVertex << " ncont " << int(ncont);
-      if (ncont == 0)
-        continue;
-      auto shift = currentVertex / TMath::C();
-      LOG(info) << " BC  t0 " << bc;
-      short t0A = ft0RecPoint.getCollisionTimeA() + shift;
-      short t0C = ft0RecPoint.getCollisionTimeC() - shift;
-      short t0AC = ft0RecPoint.getCollisionTimeMean();
-
-      /* auto recpoints = */
-      /*     pc.inputs().get>("recpoints"); */
-      auto& calib_data = pc.outputs().make>(o2::framework::OutputRef{"calib", 0});
-      calib_data.emplace_back(t0A, t0C, t0AC);
-    }
-  }
-
- private:
-  std::shared_ptr mDataRequest;
-};
-
-} // namespace o2::ft0
-
-#include "Framework/runDataProcessing.h"
-
-WorkflowSpec defineDataProcessing(ConfigContext const&)
-{
-
-  DataProcessorSpec dataProcessorSpec{
-    "RecoCalibInfoWorkflow",
-    Inputs{
-      {{"recpoints"}, "FT0", "FT0CLUSTER"},
-    },
-    Outputs{
-      {{"calib"}, "FT0", "CALIB_INFO"}},
-    AlgorithmSpec{adaptFromTask()},
-    Options{}};
-
-  WorkflowSpec workflow;
-  workflow.emplace_back(dataProcessorSpec);
-  return workflow;
-}
diff --git a/Detectors/FIT/FT0/calibration/testWorkflow/calib-global-offsets.cxx b/Detectors/FIT/FT0/calibration/testWorkflow/calib-global-offsets.cxx
deleted file mode 100644
index 1f70538c2880b..0000000000000
--- a/Detectors/FIT/FT0/calibration/testWorkflow/calib-global-offsets.cxx
+++ /dev/null
@@ -1,43 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-/// @file  calib-global-offsets.cxx
-
-#include "FT0Calibration/GlobalOffsetsCalibInfoWorkflow.h"
-#include "Framework/CompletionPolicy.h"
-#include "CommonUtils/ConfigurableParam.h"
-#include "GlobalTrackingWorkflowHelpers/InputHelper.h"
-#include "DetectorsCommonDataFormats/DetID.h"
-#include 
-
-using namespace o2::framework;
-
-void customize(std::vector& workflowOptions)
-{
-  // option allowing to set parameters
-  std::vector options{
-    {"disable-mc", o2::framework::VariantType::Bool, false, {"disable MC propagation"}},
-    {"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings ..."}}};
-
-  std::swap(workflowOptions, options);
-}
-#include "Framework/runDataProcessing.h"
-
-WorkflowSpec defineDataProcessing(ConfigContext const& configcontext)
-{
-  o2::conf::ConfigurableParam::updateFromString(configcontext.options().get("configKeyValues"));
-  auto useMC = !configcontext.options().get("disable-mc");
-
-  WorkflowSpec specs;
-  specs.emplace_back(o2::ft0::getRecoCalibInfoWorkflow(useMC));
-
-  return std::move(specs);
-}
diff --git a/Detectors/FIT/FT0/calibration/testWorkflow/ft0-collect-calib-workflow.cxx b/Detectors/FIT/FT0/calibration/testWorkflow/ft0-collect-calib-workflow.cxx
deleted file mode 100644
index 29c7d9719e0c1..0000000000000
--- a/Detectors/FIT/FT0/calibration/testWorkflow/ft0-collect-calib-workflow.cxx
+++ /dev/null
@@ -1,35 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-#include "FT0CalibSlewingCollectorSpec.h"
-#include "FT0CalibCollectorWriterSpec.h"
-#include "Framework/DataProcessorSpec.h"
-
-using namespace o2::framework;
-
-// we need to add workflow options before including Framework/runDataProcessing
-void customize(std::vector& workflowOptions)
-{
-  // option allowing to set parameters
-}
-
-// ------------------------------------------------------------------
-
-#include "Framework/runDataProcessing.h"
-
-WorkflowSpec defineDataProcessing(ConfigContext const& configcontext)
-{
-  WorkflowSpec specs;
-  specs.emplace_back(getFT0CalibCollectorDeviceSpec());
-  specs.emplace_back(getFT0CalibCollectorWriterSpec());
-
-  return specs;
-}
diff --git a/Detectors/FIT/FT0/calibration/testWorkflow/ft0-dcs-sim-workflow.cxx b/Detectors/FIT/FT0/calibration/testWorkflow/ft0-dcs-sim-workflow.cxx
deleted file mode 100644
index 0b93920ac1ffb..0000000000000
--- a/Detectors/FIT/FT0/calibration/testWorkflow/ft0-dcs-sim-workflow.cxx
+++ /dev/null
@@ -1,50 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-// // we need to add workflow options before including Framework/runDataProcessing
-// void customize(std::vector& workflowOptions)
-// {
-//   // option allowing to set parameters
-// }
-
-// ------------------------------------------------------------------
-
-#include "DCStestWorkflow/DCSRandomDataGeneratorSpec.h"
-#include "Framework/runDataProcessing.h"
-
-o2::framework::WorkflowSpec defineDataProcessing(o2::framework::ConfigContext const& configcontext)
-{
-  std::vector dphints;
-  // for testing, we use less DPs than the official ones
-  dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/HV/FT0_A/MCP_A[1..2]/actual/iMon", 250, 350});
-  dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/PM/channel[000..001]/actual/ADC[0..1]_BASELINE", 30, 150});
-  // Official list
-  // dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/HV/FT0_A/MCP_A[1..5]/actual/iMon", 250, 350});
-  // dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/HV/FT0_A/MCP_B[1..5]/actual/iMon", 250, 350});
-  // dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/HV/FT0_A/MCP_C[1..2]/actual/iMon", 250, 350});
-  // dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/HV/FT0_A/MCP_C[4..5]/actual/iMon", 250, 350});
-  // dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/HV/FT0_A/MCP_D[1..5]/actual/iMon", 250, 350});
-  // dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/HV/FT0_A/MCP_E[1..5]/actual/iMon", 250, 350});
-  // dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/HV/FT0_C/MCP_A[2..5]/actual/iMon", 250, 350});
-  // dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/HV/FT0_C/MCP_B[1..6]/actual/iMon", 250, 350});
-  // dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/HV/FT0_C/MCP_C[1..2]/actual/iMon", 250, 350});
-  // dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/HV/FT0_C/MCP_C[5..6]/actual/iMon", 250, 350});
-  // dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/HV/FT0_C/MCP_D[1..2]/actual/iMon", 250, 350});
-  // dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/HV/FT0_C/MCP_D[5..6]/actual/iMon", 250, 350});
-  // dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/HV/FT0_C/MCP_E[1..6]/actual/iMon", 250, 350});
-  // dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/HV/FT0_C/MCP_F[2..5]/actual/iMon", 250, 350});
-  // dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/HV/MCP_LC/actual/iMon", 250, 350});
-  // dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/PM/channel[000..211]/actual/ADC[0..1]_BASELINE", 30, 150});
-
-  o2::framework::WorkflowSpec specs;
-  specs.emplace_back(o2::dcs::test::getDCSRandomDataGeneratorSpec(dphints, "FT0"));
-  return specs;
-}
diff --git a/Detectors/FIT/FT0/calibration/testWorkflow/lhc-clockphase-workflow.cxx b/Detectors/FIT/FT0/calibration/testWorkflow/lhc-clockphase-workflow.cxx
deleted file mode 100644
index 4384bfb1c6e3c..0000000000000
--- a/Detectors/FIT/FT0/calibration/testWorkflow/lhc-clockphase-workflow.cxx
+++ /dev/null
@@ -1,32 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-#include "Framework/DataProcessorSpec.h"
-#include "LHCClockCalibratorSpec.h"
-
-using namespace o2::framework;
-
-// we need to add workflow options before including Framework/runDataProcessing
-void customize(std::vector& workflowOptions)
-{
-  // option allowing to set parameters
-}
-
-// ------------------------------------------------------------------
-
-#include "Framework/runDataProcessing.h"
-
-WorkflowSpec defineDataProcessing(ConfigContext const& configcontext)
-{
-  WorkflowSpec specs;
-  specs.emplace_back(o2::ft0::getLHCClockCalibDeviceSpec());
-  return specs;
-}
diff --git a/Detectors/FIT/FT0/calibration/testWorkflow/slew_upload.cxx b/Detectors/FIT/FT0/calibration/testWorkflow/slew_upload.cxx
deleted file mode 100644
index b5a60c68f37f1..0000000000000
--- a/Detectors/FIT/FT0/calibration/testWorkflow/slew_upload.cxx
+++ /dev/null
@@ -1,116 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-/// \file slew_upload.cxx
-/// \author Alla.Maevskaya@cern.ch
-
-#include 
-#include 
-#include 
-#include 
-#include "Framework/Logger.h"
-#include 
-#include 
-#include "CommonUtils/StringUtils.h"
-#include "CommonUtils/ConfigurableParam.h"
-#include "CommonUtils/NameConf.h"
-#include "CCDB/BasicCCDBManager.h"
-#include "CCDB/CcdbApi.h"
-#include "FT0Calibration/FT0CalibTimeSlewing.h"
-#include 
-#include 
-#include 
-#include 
-#include 
-
-using o2::ccdb::BasicCCDBManager;
-using o2::ccdb::CcdbApi;
-using namespace o2::ft0;
-
-namespace bpo = boost::program_options;
-
-void slew_upload(const std::string& inFileName, const std::string& mergedFileName, int nFiles);
-
-int main(int argc, char** argv)
-{
-  bpo::variables_map vm;
-  bpo::options_description opt_general("Usage:\n  " + std::string(argv[0]) +
-                                       "Upload FT0 slewing corrections\n");
-  bpo::options_description opt_hidden("");
-  bpo::options_description opt_all;
-  bpo::positional_options_description opt_pos;
-
-  try {
-    auto add_option = opt_general.add_options();
-    add_option("help,h", "Print this help message");
-    add_option("input-file", bpo::value()->default_value("collFT0.root"), "verbosity level");
-    add_option("merged-file", bpo::value()->default_value("FT0slewGraphs.root"), "input merged  file");
-    add_option("number-of-files", bpo::value()->default_value(1), "number of files to merge");
-    add_option("configKeyValues", bpo::value()->default_value(""), "comma-separated configKeyValues");
-
-    opt_all.add(opt_general).add(opt_hidden);
-    bpo::store(bpo::command_line_parser(argc, argv).options(opt_all).positional(opt_pos).run(), vm);
-
-    if (vm.count("help")) {
-      std::cout << opt_general << std::endl;
-      exit(0);
-    }
-
-    bpo::notify(vm);
-  } catch (bpo::error& e) {
-    std::cerr << "ERROR: " << e.what() << std::endl
-              << std::endl;
-    std::cerr << opt_general << std::endl;
-    exit(1);
-  } catch (std::exception& e) {
-    std::cerr << e.what() << ", application will now exit" << std::endl;
-    exit(2);
-  }
-  o2::conf::ConfigurableParam::updateFromString(vm["configKeyValues"].as());
-
-  //  o2::conf::ConfigurableParam::updateFromString(vm["configKeyValues"].as());
-  slew_upload(vm["input-file"].as(),
-              vm["merged-file"].as(),
-              vm["number-of-files"].as());
-
-  return 0;
-}
-
-void slew_upload(const std::string& inFileName, const std::string& mergedFileName, int nFiles)
-{
-  TStopwatch swTot;
-  swTot.Start();
-
-  o2::ft0::FT0CalibTimeSlewing sl;
-  sl.setSingleFileName(inFileName);
-  sl.setMergedFileName(mergedFileName);
-  sl.setNfiles(nFiles);
-  sl.mergeFilesWithTree();
-  for (int iCh = 0; iCh < o2::ft0 ::Geometry::Nchannels; ++iCh) {
-    TH2F* hist = sl.getTimeAmpHist(iCh);
-    if (hist->GetEntries() < 100000) {
-      continue;
-    }
-    sl.fillGraph(iCh, hist);
-  }
-  std::array graphs = sl.getGraphs();
-  TGraph& gr = graphs.at(29);
-  gr.Print();
-  CcdbApi api;
-  std::map metadata; // can be empty
-  api.init(o2::base::NameConf::getCCDBServer()); // or http://localhost:8080 for a local installation
-                                                 // store abitrary user object in strongly typed manner
-  api.storeAsTFileAny(&graphs, "FT0/Calib/SlewingCorr", metadata);
-
-  //
-  swTot.Stop();
-  swTot.Print();
-}
diff --git a/Detectors/FIT/FT0/calibration/workflow/FT0EventsPerBcProcessor-Workflow.cxx b/Detectors/FIT/FT0/calibration/workflow/FT0EventsPerBcProcessor-Workflow.cxx
new file mode 100644
index 0000000000000..5cef707da2cca
--- /dev/null
+++ b/Detectors/FIT/FT0/calibration/workflow/FT0EventsPerBcProcessor-Workflow.cxx
@@ -0,0 +1,48 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "FT0EventsPerBcSpec.h"
+#include "Framework/Lifetime.h"
+#include 
+
+o2::framework::WorkflowSpec defineDataProcessing(o2::framework::ConfigContext const& cfgc)
+{
+  using namespace o2::framework;
+  using o2::calibration::FT0EventsPerBcProcessor;
+  std::vector inputs;
+  inputs.emplace_back("digits", "FT0", "DIGITSBC", Lifetime::Timeframe);
+  auto ccdbRequest = std::make_shared(true,                           // orbitResetTime
+                                                                false,                          // GRPECS=true
+                                                                false,                          // GRPLHCIF
+                                                                false,                          // GRPMagField
+                                                                false,                          // askMatLUT
+                                                                o2::base::GRPGeomRequest::None, // geometry
+                                                                inputs);
+  std::vector outputs;
+  outputs.emplace_back(ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBWrapper, "EventsPerBc"}, Lifetime::Timeframe);
+  outputs.emplace_back(ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBPayload, "EventsPerBc"}, Lifetime::Timeframe);
+  DataProcessorSpec dataProcessorSpec{
+    "FT0EventsPerBcProcessor",
+    inputs,
+    outputs,
+    AlgorithmSpec(adaptFromTask(ccdbRequest)),
+    Options{
+      {"slot-len-sec", VariantType::UInt32, 3600u, {"Duration of each slot in seconds"}},
+      {"one-object-per-run", VariantType::Bool, false, {"If set, workflow creates only one calibration object per run"}},
+      {"min-entries-number", VariantType::UInt32, 5000u, {"Minimum number of entries required for a slot to be valid"}},
+      {"min-ampl-side-a", VariantType::Int, 0, {"Amplitude threshold for Side A events"}},
+      {"min-ampl-side-c", VariantType::Int, 0, {"Amplitude threshold for Side C events"}},
+      {"min-sum-of-ampl", VariantType::Int, 0, {"Amplitude threshold for sum of A-side and C-side amplitudes"}}}};
+
+  WorkflowSpec workflow;
+  workflow.emplace_back(dataProcessorSpec);
+  return workflow;
+}
\ No newline at end of file
diff --git a/Detectors/FIT/FT0/calibration/workflow/FT0EventsPerBcSpec.h b/Detectors/FIT/FT0/calibration/workflow/FT0EventsPerBcSpec.h
new file mode 100644
index 0000000000000..d493e2a606613
--- /dev/null
+++ b/Detectors/FIT/FT0/calibration/workflow/FT0EventsPerBcSpec.h
@@ -0,0 +1,128 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef O2_CALIBRATION_FT0_EVENTS_PER_BC_CALIBRATOR_H
+#define O2_CALIBRATION_FT0_EVENTS_PER_BC_CALIBRATOR_H
+
+#include "Framework/runDataProcessing.h"
+#include "CommonUtils/ConfigurableParam.h"
+#include "Framework/ConfigParamSpec.h"
+#include 
+#include "Framework/DeviceSpec.h"
+#include "Framework/WorkflowSpec.h"
+#include "Framework/Task.h"
+#include "DetectorsCalibration/Utils.h"
+#include "DetectorsBase/GRPGeomHelper.h"
+
+#include "DataFormatsFT0/Digit.h"
+#include "FT0Calibration/EventsPerBcCalibrator.h"
+
+namespace o2::calibration
+{
+class FT0EventsPerBcProcessor final : public o2::framework::Task
+{
+ public:
+  FT0EventsPerBcProcessor(std::shared_ptr request) : mCCDBRequest(request) {}
+
+  void init(o2::framework::InitContext& ic) final
+  {
+    o2::base::GRPGeomHelper::instance().setRequest(mCCDBRequest);
+    if (ic.options().hasOption("slot-len-sec")) {
+      mSlotLenSec = ic.options().get("slot-len-sec");
+    }
+    if (ic.options().hasOption("one-object-per-run")) {
+      mOneObjectPerRun = ic.options().get("one-object-per-run");
+    }
+    if (ic.options().hasOption("min-entries-number")) {
+      mMinNumberOfEntries = ic.options().get("min-entries-number");
+    }
+    if (ic.options().hasOption("min-ampl-side-a")) {
+      mMinAmplitudeSideA = ic.options().get("min-ampl-side-a");
+    }
+    if (ic.options().hasOption("min-ampl-side-c")) {
+      mMinAmplitudeSideC = ic.options().get("min-ampl-side-c");
+    }
+    if (ic.options().hasOption("min-sum-of-ampl")) {
+      mMinSumOfAmplitude = ic.options().get("min-sum-of-ampl");
+    }
+
+    mCalibrator = std::make_unique(mMinNumberOfEntries, mMinAmplitudeSideA, mMinAmplitudeSideC, mMinSumOfAmplitude);
+
+    if (mOneObjectPerRun) {
+      LOG(info) << "Only one object will be created at the end of run";
+      mCalibrator->setUpdateAtTheEndOfRunOnly();
+    }
+    if (mOneObjectPerRun == false) {
+      LOG(info) << "Defined slot interval to " << mSlotLenSec << " seconds";
+      mCalibrator->setSlotLengthInSeconds(mSlotLenSec);
+    }
+  }
+
+  void finaliseCCDB(o2::framework::ConcreteDataMatcher& matcher, void* obj)
+  {
+    o2::base::GRPGeomHelper::instance().finaliseCCDB(matcher, obj);
+  }
+
+  void run(o2::framework::ProcessingContext& pc) final
+  {
+    o2::base::GRPGeomHelper::instance().checkUpdates(pc);
+    auto digits = pc.inputs().get>("digits");
+    o2::base::TFIDInfoHelper::fillTFIDInfo(pc, mCalibrator->getCurrentTFInfo());
+    if (digits.size() == 0) {
+      return;
+    }
+    mCalibrator->process(digits);
+    if (mOneObjectPerRun == false) {
+      sendOutput(pc.outputs());
+    }
+  }
+
+  void endOfStream(o2::framework::EndOfStreamContext& ec) final
+  {
+    LOG(info) << "Received end-of-stream, checking for slot to finalize...";
+    mCalibrator->checkSlotsToFinalize();
+    sendOutput(ec.outputs());
+    mCalibrator->initOutput();
+  }
+
+  void sendOutput(o2::framework::DataAllocator& output)
+  {
+    using o2::framework::Output;
+    const auto& tvxHists = mCalibrator->getTvxPerBc();
+    auto& infos = mCalibrator->getTvxPerBcCcdbInfo();
+    for (unsigned int idx = 0; idx < tvxHists.size(); idx++) {
+      auto& info = infos[idx];
+      const auto& payload = tvxHists[idx];
+
+      auto image = o2::ccdb::CcdbApi::createObjectImage(&payload, info.get());
+      LOG(info) << "Sending object " << info->getPath() << "/" << info->getFileName() << " of size " << image->size()
+                << " bytes, valid for " << info->getStartValidityTimestamp() << " : " << info->getEndValidityTimestamp();
+      output.snapshot(Output{o2::calibration::Utils::gDataOriginCDBPayload, "EventsPerBc", idx}, *image.get());
+      output.snapshot(Output{o2::calibration::Utils::gDataOriginCDBWrapper, "EventsPerBc", idx}, *info.get());
+    }
+
+    if (tvxHists.size()) {
+      mCalibrator->initOutput();
+    }
+  }
+
+ private:
+  std::shared_ptr mCCDBRequest;
+  std::unique_ptr mCalibrator;
+  bool mOneObjectPerRun;
+  uint32_t mSlotLenSec;
+  uint32_t mMinNumberOfEntries;
+  int32_t mMinAmplitudeSideA;
+  int32_t mMinAmplitudeSideC;
+  int32_t mMinSumOfAmplitude;
+};
+} // namespace o2::calibration
+#endif
\ No newline at end of file
diff --git a/Detectors/FIT/FT0/calibration/testWorkflow/FT0TimeOffsetCalibration-Workflow.cxx b/Detectors/FIT/FT0/calibration/workflow/FT0TimeOffsetCalibration-Workflow.cxx
similarity index 81%
rename from Detectors/FIT/FT0/calibration/testWorkflow/FT0TimeOffsetCalibration-Workflow.cxx
rename to Detectors/FIT/FT0/calibration/workflow/FT0TimeOffsetCalibration-Workflow.cxx
index a9ba63da9ed7e..20504e88e5c28 100644
--- a/Detectors/FIT/FT0/calibration/testWorkflow/FT0TimeOffsetCalibration-Workflow.cxx
+++ b/Detectors/FIT/FT0/calibration/workflow/FT0TimeOffsetCalibration-Workflow.cxx
@@ -12,10 +12,9 @@
 #include "Framework/DataProcessorSpec.h"
 
 #include "CommonUtils/ConfigurableParam.h"
-#include "DataFormatsFT0/FT0ChannelTimeCalibrationObject.h"
+#include "DataFormatsFT0/SpectraInfoObject.h"
 #include "FITCalibration/FITCalibrationDevice.h"
 #include "FT0Calibration/FT0TimeOffsetSlotContainer.h"
-#include "FT0Calibration/CalibParam.h"
 
 using namespace o2::framework;
 
@@ -33,13 +32,15 @@ void customize(std::vector& workflowOptions)
 WorkflowSpec defineDataProcessing(ConfigContext const& config)
 {
   using CalibrationDeviceType = o2::fit::FITCalibrationDevice;
+                                                              o2::ft0::FT0TimeOffsetSlotContainer, o2::ft0::TimeSpectraInfoObject>;
+  std::vector inputs;
   std::vector outputs;
-  outputs.emplace_back(o2::framework::ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBPayload, "FIT_CALIB"}, o2::framework::Lifetime::Sporadic);
-  outputs.emplace_back(o2::framework::ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBWrapper, "FIT_CALIB"}, o2::framework::Lifetime::Sporadic);
+  const o2::header::DataDescription inputDataDescriptor{"TIME_SPECTRA"};
+  const o2::header::DataDescription outputDataDescriptor{"FT0_TIME_CALIB"};
+  CalibrationDeviceType::prepareVecInputSpec(inputs, o2::header::gDataOriginFT0, inputDataDescriptor);
+  CalibrationDeviceType::prepareVecOutputSpec(outputs, outputDataDescriptor);
+
   o2::conf::ConfigurableParam::updateFromString(config.options().get("configKeyValues"));
-  std::vector inputs;
-  inputs.emplace_back("calib", "FT0", "CALIB_INFO");
   auto ccdbRequest = std::make_shared(true,                           // orbitResetTime
                                                                 true,                           // GRPECS=true
                                                                 false,                          // GRPLHCIF
@@ -51,10 +52,11 @@ WorkflowSpec defineDataProcessing(ConfigContext const& config)
     "ft0-time-offset-calib",
     inputs,
     outputs,
-    o2::framework::AlgorithmSpec{o2::framework::adaptFromTask("calib", ccdbRequest)},
+    o2::framework::AlgorithmSpec{o2::framework::adaptFromTask(ccdbRequest, outputDataDescriptor)},
     o2::framework::Options{
       {"tf-per-slot", o2::framework::VariantType::UInt32, 56000u, {""}},
-      {"max-delay", o2::framework::VariantType::UInt32, 3u, {""}}}};
+      {"max-delay", o2::framework::VariantType::UInt32, 3u, {""}},
+      {"extra-info-per-slot", o2::framework::VariantType::String, "", {"Extra info for time slot(usually for debugging)"}}}};
 
   WorkflowSpec workflow;
   workflow.emplace_back(dataProcessorSpec);
diff --git a/Detectors/FIT/FT0/calibration/workflow/FT0TimeSpectraProcessor-Workflow.cxx b/Detectors/FIT/FT0/calibration/workflow/FT0TimeSpectraProcessor-Workflow.cxx
new file mode 100644
index 0000000000000..214d919196763
--- /dev/null
+++ b/Detectors/FIT/FT0/calibration/workflow/FT0TimeSpectraProcessor-Workflow.cxx
@@ -0,0 +1,146 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "CommonUtils/ConfigurableParam.h"
+#include "Framework/ConfigParamSpec.h"
+#include 
+#include "Framework/DeviceSpec.h"
+#include "Framework/WorkflowSpec.h"
+#include "Framework/Task.h"
+#include "FT0Base/Geometry.h"
+#include "DataFormatsFT0/ChannelData.h"
+#include "DataFormatsFT0/Digit.h"
+#include "DataFormatsFT0/DigitFilterParam.h"
+#include "CommonDataFormat/FlatHisto2D.h"
+
+using namespace o2::framework;
+
+namespace o2::ft0
+{
+
+class FT0TimeSpectraProcessor final : public o2::framework::Task
+{
+
+ public:
+  static constexpr int sNCHANNELS = Geometry::Nchannels;
+  static constexpr int sNCHANNELS_A = 4 * Geometry::NCellsA;
+
+  int mNbinsY{400};
+  float mMinY{-200.};
+  float mMaxY{200.};
+  int mAmpThreshold{10};
+  int mTimeWindow{153};
+  uint8_t mPMbitsToCheck{0b11111110};
+  uint8_t mPMbitsGood{0b01001000};
+  uint64_t mTrgBitsToCheck{0b11110000};
+  uint64_t mTrgBitsGood{0b10010000};
+  void init(o2::framework::InitContext& ic) final
+  {
+    mNbinsY = ic.options().get("number-bins");
+    mMinY = ic.options().get("low-edge");
+    mMaxY = ic.options().get("upper-edge");
+    LOG(info) << "Histogram parameters: " << mNbinsY << " " << mMinY << " " << mMaxY;
+    const auto& param = o2::ft0::DigitFilterParam::Instance();
+    param.printKeyValues();
+    mAmpThreshold = param.mAmpThreshold;
+    mTimeWindow = param.mTimeWindow;
+    mPMbitsGood = param.mPMbitsGood;
+    mPMbitsToCheck = param.mPMbitsToCheck;
+    mTrgBitsGood = param.mTrgBitsGood;
+    mTrgBitsToCheck = param.mTrgBitsToCheck;
+  }
+  void run(o2::framework::ProcessingContext& pc) final
+  {
+    enum { kTimeA = sNCHANNELS,
+           kTimeC = sNCHANNELS + 1,
+           kSumTimeAC = sNCHANNELS + 2,
+           kDiffTimeCA = sNCHANNELS + 3
+    };
+    const int nExtraSpectraSlots = 4; // For timeA/C sum and diff
+    auto digits = pc.inputs().get>("digits");
+    auto channels = pc.inputs().get>("channels");
+    o2::dataformats::FlatHisto2D timeSpectraInfoObject(sNCHANNELS + nExtraSpectraSlots, 0, sNCHANNELS + nExtraSpectraSlots, mNbinsY, mMinY, mMaxY);
+    for (const auto& digit : digits) {
+      const uint64_t trgWordExt = digit.mTriggers.getExtendedTrgWordFT0();
+      if ((trgWordExt & mTrgBitsToCheck) != mTrgBitsGood) {
+        continue;
+      }
+      const auto& chan = digit.getBunchChannelData(channels);
+      float timeA{0}, timeC{0};
+      int NchanA{0}, NchanC{0};
+      for (const auto& channel : chan) {
+        if (channel.QTCAmpl > mAmpThreshold && std::abs(channel.CFDTime) < mTimeWindow && ((channel.ChainQTC & mPMbitsToCheck) == mPMbitsGood && channel.ChId < sNCHANNELS)) {
+          const auto result = timeSpectraInfoObject.fill(channel.ChId, channel.CFDTime);
+          if (channel.ChId < sNCHANNELS_A) {
+            NchanA++;
+            timeA += channel.CFDTime;
+          } else {
+            NchanC++;
+            timeC += channel.CFDTime;
+          }
+        }
+      }
+      if (NchanA > 0) {
+        timeA = timeA / NchanA;
+        const auto result = timeSpectraInfoObject.fill(kTimeA, timeA);
+      }
+      if (NchanC > 0) {
+        timeC = timeC / NchanC;
+        const auto result = timeSpectraInfoObject.fill(kTimeC, timeC);
+        if (NchanA > 0) {
+          const auto resultSum = timeSpectraInfoObject.fill(kSumTimeAC, (timeA + timeC) / 2);
+          const auto resultDiff = timeSpectraInfoObject.fill(kDiffTimeCA, (timeC - timeA) / 2.);
+        }
+      }
+    }
+
+    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "TIME_SPECTRA", 0}, timeSpectraInfoObject.getBase());
+  }
+};
+
+} // namespace o2::ft0
+
+void customize(std::vector& workflowOptions)
+{
+  std::vector options;
+  options.push_back(ConfigParamSpec{"dispatcher-mode", VariantType::Bool, false, {"Dispatcher mode (FT0/SUB_DIGITSCH and FT0/SUB_DIGITSBC DPL channels should be applied as dispatcher output)."}});
+  options.push_back(ConfigParamSpec{"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}});
+  std::swap(workflowOptions, options);
+}
+
+#include "Framework/runDataProcessing.h"
+
+WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
+{
+  Inputs inputs{};
+  if (cfgc.options().get("dispatcher-mode")) {
+    inputs.push_back(InputSpec{{"channels"}, "FT0", "SUB_DIGITSCH"});
+    inputs.push_back(InputSpec{{"digits"}, "FT0", "SUB_DIGITSBC"});
+  } else {
+    inputs.push_back(InputSpec{{"channels"}, "FT0", "DIGITSCH"});
+    inputs.push_back(InputSpec{{"digits"}, "FT0", "DIGITSBC"});
+  }
+  o2::conf::ConfigurableParam::updateFromString(cfgc.options().get("configKeyValues"));
+  DataProcessorSpec dataProcessorSpec{
+    "FT0TimeSpectraProcessor",
+    inputs,
+    Outputs{
+      {{"timeSpectra"}, "FT0", "TIME_SPECTRA"}},
+    AlgorithmSpec{adaptFromTask()},
+    Options{{"number-bins", VariantType::Int, 400, {"Number of bins along Y-axis"}},
+            {"low-edge", VariantType::Float, -200.0f, {"Lower edge of first bin along Y-axis"}},
+            {"upper-edge", VariantType::Float, 200.0f, {"Upper edge of last bin along Y-axis"}}}};
+
+  WorkflowSpec workflow;
+  workflow.emplace_back(dataProcessorSpec);
+
+  return workflow;
+}
diff --git a/Detectors/FIT/FT0/dcsmonitoring/CMakeLists.txt b/Detectors/FIT/FT0/dcsmonitoring/CMakeLists.txt
new file mode 100644
index 0000000000000..3c8abdad32e6c
--- /dev/null
+++ b/Detectors/FIT/FT0/dcsmonitoring/CMakeLists.txt
@@ -0,0 +1,37 @@
+# Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+# See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+# All rights not expressly granted are reserved.
+#
+# This software is distributed under the terms of the GNU General Public
+# License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+#
+# In applying this license CERN does not waive the privileges and immunities
+# granted to it by virtue of its status as an Intergovernmental Organization
+# or submit itself to any jurisdiction.
+
+o2_add_library(FT0DCSMonitoring
+               SOURCES src/FT0DCSDataProcessor.cxx
+               PUBLIC_LINK_LIBRARIES O2::FITDCSMonitoring)
+
+o2_target_root_dictionary(FT0DCSMonitoring
+                          HEADERS include/FT0DCSMonitoring/FT0DCSConfigReader.h)
+
+o2_add_executable(dcs-sim-workflow
+                  COMPONENT_NAME ft0
+                  SOURCES workflow/ft0-dcs-sim-workflow.cxx
+                  PUBLIC_LINK_LIBRARIES O2::DCStestWorkflow)
+
+o2_add_executable(dcs-workflow
+                  COMPONENT_NAME ft0
+                  SOURCES workflow/ft0-dcs-data-workflow.cxx
+                  PUBLIC_LINK_LIBRARIES O2::DetectorsDCS
+                                        O2::Framework
+                                        O2::FT0DCSMonitoring)
+
+o2_add_executable(dcs-config-workflow
+                  COMPONENT_NAME ft0
+                  SOURCES workflow/ft0-dcs-config-workflow.cxx
+                  PUBLIC_LINK_LIBRARIES O2::DetectorsCalibration
+                                        O2::FITDCSMonitoring
+                                        O2::Framework
+                                        O2::FT0DCSMonitoring)
diff --git a/Detectors/FIT/FT0/dcsmonitoring/include/FT0DCSMonitoring/FT0DCSConfigReader.h b/Detectors/FIT/FT0/dcsmonitoring/include/FT0DCSMonitoring/FT0DCSConfigReader.h
new file mode 100644
index 0000000000000..8c8a216ac65b0
--- /dev/null
+++ b/Detectors/FIT/FT0/dcsmonitoring/include/FT0DCSMonitoring/FT0DCSConfigReader.h
@@ -0,0 +1,43 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file FT0DCSConfigReader.h
+/// \brief DCS configuration reader for FT0
+///
+/// \author Andreas Molander , University of Jyvaskyla, Finland
+
+#ifndef O2_FT0_DCSCONFIGREADER_H
+#define O2_FT0_DCSCONFIGREADER_H
+
+#include "FITDCSMonitoring/FITDCSConfigReader.h"
+#include "Rtypes.h"
+
+namespace o2
+{
+namespace ft0
+{
+
+/// DCS configuration reader for FT0
+///
+/// At the moment this class doesn't differ from the base class o2::fit::FITDCSConfigReader,
+/// which makes it obsolete. It exists only as an example for how to create detector specific
+/// DCS configuration readers later.
+class FT0DCSConfigReader : public o2::fit::FITDCSConfigReader
+{
+  // For FT0 specific processing of DCS configurations, override base class methods here.
+
+  ClassDefNV(FT0DCSConfigReader, 0);
+};
+
+} // namespace ft0
+} // namespace o2
+
+#endif // O2_FT0_DCSCONFIGREADER_H
\ No newline at end of file
diff --git a/Detectors/FIT/FT0/dcsmonitoring/include/FT0DCSMonitoring/FT0DCSDataProcessor.h b/Detectors/FIT/FT0/dcsmonitoring/include/FT0DCSMonitoring/FT0DCSDataProcessor.h
new file mode 100644
index 0000000000000..e52c2f6a0c522
--- /dev/null
+++ b/Detectors/FIT/FT0/dcsmonitoring/include/FT0DCSMonitoring/FT0DCSDataProcessor.h
@@ -0,0 +1,44 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// @file  FT0DCSDataProcessor.h
+/// @brief Task for processing FT0 DCS data
+///
+/// \author Andreas Molander , University of Jyvaskyla, Finland
+
+#ifndef O2_FT0_DATAPROCESSOR_H
+#define O2_FT0_DATAPROCESSOR_H
+
+#include "DetectorsDCS/DataPointIdentifier.h"
+#include "FITDCSMonitoring/FITDCSDataProcessor.h"
+
+#include 
+#include 
+
+namespace o2
+{
+namespace ft0
+{
+
+class FT0DCSDataProcessor : public o2::fit::FITDCSDataProcessor
+{
+ public:
+  FT0DCSDataProcessor(const std::string detectorName, const o2::header::DataDescription& dataDescription)
+    : o2::fit::FITDCSDataProcessor(detectorName, dataDescription) {}
+
+ protected:
+  std::vector getHardCodedDPIDs() override;
+}; // end class
+
+} // namespace ft0
+} // namespace o2
+
+#endif // O2_FT0_DATAPROCESSOR_H
diff --git a/Detectors/FIT/FT0/dcsmonitoring/macros/CMakeLists.txt b/Detectors/FIT/FT0/dcsmonitoring/macros/CMakeLists.txt
new file mode 100644
index 0000000000000..9d4391d8f4084
--- /dev/null
+++ b/Detectors/FIT/FT0/dcsmonitoring/macros/CMakeLists.txt
@@ -0,0 +1,15 @@
+# Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+# See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+# All rights not expressly granted are reserved.
+#
+# This software is distributed under the terms of the GNU General Public
+# License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+#
+# In applying this license CERN does not waive the privileges and immunities
+# granted to it by virtue of its status as an Intergovernmental Organization
+# or submit itself to any jurisdiction.
+
+o2_add_test_root_macro(makeFT0CCDBEntryForDCS.C
+                       PUBLIC_LINK_LIBRARIES O2::CCDB
+                                             O2::DetectorsDCS
+                       LABELS ft0)
diff --git a/Detectors/FIT/FT0/dcsmonitoring/macros/makeFT0CCDBEntryForDCS.C b/Detectors/FIT/FT0/dcsmonitoring/macros/makeFT0CCDBEntryForDCS.C
new file mode 100644
index 0000000000000..a5a1441d81726
--- /dev/null
+++ b/Detectors/FIT/FT0/dcsmonitoring/macros/makeFT0CCDBEntryForDCS.C
@@ -0,0 +1,103 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file makeFT0CCDBEntryForDCS.C
+/// \brief Macro for uploading a DCS data point definition object to CCDB
+///
+/// \author Andreas Molander , University of Jyvaskyla, Finland
+
+#include "CCDB/CcdbApi.h"
+#include "CCDB/CCDBTimeStampUtils.h"
+#include "DetectorsDCS/AliasExpander.h"
+#include "DetectorsDCS/DeliveryType.h"
+#include "DetectorsDCS/DataPointIdentifier.h"
+#include "TFile.h"
+
+#include 
+#include 
+#include 
+#include 
+
+using DPID = o2::dcs::DataPointIdentifier;
+
+int makeFT0CCDBEntryForDCS(const std::string ccdbUrl = "http://localhost:8080",
+                           const std::string fileName = "")
+{
+  std::unordered_map dpid2DataDesc;
+  std::vector aliasesHV = {"FT0/HV/FT0_A/MCP_A[1..5]/actual/iMon",
+                                        "FT0/HV/FT0_A/MCP_B[1..5]/actual/iMon",
+                                        "FT0/HV/FT0_A/MCP_C[1..2]/actual/iMon",
+                                        "FT0/HV/FT0_A/MCP_C[4..5]/actual/iMon",
+                                        "FT0/HV/FT0_A/MCP_D[1..5]/actual/iMon",
+                                        "FT0/HV/FT0_A/MCP_E[1..5]/actual/iMon",
+                                        "FT0/HV/FT0_C/MCP_A[2..5]/actual/iMon",
+                                        "FT0/HV/FT0_C/MCP_B[1..6]/actual/iMon",
+                                        "FT0/HV/FT0_C/MCP_C[1..2]/actual/iMon",
+                                        "FT0/HV/FT0_C/MCP_C[5..6]/actual/iMon",
+                                        "FT0/HV/FT0_C/MCP_D[1..2]/actual/iMon",
+                                        "FT0/HV/FT0_C/MCP_D[5..6]/actual/iMon",
+                                        "FT0/HV/FT0_C/MCP_E[1..6]/actual/iMon",
+                                        "FT0/HV/FT0_C/MCP_F[2..5]/actual/iMon",
+                                        "FT0/HV/MCP_LC/actual/iMon"};
+  std::string aliasesADC = "FT0/PM/channel[000..211]/actual/ADC[0..1]_BASELINE";
+  std::vector aliasesRates = {"FT0/Trigger1_Central/CNT_RATE",
+                                           "FT0/Trigger2_SemiCentral/CNT_RATE",
+                                           "FT0/Trigger3_Vertex/CNT_RATE",
+                                           "FT0/Trigger4_OrC/CNT_RATE",
+                                           "FT0/Trigger5_OrA/CNT_RATE",
+                                           "FT0/Background/[0..9]/CNT_RATE",
+                                           "FT0/Background/[A,B,C,D,E,F,G,H]/CNT_RATE",
+                                           "FT0/SecondaryCounter/CEplusSC/CNT_RATE"};
+
+  std::vector expAliasesHV = o2::dcs::expandAliases(aliasesHV);
+  std::vector expAliasesADC = o2::dcs::expandAlias(aliasesADC);
+  std::vector expAliasesRates = o2::dcs::expandAliases(aliasesRates);
+
+  LOG(info) << "DCS DP IDs:";
+
+  DPID dpIdTmp;
+  for (size_t i = 0; i < expAliasesHV.size(); i++) {
+    DPID::FILL(dpIdTmp, expAliasesHV[i], o2::dcs::DeliveryType::DPVAL_DOUBLE);
+    dpid2DataDesc[dpIdTmp] = "FT0DATAPOINTS";
+    LOG(info) << dpIdTmp;
+  }
+  for (size_t i = 0; i < expAliasesADC.size(); i++) {
+    DPID::FILL(dpIdTmp, expAliasesADC[i], o2::dcs::DeliveryType::DPVAL_UINT);
+    dpid2DataDesc[dpIdTmp] = "FT0DATAPOINTS";
+    LOG(info) << dpIdTmp;
+  }
+  for (size_t i = 0; i < expAliasesRates.size(); i++) {
+    DPID::FILL(dpIdTmp, expAliasesRates[i], o2::dcs::DeliveryType::DPVAL_DOUBLE);
+    dpid2DataDesc[dpIdTmp] = "FT0DATAPOINTS";
+    LOG(info) << dpIdTmp;
+  }
+
+  LOG(info) << "Total number of DPs: " << dpid2DataDesc.size();
+
+  if (!ccdbUrl.empty()) {
+    const std::string ccdbPath = "FT0/Config/DCSDPconfig";
+    LOGP(info, "Storing DCS DP definition object on {}/{}", ccdbUrl, ccdbPath);
+    o2::ccdb::CcdbApi api;
+    api.init(ccdbUrl);
+    std::map metadata;
+    long ts = o2::ccdb::getCurrentTimestamp();
+    api.storeAsTFileAny(&dpid2DataDesc, ccdbPath, metadata, ts, 99999999999999);
+  }
+
+  if (!fileName.empty()) {
+    LOG(info) << "Storing DCS DP definitions locally in " << fileName;
+    TFile file(fileName.c_str(), "recreate");
+    file.WriteObjectAny(&dpid2DataDesc, "std::unordered_map", "DCSDPconfig");
+    file.Close();
+  }
+
+  return 0;
+}
diff --git a/Detectors/FIT/FT0/dcsmonitoring/src/FT0DCSDataProcessor.cxx b/Detectors/FIT/FT0/dcsmonitoring/src/FT0DCSDataProcessor.cxx
new file mode 100644
index 0000000000000..c5049c3401365
--- /dev/null
+++ b/Detectors/FIT/FT0/dcsmonitoring/src/FT0DCSDataProcessor.cxx
@@ -0,0 +1,65 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// @file  FT0DCSDataProcessor.h
+/// @brief Task for processing FT0 DCS data
+///
+/// \author Andreas Molander , University of Jyvaskyla, Finland
+
+#include "FT0DCSMonitoring/FT0DCSDataProcessor.h"
+
+#include "DetectorsDCS/AliasExpander.h"
+#include "DetectorsDCS/DataPointIdentifier.h"
+
+#include 
+#include 
+
+std::vector o2::ft0::FT0DCSDataProcessor::getHardCodedDPIDs()
+{
+  std::vector vect;
+  std::vector aliasesHV = {"FT0/HV/FT0_A/MCP_A[1..5]/actual/iMon",
+                                        "FT0/HV/FT0_A/MCP_B[1..5]/actual/iMon",
+                                        "FT0/HV/FT0_A/MCP_C[1..2]/actual/iMon",
+                                        "FT0/HV/FT0_A/MCP_C[4..5]/actual/iMon",
+                                        "FT0/HV/FT0_A/MCP_D[1..5]/actual/iMon",
+                                        "FT0/HV/FT0_A/MCP_E[1..5]/actual/iMon",
+                                        "FT0/HV/FT0_C/MCP_A[2..5]/actual/iMon",
+                                        "FT0/HV/FT0_C/MCP_B[1..6]/actual/iMon",
+                                        "FT0/HV/FT0_C/MCP_C[1..2]/actual/iMon",
+                                        "FT0/HV/FT0_C/MCP_C[5..6]/actual/iMon",
+                                        "FT0/HV/FT0_C/MCP_D[1..2]/actual/iMon",
+                                        "FT0/HV/FT0_C/MCP_D[5..6]/actual/iMon",
+                                        "FT0/HV/FT0_C/MCP_E[1..6]/actual/iMon",
+                                        "FT0/HV/FT0_C/MCP_F[2..5]/actual/iMon",
+                                        "FT0/HV/MCP_LC/actual/iMon"};
+  std::string aliasesADC = "FT0/PM/channel[000..211]/actual/ADC[0..1]_BASELINE";
+  std::vector aliasesRates = {"FT0/Trigger1_Central/CNT_RATE",
+                                           "FT0/Trigger2_SemiCentral/CNT_RATE",
+                                           "FT0/Trigger3_Vertex/CNT_RATE",
+                                           "FT0/Trigger4_OrC/CNT_RATE",
+                                           "FT0/Trigger5_OrA/CNT_RATE",
+                                           "FT0/Background/[0..9]/CNT_RATE",
+                                           "FT0/Background/[A,B,C,D,E,F,G,H]/CNT_RATE",
+                                           "FT0/SecondaryCounter/CEplusSC/CNT_RATE"};
+  std::vector expAliasesHV = o2::dcs::expandAliases(aliasesHV);
+  std::vector expAliasesADC = o2::dcs::expandAlias(aliasesADC);
+  std::vector expAliasesRates = o2::dcs::expandAliases(aliasesRates);
+  for (const auto& i : expAliasesHV) {
+    vect.emplace_back(i, o2::dcs::DPVAL_DOUBLE);
+  }
+  for (const auto& i : expAliasesADC) {
+    vect.emplace_back(i, o2::dcs::DPVAL_UINT);
+  }
+  for (const auto& i : expAliasesRates) {
+    vect.emplace_back(i, o2::dcs::DPVAL_DOUBLE);
+  }
+  return vect;
+}
diff --git a/Detectors/FIT/FT0/dcsmonitoring/src/FT0DCSMonitoringLinkDef.h b/Detectors/FIT/FT0/dcsmonitoring/src/FT0DCSMonitoringLinkDef.h
new file mode 100644
index 0000000000000..c85dd2d378ccb
--- /dev/null
+++ b/Detectors/FIT/FT0/dcsmonitoring/src/FT0DCSMonitoringLinkDef.h
@@ -0,0 +1,18 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifdef __CLING__
+
+#pragma link off all globals;
+#pragma link off all classes;
+#pragma link off all functions;
+
+#endif
diff --git a/Detectors/FIT/FT0/dcsmonitoring/workflow/FT0DCSConfigProcessorSpec.h b/Detectors/FIT/FT0/dcsmonitoring/workflow/FT0DCSConfigProcessorSpec.h
new file mode 100644
index 0000000000000..fe6701344530b
--- /dev/null
+++ b/Detectors/FIT/FT0/dcsmonitoring/workflow/FT0DCSConfigProcessorSpec.h
@@ -0,0 +1,75 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file FT0DCSConfigProcessorSpec.cxx
+/// \brief FT0 processor spec for DCS configurations
+///
+/// \author Andreas Molander , University of Jyvaskyla, Finland
+
+#ifndef O2_FT0_DCSCONFIGPROCESSOR_H
+#define O2_FT0_DCSCONFIGPROCESSOR_H
+
+#include "FITDCSMonitoring/FITDCSConfigProcessorSpec.h"
+#include "FT0DCSMonitoring/FT0DCSConfigReader.h"
+#include "DetectorsCalibration/Utils.h"
+#include "Framework/WorkflowSpec.h"
+#include "Headers/DataHeader.h"
+
+#include 
+#include 
+
+namespace o2
+{
+namespace ft0
+{
+
+class FT0DCSConfigProcessor : public o2::fit::FITDCSConfigProcessor
+{
+  // Example of how to use another DCS config reader (subclass of o2::fit::FITDCSConfigReader)
+ public:
+  FT0DCSConfigProcessor(const std::string& detectorName, const o2::header::DataDescription& dataDescriptionDChM)
+    : o2::fit::FITDCSConfigProcessor(detectorName, dataDescriptionDChM) {}
+
+ protected:
+  void initDCSConfigReader() override
+  {
+    mDCSConfigReader = std::make_unique(FT0DCSConfigReader());
+  }
+};
+
+} // namespace ft0
+
+namespace framework
+{
+
+DataProcessorSpec getFT0DCSConfigProcessorSpec()
+{
+  o2::header::DataDescription ddDChM = "FT0_DCHM";
+  std::vector outputs;
+  outputs.emplace_back(ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBPayload, ddDChM}, Lifetime::Sporadic);
+  outputs.emplace_back(ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBWrapper, ddDChM}, Lifetime::Sporadic);
+
+  return DataProcessorSpec{
+    "ft0-dcs-config-processor",
+    Inputs{{"inputConfig", o2::header::gDataOriginFT0, "DCS_CONFIG_FILE", Lifetime::Sporadic},
+           {"inputConfigFileName", o2::header::gDataOriginFT0, "DCS_CONFIG_NAME", Lifetime::Sporadic}},
+    outputs,
+    AlgorithmSpec{adaptFromTask("FT0", ddDChM)},
+    Options{{"use-verbose-mode", VariantType::Bool, false, {"Use verbose mode"}},
+            {"filename-dchm", VariantType::String, "FT0-deadchannels.txt", {"Dead channel map file name"}},
+            {"valid-days-dchm", VariantType::UInt32, 180u, {"Dead channel map validity in days"}},
+            {"no-validate", VariantType::Bool, false, {"Don't validate the CCDB uploads"}}}};
+}
+
+} // namespace framework
+} // namespace o2
+
+#endif // O2_FT0_DCSCONFIGPROCESSOR_H
diff --git a/Detectors/FIT/FT0/dcsmonitoring/workflow/FT0DCSDataProcessorSpec.h b/Detectors/FIT/FT0/dcsmonitoring/workflow/FT0DCSDataProcessorSpec.h
new file mode 100644
index 0000000000000..86b342d3f4bb1
--- /dev/null
+++ b/Detectors/FIT/FT0/dcsmonitoring/workflow/FT0DCSDataProcessorSpec.h
@@ -0,0 +1,51 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// @file  FT0DCSDataProcessorSpec.h
+/// @brief DataProcessorSpec for FT0 DCS data
+///
+/// \author Andreas Molander , University of Jyvaskyla, Finland
+
+#ifndef O2_FT0_DATAPROCESSORSPEC_H
+#define O2_FT0_DATAPROCESSORSPEC_H
+
+#include "DetectorsCalibration/Utils.h"
+#include "FT0DCSMonitoring/FT0DCSDataProcessor.h"
+#include "Framework/Task.h"
+#include "Framework/WorkflowSpec.h"
+
+namespace o2
+{
+namespace framework
+{
+
+DataProcessorSpec getFT0DCSDataProcessorSpec()
+{
+  o2::header::DataDescription ddDCSDPs = "FT0_DCSDPs";
+  std::vector outputs;
+  outputs.emplace_back(ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBPayload, ddDCSDPs}, Lifetime::Sporadic);
+  outputs.emplace_back(ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBWrapper, ddDCSDPs}, Lifetime::Sporadic);
+
+  return DataProcessorSpec{
+    "ft0-dcs-data-processor",
+    Inputs{{"input", "DCS", "FT0DATAPOINTS"}},
+    outputs,
+    AlgorithmSpec{adaptFromTask("FT0", ddDCSDPs)},
+    Options{{"ccdb-path", VariantType::String, "http://localhost:8080", {"Path to CCDB"}},
+            {"use-ccdb-to-configure", VariantType::Bool, false, {"Use CCDB to configure"}},
+            {"use-verbose-mode", VariantType::Bool, false, {"Use verbose mode"}},
+            {"DPs-update-interval", VariantType::Int64, 600ll, {"Interval (in s) after which to update the DPs CCDB entry"}}}};
+}
+
+} // namespace framework
+} // namespace o2
+
+#endif // O2_FT0_DATAPROCESSORSPEC_H
diff --git a/Detectors/FIT/FT0/dcsmonitoring/workflow/ft0-dcs-config-workflow.cxx b/Detectors/FIT/FT0/dcsmonitoring/workflow/ft0-dcs-config-workflow.cxx
new file mode 100644
index 0000000000000..e73f484a9808f
--- /dev/null
+++ b/Detectors/FIT/FT0/dcsmonitoring/workflow/ft0-dcs-config-workflow.cxx
@@ -0,0 +1,35 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file ft0-dcs-config-workflow.cxx
+/// \brief Workflow for FT0 DCS configuration processing
+///
+/// \author Andreas Molander , University of Jyvaskyla, Finland
+
+#include "Framework/DataProcessorSpec.h"
+#include "FT0DCSConfigProcessorSpec.h"
+
+using namespace o2::framework;
+
+// we need to add workflow options before including Framework/runDataProcessing.h
+void customize(std::vector& workflowOptions)
+{
+  // option allowing to set parameters
+}
+
+#include "Framework/runDataProcessing.h"
+
+WorkflowSpec defineDataProcessing(o2::framework::ConfigContext const& configcontext)
+{
+  WorkflowSpec specs;
+  specs.emplace_back(getFT0DCSConfigProcessorSpec());
+  return specs;
+}
diff --git a/Detectors/FIT/FT0/calibration/testWorkflow/ft0-dcs-data-workflow.cxx b/Detectors/FIT/FT0/dcsmonitoring/workflow/ft0-dcs-data-workflow.cxx
similarity index 86%
rename from Detectors/FIT/FT0/calibration/testWorkflow/ft0-dcs-data-workflow.cxx
rename to Detectors/FIT/FT0/dcsmonitoring/workflow/ft0-dcs-data-workflow.cxx
index 5e4306c387c8f..935e523e24291 100644
--- a/Detectors/FIT/FT0/calibration/testWorkflow/ft0-dcs-data-workflow.cxx
+++ b/Detectors/FIT/FT0/dcsmonitoring/workflow/ft0-dcs-data-workflow.cxx
@@ -9,18 +9,26 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 
+/// \file ft0-dcs-data-workflow.cxx
+/// \brief Workflow for FT0 DCS data processing
+///
+/// \author Andreas Molander , University of Jyvaskyla, Finland
+
 #include "DetectorsDCS/DataPointIdentifier.h"
 #include "DetectorsDCS/DataPointValue.h"
+#include "FT0DCSDataProcessorSpec.h"
 #include "Framework/TypeTraits.h"
+
 #include 
+
 namespace o2::framework
 {
 template <>
 struct has_root_dictionary, void> : std::true_type {
 };
 } // namespace o2::framework
+
 #include "Framework/DataProcessorSpec.h"
-#include "FT0DCSDataProcessorSpec.h"
 
 using namespace o2::framework;
 
diff --git a/Detectors/FIT/FT0/dcsmonitoring/workflow/ft0-dcs-sim-workflow.cxx b/Detectors/FIT/FT0/dcsmonitoring/workflow/ft0-dcs-sim-workflow.cxx
new file mode 100644
index 0000000000000..58802f0e51d16
--- /dev/null
+++ b/Detectors/FIT/FT0/dcsmonitoring/workflow/ft0-dcs-sim-workflow.cxx
@@ -0,0 +1,51 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file ft0-dcs-sim-workflow.cxx
+/// \brief Simulate DCS data for FT0
+///
+/// \author Andreas Molander , University of Jyvaskyla, Finland
+
+#include "DCStestWorkflow/DCSRandomDataGeneratorSpec.h"
+#include "Framework/runDataProcessing.h"
+
+o2::framework::WorkflowSpec defineDataProcessing(o2::framework::ConfigContext const& configcontext)
+{
+  std::vector dphints;
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/HV/FT0_A/MCP_A[1..5]/actual/iMon", 250, 350});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/HV/FT0_A/MCP_B[1..5]/actual/iMon", 250, 350});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/HV/FT0_A/MCP_C[1..2]/actual/iMon", 250, 350});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/HV/FT0_A/MCP_C[4..5]/actual/iMon", 250, 350});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/HV/FT0_A/MCP_D[1..5]/actual/iMon", 250, 350});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/HV/FT0_A/MCP_E[1..5]/actual/iMon", 250, 350});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/HV/FT0_C/MCP_A[2..5]/actual/iMon", 250, 350});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/HV/FT0_C/MCP_B[1..6]/actual/iMon", 250, 350});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/HV/FT0_C/MCP_C[1..2]/actual/iMon", 250, 350});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/HV/FT0_C/MCP_C[5..6]/actual/iMon", 250, 350});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/HV/FT0_C/MCP_D[1..2]/actual/iMon", 250, 350});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/HV/FT0_C/MCP_D[5..6]/actual/iMon", 250, 350});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/HV/FT0_C/MCP_E[1..6]/actual/iMon", 250, 350});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/HV/FT0_C/MCP_F[2..5]/actual/iMon", 250, 350});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/HV/MCP_LC/actual/iMon", 250, 350});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/PM/channel[000..211]/actual/ADC[0..1]_BASELINE", 30, 150});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/Trigger1_Central/CNT_RATE", 0, 5000000});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/Trigger2_SemiCentral/CNT_RATE", 0, 5000000});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/Trigger3_Vertex/CNT_RATE", 0, 5000000});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/Trigger4_OrC/CNT_RATE", 0, 5000000});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/Trigger5_OrA/CNT_RATE", 0, 5000000});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/Background/[0..9]/CNT_RATE", 0, 50000});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/Background/[A,B,C,D,E,F,G,H]/CNT_RATE", 0, 50000});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FT0/SecondaryCounter/CEplusSC/CNT_RATE", 0, 5000000});
+
+  o2::framework::WorkflowSpec specs;
+  specs.emplace_back(o2::dcs::test::getDCSRandomDataGeneratorSpec(dphints, "FT0"));
+  return specs;
+}
diff --git a/Detectors/FIT/FT0/macros/CMakeLists.txt b/Detectors/FIT/FT0/macros/CMakeLists.txt
index c4ed27d2513ba..17491ca4962c1 100644
--- a/Detectors/FIT/FT0/macros/CMakeLists.txt
+++ b/Detectors/FIT/FT0/macros/CMakeLists.txt
@@ -1,14 +1,21 @@
-# Copyright CERN and copyright holders of ALICE O2. This software is distributed
-# under the terms of the GNU General Public License v3 (GPL Version 3), copied
-# verbatim in the file "COPYING".
+# Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+# See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+# All rights not expressly granted are reserved.
 #
-# See http://alice-o2.web.cern.ch/license for full licensing information.
+# This software is distributed under the terms of the GNU General Public
+# License v3 (GPL Version 3), copied verbatim in the file "COPYING".
 #
 # In applying this license CERN does not waive the privileges and immunities
-# granted to it by virtue of its status as an Intergovernmental Organization or
-# submit itself to any jurisdiction.
+# granted to it by virtue of its status as an Intergovernmental Organization
+# or submit itself to any jurisdiction.
 
 o2_add_test_root_macro(FT0Misaligner.C
                        PUBLIC_LINK_LIBRARIES O2::CCDB
                        O2::FT0Simulation
                        LABELS ft0)
+
+o2_add_test_root_macro(FT0readEventsPerBc.C
+                       PUBLIC_LINK_LIBRARIES
+                       O2::CCDB
+                       O2::DataFormatsFT0
+                       LABELS ft0)
diff --git a/Detectors/FIT/FT0/macros/FT0Misaligner.C b/Detectors/FIT/FT0/macros/FT0Misaligner.C
index 7585411066934..16476ae3b8ccc 100644
--- a/Detectors/FIT/FT0/macros/FT0Misaligner.C
+++ b/Detectors/FIT/FT0/macros/FT0Misaligner.C
@@ -1,15 +1,30 @@
+// Copyright 2021-2025 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file  FT0Misaligner.C
+/// \brief ROOT macro for creating an FT0 geometry alignment object. Based on ITSMisaligner.C
+///
+/// \author Andreas Molander andreas.molander@cern.ch, Alla Maevskaya
+
 #if !defined(__CLING__) || defined(__ROOTCLING__)
-//#define ENABLE_UPGRADES
+
+#include "CCDB/CcdbApi.h"
 #include "DetectorsCommonDataFormats/DetID.h"
 #include "DetectorsCommonDataFormats/DetectorNameConf.h"
 #include "DetectorsCommonDataFormats/AlignParam.h"
-#include "DetectorsBase/GeometryManager.h"
-#include "CCDB/CcdbApi.h"
-#include "FT0Base/Geometry.h"
-#include 
+
 #include 
 #include 
 #include 
+
 #endif
 
 using AlgPar = std::array;
@@ -23,19 +38,15 @@ void FT0Misaligner(const std::string& ccdbHost = "http://ccdb-test.cern.ch:8080"
                    const std::string& fileName = "FT0Alignment.root")
 {
   std::vector params;
-  o2::base::GeometryManager::loadGeometry("", false);
-  //  auto geom = o2::ft0::Geometry::Instance();
   AlgPar pars;
   bool glo = true;
 
   o2::detectors::DetID detFT0("FT0");
 
-  // FT0 detector
-  //set A side
   std::string symNameA = "FT0A";
   pars = generateMisalignment(xA, yA, zA, psiA, thetaA, phiA);
   params.emplace_back(symNameA.c_str(), -1, pars[0], pars[1], pars[2], pars[3], pars[4], pars[5], glo);
-  //set C side
+
   std::string symNameC = "FT0C";
   pars = generateMisalignment(xC, yC, zC, psiC, thetaC, phiC);
   params.emplace_back(symNameC.c_str(), -1, pars[0], pars[1], pars[2], pars[3], pars[4], pars[5], glo);
@@ -44,7 +55,7 @@ void FT0Misaligner(const std::string& ccdbHost = "http://ccdb-test.cern.ch:8080"
     std::string path = objectPath.empty() ? o2::base::DetectorNameConf::getAlignmentPath(detFT0) : objectPath;
     LOGP(info, "Storing alignment object on {}/{}", ccdbHost, path);
     o2::ccdb::CcdbApi api;
-    map metadata; // can be empty
+    std::map metadata; // can be empty
     api.init(ccdbHost.c_str());   // or http://localhost:8080 for a local installation
     // store abitrary user object in strongly typed manner
     api.storeAsTFileAny(¶ms, path, metadata, tmin, tmax);
@@ -57,14 +68,15 @@ void FT0Misaligner(const std::string& ccdbHost = "http://ccdb-test.cern.ch:8080"
     algFile.Close();
   }
 }
+
 AlgPar generateMisalignment(double x, double y, double z, double psi, double theta, double phi)
 {
   AlgPar pars;
-  pars[0] = gRandom->Gaus(0, x);
-  pars[1] = gRandom->Gaus(0, y);
-  pars[2] = gRandom->Gaus(0, z);
-  pars[3] = gRandom->Gaus(0, psi);
-  pars[4] = gRandom->Gaus(0, theta);
-  pars[5] = gRandom->Gaus(0, phi);
+  pars[0] = x;
+  pars[1] = y;
+  pars[2] = z;
+  pars[3] = psi;
+  pars[4] = theta;
+  pars[5] = phi;
   return std::move(pars);
 }
diff --git a/Detectors/FIT/FT0/macros/FT0readEventsPerBc.C b/Detectors/FIT/FT0/macros/FT0readEventsPerBc.C
new file mode 100644
index 0000000000000..c6afc86389b9b
--- /dev/null
+++ b/Detectors/FIT/FT0/macros/FT0readEventsPerBc.C
@@ -0,0 +1,52 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#if !defined(__CLING__) || defined(__ROOTCLING__)
+#include 
+#include 
+#endif
+
+#include "CCDB/CcdbApi.h"
+#include "CCDB/CCDBTimeStampUtils.h"
+#include "TH1F.h"
+#include "DataFormatsFT0/EventsPerBc.h"
+#include "Framework/Logger.h"
+#include "CommonConstants/LHCConstants.h"
+
+std::unique_ptr hist;
+std::unique_ptr canvas;
+
+void FT0readEventsPerBc(std::string ccdbUrl, long timestamp)
+{
+  o2::ccdb::CcdbApi ccdbApi;
+  ccdbApi.init(ccdbUrl);
+  const std::string ccdbPath = "FT0/Calib/EventsPerBc";
+  std::map metadata;
+
+  if (timestamp < 0) {
+    timestamp = o2::ccdb::getCurrentTimestamp();
+  }
+
+  std::unique_ptr events(ccdbApi.retrieveFromTFileAny(ccdbPath, metadata, timestamp));
+
+  if (!events) {
+    LOGP(fatal, "EventsPerBc object not found in {}/{} for timestamp {}.", ccdbUrl, ccdbPath, timestamp);
+    return;
+  }
+
+  hist = std::make_unique("eventsPerBcHist", "Events per BC", o2::constants::lhc::LHCMaxBunches, 0, o2::constants::lhc::LHCMaxBunches - 1);
+  for (int idx = 0; idx < o2::constants::lhc::LHCMaxBunches; idx++) {
+    hist->Fill(idx, events->histogram[idx]);
+  }
+  canvas = std::make_unique();
+  hist->Draw();
+  canvas->Draw();
+}
\ No newline at end of file
diff --git a/Detectors/FIT/FT0/raw/include/FT0Raw/DataBlockFT0.h b/Detectors/FIT/FT0/raw/include/FT0Raw/DataBlockFT0.h
index ba9ff367794be..6275d80856cb5 100644
--- a/Detectors/FIT/FT0/raw/include/FT0Raw/DataBlockFT0.h
+++ b/Detectors/FIT/FT0/raw/include/FT0Raw/DataBlockFT0.h
@@ -9,7 +9,7 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 //
-//file DataBlockFT0.h class  for RAW data format data blocks at FT0
+// file DataBlockFT0.h class  for RAW data format data blocks at FT0
 //
 // Artur.Furs
 // afurs@cern.ch
@@ -23,17 +23,17 @@ namespace o2
 {
 namespace ft0
 {
-//Raw event data for FT0
+// Raw event data for FT0
 using RawHeaderPM = o2::ft0::EventHeader;
 using RawDataPM = o2::ft0::EventData;
 using RawHeaderTCM = o2::ft0::EventHeader;
 using RawDataTCM = o2::ft0::TCMdata;
 using RawHeaderTCMext = o2::ft0::EventHeader;
 using RawDataTCMext = o2::ft0::TCMdataExtended;
-//Data block for FT0 modules
-using DataBlockPM = o2::fit::DataBlockPM;
-using DataBlockTCM = o2::fit::DataBlockTCM;
-using DataBlockTCMext = o2::fit::DataBlockTCMext;
+// Data block for FT0 modules, no padding - DataBlockConfig
+using DataBlockPM = o2::fit::DataBlockPM, RawHeaderPM, RawDataPM>;
+using DataBlockTCM = o2::fit::DataBlockTCM, RawHeaderTCM, RawDataTCM>;
+using DataBlockTCMext = o2::fit::DataBlockTCMext, RawHeaderTCMext, RawDataTCM, RawDataTCMext>;
 } // namespace ft0
 } // namespace o2
 #endif
diff --git a/Detectors/FIT/FT0/raw/include/FT0Raw/DigitBlockFT0.h b/Detectors/FIT/FT0/raw/include/FT0Raw/DigitBlockFT0.h
index 716d6795e5ad2..84d2d1a0bcd3a 100644
--- a/Detectors/FIT/FT0/raw/include/FT0Raw/DigitBlockFT0.h
+++ b/Detectors/FIT/FT0/raw/include/FT0Raw/DigitBlockFT0.h
@@ -9,7 +9,7 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 //
-//file DigitBlockFT0.h class  for proccessing RAW data into Digits
+// file DigitBlockFT0.h class  for proccessing RAW data into Digits
 //
 // Artur.Furs
 // afurs@cern.ch
@@ -25,10 +25,10 @@ namespace o2
 {
 namespace ft0
 {
-//Normal data taking mode
-using DigitBlockFT0 = DigitBlockFIT;
-//TCM extended data taking mode
-using DigitBlockFT0ext = DigitBlockFIText;
+// Normal data taking mode
+using DigitBlockFT0 = o2::fit::DigitBlockFIT;
+// TCM extended data taking mode
+using DigitBlockFT0ext = o2::fit::DigitBlockFIText;
 } // namespace ft0
 } // namespace o2
 #endif
diff --git a/Detectors/FIT/FT0/raw/include/FT0Raw/RawWriterFT0.h b/Detectors/FIT/FT0/raw/include/FT0Raw/RawWriterFT0.h
index 951fe42cc11fc..e9b8a83407b20 100644
--- a/Detectors/FIT/FT0/raw/include/FT0Raw/RawWriterFT0.h
+++ b/Detectors/FIT/FT0/raw/include/FT0Raw/RawWriterFT0.h
@@ -9,7 +9,7 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 //
-//file RawWriterFT0.h Raw writer class for FT0
+// file RawWriterFT0.h Raw writer class for FT0
 //
 // Artur.Furs
 // afurs@cern.ch
@@ -24,10 +24,12 @@ namespace o2
 {
 namespace ft0
 {
-//Normal TCM mode
+// Normal TCM mode
 using RawWriterFT0 = o2::fit::RawWriterFIT;
-//Extended TCM mode
-//using RawWriterFT0ext = o2::fit::RawWriterFIT;
+using RawWriterFT0_padded = o2::fit::RawWriterFIT;
+
+// Extended TCM mode
+// using RawWriterFT0ext = o2::fit::RawWriterFIT;
 } // namespace ft0
 } // namespace o2
 
diff --git a/Detectors/FIT/FT0/raw/src/DataBlockFT0.cxx b/Detectors/FIT/FT0/raw/src/DataBlockFT0.cxx
index f08f0bc1042db..69be24e0e0f9b 100644
--- a/Detectors/FIT/FT0/raw/src/DataBlockFT0.cxx
+++ b/Detectors/FIT/FT0/raw/src/DataBlockFT0.cxx
@@ -10,3 +10,7 @@
 // or submit itself to any jurisdiction.
 
 #include "FT0Raw/DataBlockFT0.h"
+
+template class o2::fit::DataBlockPM, o2::ft0::RawHeaderPM, o2::ft0::RawDataPM>;
+template class o2::fit::DataBlockTCM, o2::ft0::RawHeaderTCM, o2::ft0::RawDataTCM>;
+template class o2::fit::DataBlockTCMext, o2::ft0::RawHeaderTCMext, o2::ft0::RawDataTCM, o2::ft0::RawDataTCMext>;
diff --git a/Detectors/FIT/FT0/raw/src/DigitBlockFT0.cxx b/Detectors/FIT/FT0/raw/src/DigitBlockFT0.cxx
index b474afff032bd..80ecb470f0a62 100644
--- a/Detectors/FIT/FT0/raw/src/DigitBlockFT0.cxx
+++ b/Detectors/FIT/FT0/raw/src/DigitBlockFT0.cxx
@@ -10,3 +10,6 @@
 // or submit itself to any jurisdiction.
 
 #include "FT0Raw/DigitBlockFT0.h"
+
+template class o2::fit::DigitBlockFIT;
+template class o2::fit::DigitBlockFIText;
\ No newline at end of file
diff --git a/Detectors/FIT/FT0/raw/src/RawReaderFT0Base.cxx b/Detectors/FIT/FT0/raw/src/RawReaderFT0Base.cxx
index dea616c706ccc..1e5984c5aea3e 100644
--- a/Detectors/FIT/FT0/raw/src/RawReaderFT0Base.cxx
+++ b/Detectors/FIT/FT0/raw/src/RawReaderFT0Base.cxx
@@ -10,3 +10,6 @@
 // or submit itself to any jurisdiction.
 
 #include "FT0Raw/RawReaderFT0Base.h"
+
+template class o2::fit::RawReaderBaseFIT;
+template class o2::fit::RawReaderBaseFIT;
diff --git a/Detectors/FIT/FT0/raw/src/RawWriterFT0.cxx b/Detectors/FIT/FT0/raw/src/RawWriterFT0.cxx
index cd57b0ea4fe0e..f563e9de8e1fc 100644
--- a/Detectors/FIT/FT0/raw/src/RawWriterFT0.cxx
+++ b/Detectors/FIT/FT0/raw/src/RawWriterFT0.cxx
@@ -10,3 +10,6 @@
 // or submit itself to any jurisdiction.
 
 #include "FT0Raw/RawWriterFT0.h"
+
+template class o2::fit::RawWriterFIT;
+template class o2::fit::RawWriterFIT;
diff --git a/Detectors/FIT/FT0/reconstruction/CMakeLists.txt b/Detectors/FIT/FT0/reconstruction/CMakeLists.txt
index e76155eebd158..9a4a14af6aca6 100644
--- a/Detectors/FIT/FT0/reconstruction/CMakeLists.txt
+++ b/Detectors/FIT/FT0/reconstruction/CMakeLists.txt
@@ -25,8 +25,7 @@ o2_add_library(FT0Reconstruction
                                      O2::DetectorsCalibration)
 
 o2_target_root_dictionary(FT0Reconstruction
-                          HEADERS include/FT0Reconstruction/CollisionTimeRecoTask.h
-                                  include/FT0Reconstruction/InteractionTag.h)
+                          HEADERS include/FT0Reconstruction/InteractionTag.h)
 
 o2_add_executable(
   test-raw-conversion
diff --git a/Detectors/FIT/FT0/reconstruction/include/FT0Reconstruction/CTFCoder.h b/Detectors/FIT/FT0/reconstruction/include/FT0Reconstruction/CTFCoder.h
index 7ea19e99b753e..5dc367204e1a3 100644
--- a/Detectors/FIT/FT0/reconstruction/include/FT0Reconstruction/CTFCoder.h
+++ b/Detectors/FIT/FT0/reconstruction/include/FT0Reconstruction/CTFCoder.h
@@ -26,7 +26,6 @@
 #include "DetectorsCommonDataFormats/DetID.h"
 #include "FT0Base/FT0DigParam.h"
 #include "DetectorsBase/CTFCoderBase.h"
-#include "rANS/rans.h"
 
 class TTree;
 
@@ -35,10 +34,10 @@ namespace o2
 namespace ft0
 {
 
-class CTFCoder : public o2::ctf::CTFCoderBase
+class CTFCoder final : public o2::ctf::CTFCoderBase
 {
  public:
-  CTFCoder(o2::ctf::CTFCoderBase::OpType op) : o2::ctf::CTFCoderBase(op, CTF::getNBlocks(), o2::detectors::DetID::FT0) {}
+  CTFCoder(o2::ctf::CTFCoderBase::OpType op, const std::string& ctfdictOpt = "none") : o2::ctf::CTFCoderBase(op, CTF::getNBlocks(), o2::detectors::DetID::FT0, 1.f, ctfdictOpt) {}
   ~CTFCoder() final = default;
 
   /// entropy-encode digits to buffer with CTF
@@ -73,15 +72,15 @@ o2::ctf::CTFIOSize CTFCoder::encode(VEC& buff, const gsl::span& dig
   using MD = o2::ctf::Metadata::OptStore;
   // what to do which each field: see o2::ctd::Metadata explanation
   constexpr MD optField[CTF::getNBlocks()] = {
-    MD::EENCODE, // BLC_trigger
-    MD::EENCODE, // BLC_bcInc
-    MD::EENCODE, // BLC_orbitInc
-    MD::EENCODE, // BLC_nChan
-    MD::EENCODE, // BLC_status
-    MD::EENCODE, // BLC_idChan
-    MD::EENCODE, // BLC_qtcChain
-    MD::EENCODE, // BLC_cfdTime
-    MD::EENCODE  // BLC_qtcAmpl
+    MD::EENCODE_OR_PACK, // BLC_trigger
+    MD::EENCODE_OR_PACK, // BLC_bcInc
+    MD::EENCODE_OR_PACK, // BLC_orbitInc
+    MD::EENCODE_OR_PACK, // BLC_nChan
+    MD::EENCODE_OR_PACK, // BLC_status
+    MD::EENCODE_OR_PACK, // BLC_idChan
+    MD::EENCODE_OR_PACK, // BLC_qtcChain
+    MD::EENCODE_OR_PACK, // BLC_cfdTime
+    MD::EENCODE_OR_PACK  // BLC_qtcAmpl
   };
   CompressedDigits cd;
   if (mExtHeader.isValidDictTimeStamp()) {
@@ -102,11 +101,10 @@ o2::ctf::CTFIOSize CTFCoder::encode(VEC& buff, const gsl::span& dig
 
   ec->setHeader(cd.header);
   assignDictVersion(static_cast(ec->getHeader()));
-  ec->getANSHeader().majorVersion = 0;
-  ec->getANSHeader().minorVersion = 1;
+  ec->setANSHeader(mANSVersion);
   // at every encoding the buffer might be autoexpanded, so we don't work with fixed pointer ec
   o2::ctf::CTFIOSize iosize;
-#define ENCODEFT0(part, slot, bits) CTF::get(buff.data())->encode(part, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)].get(), getMemMarginFactor());
+#define ENCODEFT0(part, slot, bits) CTF::get(buff.data())->encode(part, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)], getMemMarginFactor());
   // clang-format off
   iosize += ENCODEFT0(cd.trigger,     CTF::BLC_trigger,  0);
   iosize += ENCODEFT0(cd.bcInc,       CTF::BLC_bcInc,    0);
@@ -134,7 +132,7 @@ o2::ctf::CTFIOSize CTFCoder::decode(const CTF::base& ec, VDIG& digitVec, VCHAN&
   checkDictVersion(hd);
   ec.print(getPrefix(), mVerbosity);
   o2::ctf::CTFIOSize iosize;
-#define DECODEFT0(part, slot) ec.decode(part, int(slot), mCoders[int(slot)].get())
+#define DECODEFT0(part, slot) ec.decode(part, int(slot), mCoders[int(slot)])
   // clang-format off
   iosize += DECODEFT0(cd.trigger,     CTF::BLC_trigger);
   iosize += DECODEFT0(cd.bcInc,       CTF::BLC_bcInc);
@@ -232,12 +230,31 @@ void CTFCoder::compress(CompressedDigits& cd, const gsl::span& digi
 {
   // convert digits/channel to their compressed version
   cd.clear();
+  cd.header.det = mDet;
   if (!digitVec.size()) {
     return;
   }
-  const auto& dig0 = digitVec[0];
-  cd.header.det = mDet;
-  cd.header.nTriggers = digitVec.size();
+  uint32_t firstDig = digitVec.size(), nDigSel = digitVec.size(), nChanSel = channelVec.size();
+  std::vector reject(digitVec.size());
+  if (mIRFrameSelector.isSet()) {
+    for (size_t id = 0; id < digitVec.size(); id++) {
+      if (mIRFrameSelector.check(digitVec[id].mIntRecord) < 0) {
+        reject[id] = true;
+        nDigSel--;
+        nChanSel -= digitVec[id].ref.getEntries();
+      } else if (firstDig == digitVec.size()) {
+        firstDig = id;
+      }
+    }
+  } else {
+    firstDig = 0;
+  }
+  if (nDigSel == 0) { // nothing is selected
+    return;
+  }
+
+  const auto& dig0 = digitVec[firstDig];
+  cd.header.nTriggers = nDigSel;
   cd.header.firstOrbit = dig0.getOrbit();
   cd.header.firstBC = dig0.getBC();
   cd.header.triggerGate = FT0DigParam::Instance().mTime_trg_gate;
@@ -248,38 +265,42 @@ void CTFCoder::compress(CompressedDigits& cd, const gsl::span& digi
   cd.eventStatus.resize(cd.header.nTriggers);
   cd.nChan.resize(cd.header.nTriggers);
 
-  cd.idChan.resize(channelVec.size());
-  cd.qtcChain.resize(channelVec.size());
-  cd.cfdTime.resize(channelVec.size());
-  cd.qtcAmpl.resize(channelVec.size());
+  cd.idChan.resize(nChanSel);
+  cd.qtcChain.resize(nChanSel);
+  cd.cfdTime.resize(nChanSel);
+  cd.qtcAmpl.resize(nChanSel);
 
   uint16_t prevBC = cd.header.firstBC;
   uint32_t prevOrbit = cd.header.firstOrbit;
-  uint32_t ccount = 0;
-  for (uint32_t idig = 0; idig < cd.header.nTriggers; idig++) {
+  uint32_t ccount = 0, dcount = 0;
+  for (uint32_t idig = 0; idig < digitVec.size(); idig++) {
+    if (reject[idig]) {
+      continue;
+    }
     const auto& digit = digitVec[idig];
     const auto chanels = digit.getBunchChannelData(channelVec); // we assume the channels are sorted
 
     // fill trigger info
-    cd.trigger[idig] = digit.getTriggers().getTriggersignals();
-    cd.eventStatus[idig] = digit.getEventStatusWord();
+    cd.trigger[dcount] = digit.getTriggers().getTriggersignals();
+    cd.eventStatus[dcount] = digit.getEventStatusWord();
     if (prevOrbit == digit.getOrbit()) {
-      cd.bcInc[idig] = digit.getBC() - prevBC;
-      cd.orbitInc[idig] = 0;
+      cd.bcInc[dcount] = digit.getBC() - prevBC;
+      cd.orbitInc[dcount] = 0;
     } else {
-      cd.bcInc[idig] = digit.getBC();
-      cd.orbitInc[idig] = digit.getOrbit() - prevOrbit;
+      cd.bcInc[dcount] = digit.getBC();
+      cd.orbitInc[dcount] = digit.getOrbit() - prevOrbit;
     }
     prevBC = digit.getBC();
     prevOrbit = digit.getOrbit();
     // fill channels info
-    cd.nChan[idig] = chanels.size();
-    if (!cd.nChan[idig]) {
+    cd.nChan[dcount] = chanels.size();
+    if (!cd.nChan[dcount]) {
       LOG(debug) << "Digits with no channels";
+      dcount++;
       continue;
     }
     uint8_t prevChan = 0;
-    for (uint8_t ic = 0; ic < cd.nChan[idig]; ic++) {
+    for (uint8_t ic = 0; ic < cd.nChan[dcount]; ic++) {
       if constexpr (MINOR_VERSION == 0 && MAJOR_VERSION == 1) {
         cd.idChan[ccount] = chanels[ic].ChId - prevChan; // Old method, lets keep it for a while
       } else {
@@ -291,6 +312,7 @@ void CTFCoder::compress(CompressedDigits& cd, const gsl::span& digi
       prevChan = chanels[ic].ChId;
       ccount++;
     }
+    dcount++;
   }
 }
 
diff --git a/Detectors/FIT/FT0/reconstruction/include/FT0Reconstruction/CollisionTimeRecoTask.h b/Detectors/FIT/FT0/reconstruction/include/FT0Reconstruction/CollisionTimeRecoTask.h
index 4f42f759bc342..9f6cd500b9e74 100644
--- a/Detectors/FIT/FT0/reconstruction/include/FT0Reconstruction/CollisionTimeRecoTask.h
+++ b/Detectors/FIT/FT0/reconstruction/include/FT0Reconstruction/CollisionTimeRecoTask.h
@@ -10,21 +10,21 @@
 // or submit itself to any jurisdiction.
 
 /// \file CollisionTimeRecoTask.h
-/// \brief Definition of the FIT collision time reconstruction task
-#ifndef ALICEO2_FIT_COLLISIONTIMERECOTASK_H
-#define ALICEO2_FIT_COLLISIONTIMERECOTASK_H
+/// \brief Definition of the FT0 collision time reconstruction task
+#ifndef ALICEO2_FT0_COLLISIONTIMERECOTASK_H
+#define ALICEO2_FT0_COLLISIONTIMERECOTASK_H
 
+#include "FT0Base/Geometry.h"
 #include "DataFormatsFT0/Digit.h"
 #include "DataFormatsFT0/ChannelData.h"
 #include "DataFormatsFT0/RecPoints.h"
-#include "CommonDataFormat/InteractionRecord.h"
-#include "CommonDataFormat/TimeStamp.h"
 #include "DataFormatsFT0/FT0ChannelTimeCalibrationObject.h"
-#include "FT0Base/Geometry.h"
+#include "DataFormatsFT0/SpectraInfoObject.h"
+#include "DataFormatsFT0/SlewingCoef.h"
+#include "DataFormatsFIT/DeadChannelMap.h"
 #include 
-#include 
-#include 
 #include 
+#include 
 #include 
 
 namespace o2
@@ -43,24 +43,32 @@ class CollisionTimeRecoTask
                Vertex };
   CollisionTimeRecoTask() = default;
   ~CollisionTimeRecoTask() = default;
-  o2::ft0::RecPoints process(o2::ft0::Digit const& bcd,
-                             gsl::span inChData,
-                             gsl::span outChData);
+  void processTF(const gsl::span& digits,
+                 const gsl::span& channels,
+                 std::vector& vecRecPoints,
+                 std::vector& vecChData);
+
+  o2::ft0::RecPoints processDigit(const o2::ft0::Digit& digit,
+                                  const gsl::span inChData,
+                                  std::vector& outChData);
   void FinishTask();
-  void SetChannelOffset(o2::ft0::FT0ChannelTimeCalibrationObject const*
-                          caliboffsets) { mCalibOffset = caliboffsets; };
-  void SetSlew(std::array* calibslew)
+  void SetTimeCalibObject(o2::ft0::TimeSpectraInfoObject const* timeCalibObject) { mTimeCalibObject = timeCalibObject; };
+  void SetSlewingCalibObject(o2::ft0::SlewingCoef const* calibSlew)
   {
-    LOG(info) << "@@@SetSlew " << calibslew->size();
-    mCalibSlew = calibslew;
+    LOG(info) << "Init for slewing calib object";
+    mCalibSlew = calibSlew->makeSlewingPlots();
   };
-  int getOffset(int channel, int amp);
+  void SetDeadChannelMap(const o2::fit::DeadChannelMap* deadChannelMap)
+  {
+    LOG(info) << "Updated dead channel map for CollisionTimeRecoTask";
+    mDeadChannelMap = deadChannelMap;
+  }
+  float getTimeInPS(const o2::ft0::ChannelData& channelData);
 
  private:
-  o2::ft0::FT0ChannelTimeCalibrationObject const* mCalibOffset;
-  std::array* mCalibSlew = nullptr;
-
-  ClassDefNV(CollisionTimeRecoTask, 3);
+  o2::ft0::TimeSpectraInfoObject const* mTimeCalibObject = nullptr;
+  const o2::fit::DeadChannelMap* mDeadChannelMap = nullptr;
+  typename o2::ft0::SlewingCoef::SlewingPlots_t mCalibSlew{};
 };
 } // namespace ft0
 } // namespace o2
diff --git a/Detectors/FIT/FT0/reconstruction/include/FT0Reconstruction/InteractionTag.h b/Detectors/FIT/FT0/reconstruction/include/FT0Reconstruction/InteractionTag.h
index be4b0ee032c20..c0007ac3a4188 100644
--- a/Detectors/FIT/FT0/reconstruction/include/FT0Reconstruction/InteractionTag.h
+++ b/Detectors/FIT/FT0/reconstruction/include/FT0Reconstruction/InteractionTag.h
@@ -26,11 +26,12 @@ namespace ft0
 
 // These are configurable params for FT0 selection as interaction tag
 struct InteractionTag : public o2::conf::ConfigurableParamHelper {
-  int minAmplitudeAC = 20; ///< use only FT0 triggers with high enough amplitude
-
+  int minAmplitudeAC = 2; ///< use only FT0 triggers with high enough amplitude
+  int minAmplitudeA = 1;
+  int minAmplitudeC = 1;
   bool isSelected(const RecPoints& rp) const
   {
-    return rp.isValidTime(RecPoints::TimeMean) && (rp.getTrigger().getAmplA() + rp.getTrigger().getAmplC()) > minAmplitudeAC;
+    return rp.getTrigger().getVertex() && rp.getTrigger().getAmplA() >= minAmplitudeA && rp.getTrigger().getAmplC() >= minAmplitudeC && (rp.getTrigger().getAmplA() + rp.getTrigger().getAmplC()) >= minAmplitudeAC;
   }
 
   O2ParamDef(InteractionTag, "ft0tag");
diff --git a/Detectors/FIT/FT0/reconstruction/src/CTFCoder.cxx b/Detectors/FIT/FT0/reconstruction/src/CTFCoder.cxx
index 6e840c52772cb..349c2edf796a6 100644
--- a/Detectors/FIT/FT0/reconstruction/src/CTFCoder.cxx
+++ b/Detectors/FIT/FT0/reconstruction/src/CTFCoder.cxx
@@ -53,7 +53,7 @@ void CTFCoder::createCoders(const std::vector& bufVec, o2::ctf::CTFCoderBa
 {
   const auto ctf = CTF::getImage(bufVec.data());
   CompressedDigits cd; // just to get member types
-#define MAKECODER(part, slot) createCoder(op, ctf.getFrequencyTable(slot), int(slot))
+#define MAKECODER(part, slot) createCoder(op, std::get>(ctf.getDictionary(slot, mANSVersion)), int(slot))
   // clang-format off
   MAKECODER(cd.trigger,      CTF::BLC_trigger);
   MAKECODER(cd.bcInc,        CTF::BLC_bcInc);
@@ -71,22 +71,18 @@ void CTFCoder::createCoders(const std::vector& bufVec, o2::ctf::CTFCoderBa
 size_t CTFCoder::estimateCompressedSize(const CompressedDigits& cd)
 {
   size_t sz = 0;
-  // clang-format off
   // RS FIXME this is very crude estimate, instead, an empirical values should be used
-#define VTP(vec) typename std::remove_reference::type::value_type
-#define ESTSIZE(vec, slot) mCoders[int(slot)] ?                         \
-  rans::calculateMaxBufferSize(vec.size(), reinterpret_cast*>(mCoders[int(slot)].get())->getAlphabetRangeBits(), sizeof(VTP(vec)) ) : vec.size()*sizeof(VTP(vec))
-  sz += ESTSIZE(cd.trigger,      CTF::BLC_trigger);
-  sz += ESTSIZE(cd.bcInc,        CTF::BLC_bcInc);
-  sz += ESTSIZE(cd.orbitInc,     CTF::BLC_orbitInc);
-  sz += ESTSIZE(cd.nChan,        CTF::BLC_nChan);
-  sz += ESTSIZE(cd.eventStatus,  CTF::BLC_status);
-  sz += ESTSIZE(cd.idChan,       CTF::BLC_idChan);
-  sz += ESTSIZE(cd.qtcChain,     CTF::BLC_qtcChain);
-  sz += ESTSIZE(cd.cfdTime,      CTF::BLC_cfdTime);
-  sz += ESTSIZE(cd.qtcAmpl,      CTF::BLC_qtcAmpl);
+  sz += estimateBufferSize(static_cast(CTF::BLC_trigger), cd.trigger);
+  sz += estimateBufferSize(static_cast(CTF::BLC_bcInc), cd.bcInc);
+  sz += estimateBufferSize(static_cast(CTF::BLC_orbitInc), cd.orbitInc);
+  sz += estimateBufferSize(static_cast(CTF::BLC_nChan), cd.nChan);
+  sz += estimateBufferSize(static_cast(CTF::BLC_status), cd.eventStatus);
+  sz += estimateBufferSize(static_cast(CTF::BLC_idChan), cd.idChan);
+  sz += estimateBufferSize(static_cast(CTF::BLC_qtcChain), cd.qtcChain);
+  sz += estimateBufferSize(static_cast(CTF::BLC_cfdTime), cd.cfdTime);
+  sz += estimateBufferSize(static_cast(CTF::BLC_qtcAmpl), cd.qtcAmpl);
   // clang-format on
 
-  LOG(info) << "Estimated output size is " << sz << " bytes";
+  LOG(debug) << "Estimated output size is " << sz << " bytes";
   return sz;
-}
+}
\ No newline at end of file
diff --git a/Detectors/FIT/FT0/reconstruction/src/CollisionTimeRecoTask.cxx b/Detectors/FIT/FT0/reconstruction/src/CollisionTimeRecoTask.cxx
index 468437539997d..3e3ffe52671e9 100644
--- a/Detectors/FIT/FT0/reconstruction/src/CollisionTimeRecoTask.cxx
+++ b/Detectors/FIT/FT0/reconstruction/src/CollisionTimeRecoTask.cxx
@@ -10,15 +10,17 @@
 // or submit itself to any jurisdiction.
 
 /// \file  CollisionTimeRecoTask.cxx
-/// \brief Implementation of the FIT reconstruction task
+/// \brief Implementation of the FT0 reconstruction task
 
 #include "FT0Reconstruction/CollisionTimeRecoTask.h"
-#include "FairLogger.h" // for LOG
+#include  // for LOG
 #include "DataFormatsFT0/RecPoints.h"
 #include "FT0Base/Geometry.h"
 #include "FT0Base/FT0DigParam.h"
 #include 
 #include 
+#include 
+#include 
 #include 
 #include 
 #include 
@@ -27,45 +29,76 @@
 #include 
 
 using namespace o2::ft0;
+using RP = o2::ft0::RecPoints;
 
-o2::ft0::RecPoints CollisionTimeRecoTask::process(o2::ft0::Digit const& bcd,
-                                                  gsl::span inChData,
-                                                  gsl::span outChData)
+void CollisionTimeRecoTask::processTF(const gsl::span& digits,
+                                      const gsl::span& channels,
+                                      std::vector& vecRecPoints,
+                                      std::vector& vecChData)
+{
+  for (const auto& digit : digits) {
+    if (!ChannelFilterParam::Instance().checkTCMbits(digit.getTriggers().getTriggersignals())) {
+      continue;
+    }
+    const auto channelsPerDigit = digit.getBunchChannelData(channels);
+    vecRecPoints.emplace_back(processDigit(digit, channelsPerDigit, vecChData));
+  }
+}
+RP CollisionTimeRecoTask::processDigit(const o2::ft0::Digit& digit,
+                                       const gsl::span inChData,
+                                       std::vector& outChData)
 {
   LOG(debug) << "Running reconstruction on new event";
+  const int firstEntry = outChData.size();
+  unsigned int ndigitsA = 0;
+  unsigned int ndigitsC = 0;
+  float sideAtime = 0;
+  float sideCtime = 0;
 
-  Int_t ndigitsC = 0, ndigitsA = 0;
-
-  constexpr Int_t nMCPsA = 4 * Geometry::NCellsA;
+  constexpr int nMCPsA = 4 * Geometry::NCellsA;
 
-  Float_t sideAtime = 0, sideCtime = 0;
+  int nch{0};
+  bool isActiveA = false;
+  bool isActiveC = false;
+  bool isFlangeEvent = false;
 
-  int nch = inChData.size();
-  const auto parInv = FT0DigParam::Instance().mMV_2_NchannelsInverse;
-  for (int ich = 0; ich < nch; ich++) {
-    int offsetChannel = getOffset(int(inChData[ich].ChId), inChData[ich].QTCAmpl);
-    outChData[ich] = o2::ft0::ChannelDataFloat{inChData[ich].ChId,
-                                               (inChData[ich].CFDTime - offsetChannel) * Geometry::ChannelWidth,
-                                               (float)inChData[ich].QTCAmpl,
-                                               inChData[ich].ChainQTC};
-
-    //  only signals with amplitude participate in collision time
-    if (outChData[ich].QTCAmpl > FT0DigParam::Instance().mAmpThresholdForReco && std::abs(outChData[ich].CFDTime) < FT0DigParam::Instance().mTimeThresholdForReco) {
-      if (outChData[ich].ChId < nMCPsA) {
-        sideAtime += outChData[ich].CFDTime;
+  for (const auto& channelData : inChData) {
+    if (channelData.ChId >= NCHANNELS) {
+      // Reference channels shouldn't participate in reco at all!
+      continue;
+    }
+    if (mDeadChannelMap && !mDeadChannelMap->isChannelAlive(channelData.ChId)) {
+      LOG(debug) << "Channel " << channelData.ChId << " is dead - discarding data";
+      continue;
+    }
+    const float timeInPS = getTimeInPS(channelData);
+    if (ChannelFilterParam::Instance().checkAll(channelData)) {
+      outChData.emplace_back(channelData.ChId, timeInPS, (float)channelData.QTCAmpl, channelData.ChainQTC);
+      nch++;
+    }
+    const bool isOkForTimeCalc = TimeFilterParam::Instance().checkAll(channelData);
+    //  only signals which satisfy conditions may participate in time calculation
+    if (channelData.ChId < nMCPsA) {
+      // A-side
+      if (isOkForTimeCalc) {
+        sideAtime += timeInPS;
         ndigitsA++;
-      } else {
-        sideCtime += outChData[ich].CFDTime;
-        LOG(debug) << "cfd " << outChData[ich].ChId << " dig " << 13.2 * inChData[ich].CFDTime << " rec " << outChData[ich].CFDTime << " amp " << (float)inChData[ich].QTCAmpl << " offset " << offsetChannel;
+      }
+      isActiveA = true;
+    } else {
+      // C-side
+      if (isOkForTimeCalc) {
+        sideCtime += timeInPS;
         ndigitsC++;
       }
+      isActiveC = true;
+      isFlangeEvent |= channelData.CFDTime < -350 && channelData.CFDTime > -450;
     }
   }
-  auto sDummyCollissionTime = o2::ft0::RecPoints::sDummyCollissionTime;
-  std::array mCollisionTime = {sDummyCollissionTime, sDummyCollissionTime, sDummyCollissionTime, sDummyCollissionTime};
-  // !!!! tobe done::should be fix with ITS vertex
-  mCollisionTime[TimeA] = (ndigitsA > 0) ? sideAtime / ndigitsA : sDummyCollissionTime; // 2 * o2::InteractionRecord::DummyTime;
-  mCollisionTime[TimeC] = (ndigitsC > 0) ? sideCtime / ndigitsC : sDummyCollissionTime; //2 * o2::InteractionRecord::DummyTime;
+  std::array mCollisionTime = {RP::sDummyCollissionTime, RP::sDummyCollissionTime, RP::sDummyCollissionTime, RP::sDummyCollissionTime};
+
+  mCollisionTime[TimeA] = (ndigitsA > 0) ? std::round(sideAtime / ndigitsA) : RP::sDummyCollissionTime; // 2 * o2::InteractionRecord::DummyTime;
+  mCollisionTime[TimeC] = (ndigitsC > 0) ? std::round(sideCtime / ndigitsC) : RP::sDummyCollissionTime; // 2 * o2::InteractionRecord::DummyTime;
 
   if (ndigitsA > 0 && ndigitsC > 0) {
     mCollisionTime[Vertex] = (mCollisionTime[TimeA] - mCollisionTime[TimeC]) / 2.;
@@ -73,9 +106,8 @@ o2::ft0::RecPoints CollisionTimeRecoTask::process(o2::ft0::Digit const& bcd,
   } else {
     mCollisionTime[TimeMean] = std::min(mCollisionTime[TimeA], mCollisionTime[TimeC]);
   }
-  LOG(debug) << " Nch " << nch << " Collision time " << mCollisionTime[TimeA] << " " << mCollisionTime[TimeC] << " " << mCollisionTime[TimeMean] << " " << mCollisionTime[Vertex];
-  return RecPoints{
-    mCollisionTime, bcd.ref.getFirstEntry(), bcd.ref.getEntries(), bcd.mIntRecord, bcd.mTriggers};
+  const uint8_t extraTrgWord = RecPoints::makeExtraTrgWord(isActiveA, isActiveC, isFlangeEvent);
+  return RecPoints(firstEntry, nch, digit.mIntRecord, mCollisionTime, digit.mTriggers, extraTrgWord);
 }
 //______________________________________________________
 void CollisionTimeRecoTask::FinishTask()
@@ -83,18 +115,37 @@ void CollisionTimeRecoTask::FinishTask()
   // finalize digitization, if needed, flash remaining digits
   // if (!mContinuous)   return;
 }
-//______________________________________________________
-int CollisionTimeRecoTask::getOffset(int channel, int amp)
+
+float CollisionTimeRecoTask::getTimeInPS(const o2::ft0::ChannelData& channelData)
 {
-  if (!mCalibOffset) {
-    return 0;
-  }
-  int offsetChannel = mCalibOffset->mTimeOffsets[channel];
-  double slewoffset = 0;
-  if (mCalibSlew) {
-    TGraph& gr = mCalibSlew->at(channel);
-    slewoffset = gr.Eval(amp);
+  // Getting time offset
+  float offsetChannel{0};
+  if (mTimeCalibObject) {
+    // Temporary, will be changed to status bit checking
+    // Check statistics
+    const auto& stat = mTimeCalibObject->mTime[channelData.ChId].mStat;
+    const bool isEnoughStat = stat > CalibParam::Instance().mMaxEntriesThreshold;
+    const bool isNotGoogStat = stat > CalibParam::Instance().mMinEntriesThreshold && !isEnoughStat;
+    // Check fit quality
+    const auto& meanGaus = mTimeCalibObject->mTime[channelData.ChId].mGausMean;
+    const auto& meanHist = mTimeCalibObject->mTime[channelData.ChId].mStatMean;
+    const auto& sigmaGaus = mTimeCalibObject->mTime[channelData.ChId].mGausRMS;
+    const auto& rmsHist = mTimeCalibObject->mTime[channelData.ChId].mStatRMS;
+    const bool isGoodFitResult = (mTimeCalibObject->mTime[channelData.ChId].mStatusBits & 1) > 0;
+    const bool isBadFit = std::abs(meanGaus - meanHist) > CalibParam::Instance().mMaxDiffMean || rmsHist < CalibParam::Instance().mMinRMS || sigmaGaus > CalibParam::Instance().mMaxSigma;
+
+    if (isEnoughStat && isGoodFitResult && !isBadFit) {
+      offsetChannel = meanGaus;
+    } else if ((isNotGoogStat || isEnoughStat) && isBadFit) {
+      offsetChannel = meanHist;
+    }
   }
-  LOG(debug) << "CollisionTimeRecoTask::getOffset(int channel, int amp) " << channel << " " << amp << " " << offsetChannel << " " << slewoffset;
-  return offsetChannel + int(slewoffset);
+  // Getting slewing offset
+  float slewoffset{0};
+  const auto& gr = mCalibSlew[static_cast(channelData.getFlag(o2::ft0::ChannelData::EEventDataBit::kNumberADC))][channelData.ChId];
+  slewoffset = gr.Eval(channelData.QTCAmpl);
+
+  // Final calculation
+  const float globalOffset = offsetChannel + slewoffset;
+  return (static_cast(channelData.CFDTime) - globalOffset) * Geometry::ChannelWidth;
 }
diff --git a/Detectors/FIT/FT0/reconstruction/src/FT0ReconstructionLinkDef.h b/Detectors/FIT/FT0/reconstruction/src/FT0ReconstructionLinkDef.h
index 81153dfff9132..e35f871e91c7c 100644
--- a/Detectors/FIT/FT0/reconstruction/src/FT0ReconstructionLinkDef.h
+++ b/Detectors/FIT/FT0/reconstruction/src/FT0ReconstructionLinkDef.h
@@ -15,7 +15,6 @@
 #pragma link off all classes;
 #pragma link off all functions;
 
-#pragma link C++ class o2::ft0::CollisionTimeRecoTask + ;
 #pragma link C++ class o2::ft0::InteractionTag + ;
 
 #endif
diff --git a/Detectors/FIT/FT0/simulation/CMakeLists.txt b/Detectors/FIT/FT0/simulation/CMakeLists.txt
index f94d085d5b48e..1559649755c4b 100644
--- a/Detectors/FIT/FT0/simulation/CMakeLists.txt
+++ b/Detectors/FIT/FT0/simulation/CMakeLists.txt
@@ -31,12 +31,8 @@ o2_target_root_dictionary(FT0Simulation
 o2_add_executable(digi2raw
         COMPONENT_NAME ft0
         SOURCES src/digit2raw.cxx
-        PUBLIC_LINK_LIBRARIES O2::FT0Simulation
-                              O2::DetectorsRaw
-                              O2::DetectorsCommonDataFormats
-                              O2::CommonUtils
-                              Boost::program_options
-                              O2::FT0Raw)
+        PUBLIC_LINK_LIBRARIES O2::FT0Raw
+                              Boost::program_options)
 
 o2_data_file(COPY data DESTINATION Detectors/FT0/simulation)
 
diff --git a/Detectors/FIT/FT0/simulation/include/FT0Simulation/Digitizer.h b/Detectors/FIT/FT0/simulation/include/FT0Simulation/Digitizer.h
index bd5693707673f..3a035a247c6ef 100644
--- a/Detectors/FIT/FT0/simulation/include/FT0Simulation/Digitizer.h
+++ b/Detectors/FIT/FT0/simulation/include/FT0Simulation/Digitizer.h
@@ -13,6 +13,7 @@
 #define ALICEO2_FT0_DIGITIZER_H
 
 #include "CommonDataFormat/InteractionRecord.h"
+#include "DataFormatsFIT/DeadChannelMap.h"
 #include "DataFormatsFT0/Digit.h"
 #include "DataFormatsFT0/ChannelData.h"
 #include "DataFormatsFT0/MCLabel.h"
@@ -73,6 +74,7 @@ class Digitizer
 
   void SetChannelOffset(o2::ft0::FT0ChannelTimeCalibrationObject const*
                           caliboffsets) { mCalibOffset = caliboffsets; };
+  void setDeadChannelMap(o2::fit::DeadChannelMap const* deadChannelMap) { mDeadChannelMap = deadChannelMap; };
   double getTimeOffsetWrtBC() const { return mIntRecord.getTimeOffsetWrtBC(); }
 
   struct CFDOutput {
@@ -165,6 +167,7 @@ class Digitizer
                o2::dataformats::MCTruthContainer& labels);
 
   o2::ft0::FT0ChannelTimeCalibrationObject const* mCalibOffset = nullptr;
+  o2::fit::DeadChannelMap const* mDeadChannelMap = nullptr;
 
   ClassDefNV(Digitizer, 3);
 };
diff --git a/Detectors/FIT/FT0/simulation/src/Detector.cxx b/Detectors/FIT/FT0/simulation/src/Detector.cxx
index dc8d982309001..ea856eb204802 100644
--- a/Detectors/FIT/FT0/simulation/src/Detector.cxx
+++ b/Detectors/FIT/FT0/simulation/src/Detector.cxx
@@ -26,7 +26,7 @@
 #include "TVector3.h"
 
 #include "FairRootManager.h" // for FairRootManager
-#include "FairLogger.h"
+#include 
 #include "FairVolume.h"
 
 #include "FairRootManager.h"
@@ -36,7 +36,7 @@
 #include 
 #include "FT0Base/Geometry.h"
 #include "FT0Simulation/Detector.h"
-#include "SimulationDataFormat/Stack.h"
+#include "DetectorsBase/Stack.h"
 
 using namespace o2::ft0;
 using o2::ft0::Geometry;
@@ -882,18 +882,10 @@ TGeoVolume* Detector::constructFrameCGeometry()
   Double_t yPMT[NCellsC];
   Double_t zPMT[NCellsC];
 
-  Double_t aPMT[NCellsC];
-  Double_t bPMT[NCellsC];
-  Double_t gPMT[NCellsC];
-
   Double_t xQrad[NCellsC];
   Double_t yQrad[NCellsC];
   Double_t zQrad[NCellsC];
 
-  Double_t aQrad[NCellsC];
-  Double_t bQrad[NCellsC];
-  Double_t gQrad[NCellsC];
-
   Double_t rotC[NCellsC];
   Double_t comC[NCellsC];
 
@@ -903,31 +895,10 @@ TGeoVolume* Detector::constructFrameCGeometry()
     yPMT[i] = scalePMT * yc2[i];
     zPMT[i] = scalePMT * zc2[i];
 
-    aPMT[i] = TMath::ATan(yPMT[i] / xPMT[i]) - TMath::Pi() / 2 + 2 * TMath::Pi();
-    if (xPMT[i] < 0) {
-      bPMT[i] = TMath::ACos(zPMT[i] / crad);
-    } else {
-      bPMT[i] = -1 * TMath::ACos(zPMT[i] / crad);
-    }
-
-    aPMT[i] *= 180 / TMath::Pi();
-    bPMT[i] *= 180 / TMath::Pi();
-    gPMT[i] = -1 * aPMT[i];
-
     // Quartz radiator transformations
     xQrad[i] = scaleQrad * xc2[i];
     yQrad[i] = scaleQrad * yc2[i];
     zQrad[i] = scaleQrad * zc2[i];
-
-    aQrad[i] = TMath::ATan(yQrad[i] / xQrad[i]) - TMath::Pi() / 2 + 2 * TMath::Pi();
-    if (xQrad[i] < 0) {
-      bQrad[i] = TMath::ACos(zQrad[i] / crad);
-    } else {
-      bQrad[i] = -1 * TMath::ACos(zQrad[i] / crad);
-    }
-    aQrad[i] *= 180 / TMath::Pi();
-    bQrad[i] *= 180 / TMath::Pi();
-    gQrad[i] = -1 * aQrad[i];
   }
 
   TString nameRot;
diff --git a/Detectors/FIT/FT0/simulation/src/Digitizer.cxx b/Detectors/FIT/FT0/simulation/src/Digitizer.cxx
index e917e59b1dd88..aca012f1bc5a9 100644
--- a/Detectors/FIT/FT0/simulation/src/Digitizer.cxx
+++ b/Detectors/FIT/FT0/simulation/src/Digitizer.cxx
@@ -13,7 +13,8 @@
 #include "FT0Simulation/DigitizationConstants.h"
 #include "FT0Base/FT0DigParam.h"
 #include "SimulationDataFormat/MCTruthContainer.h"
-#include 
+#include "CommonConstants/PhysicsConstants.h"
+#include "CommonDataFormat/InteractionRecord.h"
 
 #include "TMath.h"
 #include "TRandom.h"
@@ -193,7 +194,7 @@ void Digitizer::process(const std::vector* hits,
                         o2::dataformats::MCTruthContainer& label)
 {
   ;
-  //Calculating signal time, amplitude in mean_time +- time_gate --------------
+  // Calculating signal time, amplitude in mean_time +- time_gate --------------
   LOG(debug) << " process firstBCinDeque " << firstBCinDeque << " mIntRecord " << mIntRecord;
   if (firstBCinDeque != mIntRecord) {
     flush(digitsBC, digitsCh, digitsTrig, label);
@@ -204,20 +205,33 @@ void Digitizer::process(const std::vector* hits,
     if (hit.GetEnergyLoss() > 0) {
       continue;
     }
-    const auto& params = FT0DigParam::Instance();
+
     Int_t hit_ch = hit.GetDetectorID();
+
+    // If the dead channel map is used, and the channel with ID 'hit_ch' is dead, don't process this hit.
+    if (mDeadChannelMap && !mDeadChannelMap->isChannelAlive(hit_ch)) {
+      continue;
+    }
+
+    const auto& params = FT0DigParam::Instance();
+
     Bool_t is_A_side = (hit_ch < 4 * mGeometry.NCellsA);
-    Float_t time_compensate = is_A_side ? params.mA_side_cable_cmps : params.mC_side_cable_cmps;
-    Double_t hit_time = hit.GetTime() - time_compensate;
+
+    // Subtract time-of-flight from hit time
+    const Float_t timeOfFlight = hit.GetPos().R() / o2::constants::physics::LightSpeedCm2NS;
+    const Float_t timeOffset = is_A_side ? params.hitTimeOffsetA : params.hitTimeOffsetC;
+    Double_t hit_time = hit.GetTime() - timeOfFlight + timeOffset + mIntRecord.getTimeOffsetWrtBC();
+
     if (hit_time > 150) {
-      continue; //not collect very slow particles
+      continue; // not collect very slow particles
     }
+
     auto relBC = o2::InteractionRecord{hit_time};
     if (mCache.size() <= relBC.bc) {
       mCache.resize(relBC.bc + 1);
     }
     mCache[relBC.bc].hits.emplace_back(BCCache::particle{hit_ch, hit_time - relBC.bc2ns()});
-    //charge particles in MCLabel
+    // charge particles in MCLabel
     Int_t parentID = hit.GetTrackID();
     if (parentID != parent) {
       mCache[relBC.bc].labels.emplace(parentID, mEventID, mSrcID, hit_ch);
@@ -248,6 +262,9 @@ void Digitizer::storeBC(BCCache& bc,
     auto channel_begin = channel_end;
     channel_end = std::find_if(channel_begin, particles.end(),
                                [ipmt](BCCache::particle const& p) { return p.hit_ch != ipmt; });
+
+    // The hits between 'channel_begin' and 'channel_end' now contains all hits for channel 'ipmt'
+
     if (channel_end - channel_begin < params.mAmp_trsh) {
       continue;
     }
@@ -263,12 +280,12 @@ void Digitizer::storeBC(BCCache& bc,
     if (!cfd.particle) {
       continue;
     }
-    //miscalibrate CFD with cahnnel offsets
+    // miscalibrate CFD with cahnnel offsets
     int miscalib = 0;
     if (mCalibOffset) {
       miscalib = mCalibOffset->mTimeOffsets[ipmt];
     }
-    int smeared_time = 1000. * (*cfd.particle - params.mCfdShift) * params.mChannelWidthInverse + miscalib + int(1000. * mIntRecord.getTimeOffsetWrtBC() * params.mChannelWidthInverse);
+    int smeared_time = 1000. * (*cfd.particle - params.mCfdShift) * params.mChannelWidthInverse + miscalib; // + int(1000. * mIntRecord.getTimeOffsetWrtBC() * params.mChannelWidthInverse);
     bool is_time_in_signal_gate = (smeared_time > -params.mTime_trg_gate && smeared_time < params.mTime_trg_gate);
     float charge = measure_amplitude(channel_times) * params.mCharge2amp;
     float amp = is_time_in_signal_gate ? params.mMV_2_Nchannels * charge : 0;
@@ -277,6 +294,10 @@ void Digitizer::storeBC(BCCache& bc,
     }
 
     LOG(debug) << mEventID << " bc " << firstBCinDeque.bc << " orbit " << firstBCinDeque.orbit << ", ipmt " << ipmt << ", smeared_time " << smeared_time << " nStored " << nStored << " offset " << miscalib;
+    if (is_time_in_signal_gate) {
+      chain |= (1 << o2::ft0::ChannelData::EEventDataBit::kIsCFDinADCgate);
+      chain |= (1 << o2::ft0::ChannelData::EEventDataBit::kIsEventInTVDC);
+    }
     digitsCh.emplace_back(ipmt, smeared_time, int(amp), chain);
     nStored++;
 
@@ -310,10 +331,12 @@ void Digitizer::storeBC(BCCache& bc,
   isVertex = is_A && is_C && (vertex_time > -params.mTime_trg_gate && vertex_time < params.mTime_trg_gate);
   LOG(debug) << " A " << is_A << " timeA " << timeA << " mean_time_A " << mean_time_A << "  n_hit_A " << n_hit_A << " C " << is_C << " timeC " << timeC << " mean_time_C " << mean_time_C << "  n_hit_C " << n_hit_C << " vertex_time " << vertex_time;
   Triggers triggers;
-  const bool unusedBitsInSim = false; // bits related to laser and data validity
+  bool isLaser = false;
+  bool isOutputsAreBlocked = false;
+  bool isDataValid = true;
   if (nStored > 0) {
     triggers.setTriggers(is_A, is_C, isVertex, is_Central, is_SemiCentral, int8_t(n_hit_A), int8_t(n_hit_C),
-                         amplA, amplC, timeA, timeC, unusedBitsInSim, unusedBitsInSim, unusedBitsInSim);
+                         amplA, amplC, timeA, timeC, isLaser, isOutputsAreBlocked, isDataValid);
     digitsBC.emplace_back(first, nStored, firstBCinDeque, triggers, mEventID - 1);
     digitsTrig.emplace_back(firstBCinDeque, is_A, is_C, isVertex, is_Central, is_SemiCentral);
     size_t const nBC = digitsBC.size();
@@ -323,12 +346,12 @@ void Digitizer::storeBC(BCCache& bc,
   }
   // Debug output -------------------------------------------------------------
 
-  LOG(info) << "Event ID: " << mEventID << ", bc " << firstBCinDeque.bc << ", N hit " << bc.hits.size();
-  LOG(info) << "N hit A: " << int(triggers.getNChanA()) << " N hit C: " << int(triggers.getNChanC()) << " summ ampl A: " << int(triggers.getAmplA())
-            << " summ ampl C: " << int(triggers.getAmplC()) << " mean time A: " << triggers.getTimeA()
-            << " mean time C: " << triggers.getTimeC() << " nStored " << nStored;
+  LOG(debug) << "Event ID: " << mEventID << ", bc " << firstBCinDeque.bc << ", N hit " << bc.hits.size();
+  LOG(debug) << "N hit A: " << int(triggers.getNChanA()) << " N hit C: " << int(triggers.getNChanC()) << " summ ampl A: " << int(triggers.getAmplA())
+             << " summ ampl C: " << int(triggers.getAmplC()) << " mean time A: " << triggers.getTimeA()
+             << " mean time C: " << triggers.getTimeC() << " nStored " << nStored;
 
-  LOG(info) << "IS A " << triggers.getOrA() << " IsC " << triggers.getOrC() << " vertex " << triggers.getVertex() << " is Central " << triggers.getCen() << " is SemiCentral " << triggers.getSCen();
+  LOG(debug) << "IS A " << triggers.getOrA() << " IsC " << triggers.getOrC() << " vertex " << triggers.getVertex() << " is Central " << triggers.getCen() << " is SemiCentral " << triggers.getSCen();
 }
 
 //------------------------------------------------------------------------
diff --git a/Detectors/FIT/FT0/simulation/src/digit2raw.cxx b/Detectors/FIT/FT0/simulation/src/digit2raw.cxx
index b72ac83cbffd6..b700f06e9985f 100644
--- a/Detectors/FIT/FT0/simulation/src/digit2raw.cxx
+++ b/Detectors/FIT/FT0/simulation/src/digit2raw.cxx
@@ -13,21 +13,13 @@
 /// \author ruben.shahoyan@cern.ch afurs@cern.ch
 
 #include 
-#include 
 #include 
-#include "CommonUtils/StringUtils.h"
-#include "CommonUtils/ConfigurableParam.h"
-#include "DetectorsRaw/HBFUtils.h"
 #include "FT0Raw/RawWriterFT0.h"
-#include "DataFormatsParameters/GRPObject.h"
 
 /// MC->raw conversion for FT0
 
 namespace bpo = boost::program_options;
 
-void digit2raw(const std::string& inpName, const std::string& outDir, int verbosity, const std::string& fileFor, uint32_t rdhV, bool noEmptyHBF, const std::string& flpName,
-               const std::string& ccdbUrl, const std::string& lutPath, int superPageSizeInB = 1024 * 1024);
-
 int main(int argc, char** argv)
 {
   bpo::variables_map vm;
@@ -38,21 +30,10 @@ int main(int argc, char** argv)
   bpo::positional_options_description opt_pos;
 
   try {
-    auto add_option = opt_general.add_options();
-    add_option("help,h", "Print this help message");
-    add_option("verbosity,v", bpo::value()->default_value(0), "verbosity level");
-    //    add_option("input-file,i", bpo::value()->default_value(o2::base::NameConf::getDigitsFileName(o2::detectors::DetID::FT0)),"input FT0 digits file"); // why not used?
-    add_option("input-file,i", bpo::value()->default_value("ft0digits.root"), "input FT0 digits file");
-    add_option("flp-name", bpo::value()->default_value("alio2-cr1-flp200"), "single file per: all,flp,cru,link");
-    add_option("file-for,f", bpo::value()->default_value("all"), "single file per: all,flp,cru,link");
-    add_option("output-dir,o", bpo::value()->default_value("./"), "output directory for raw data");
-    uint32_t defRDH = o2::raw::RDHUtils::getVersion();
-    add_option("rdh-version,r", bpo::value()->default_value(defRDH), "RDH version to use");
-    add_option("no-empty-hbf,e", bpo::value()->default_value(false)->implicit_value(true), "do not create empty HBF pages (except for HBF starting TF)");
-    add_option("hbfutils-config,u", bpo::value()->default_value(std::string(o2::base::NameConf::DIGITIZATIONCONFIGFILE)), "config file for HBFUtils (or none)");
-    add_option("configKeyValues", bpo::value()->default_value(""), "comma-separated configKeyValues");
-    add_option("ccdb-path", bpo::value()->default_value(""), "CCDB url which contains LookupTable");
-    add_option("lut-path", bpo::value()->default_value(""), "LookupTable path, e.g. FT0/LookupTable");
+    opt_general.add_options()("help,h", "Print this help message");
+    // config with FT0 defaults
+    o2::fit::DigitToRawConfig::configureExecOptions(opt_general, "ft0digits.root", "alio2-cr1-flp200");
+    // common part
     opt_all.add(opt_general).add(opt_hidden);
     bpo::store(bpo::command_line_parser(argc, argv).options(opt_all).positional(opt_pos).run(), vm);
 
@@ -71,59 +52,12 @@ int main(int argc, char** argv)
     std::cerr << e.what() << ", application will now exit" << std::endl;
     exit(2);
   }
-
-  std::string confDig = vm["hbfutils-config"].as();
-  if (!confDig.empty() && confDig != "none") {
-    o2::conf::ConfigurableParam::updateFromFile(confDig, "HBFUtils");
+  const o2::fit::DigitToRawConfig cfg(vm);
+  if (!cfg.mEnablePadding) {
+    o2::fit::DigitToRawDevice::digit2raw(cfg);
+  } else {
+    o2::fit::DigitToRawDevice::digit2raw(cfg);
   }
-  o2::conf::ConfigurableParam::updateFromString(vm["configKeyValues"].as());
-  digit2raw(vm["input-file"].as(),
-            vm["output-dir"].as(),
-            vm["verbosity"].as(),
-            vm["file-for"].as(),
-            vm["rdh-version"].as(),
-            vm["no-empty-hbf"].as(),
-            vm["flp-name"].as(),
-            vm["ccdb-path"].as(),
-            vm["lut-path"].as());
-
   o2::raw::HBFUtils::Instance().print();
-
   return 0;
 }
-
-void digit2raw(const std::string& inpName, const std::string& outDir, int verbosity, const std::string& fileFor, uint32_t rdhV, bool noEmptyHBF, const std::string& flpName, const std::string& ccdbUrl, const std::string& lutPath, int superPageSizeInB)
-{
-  TStopwatch swTot;
-  swTot.Start();
-  o2::ft0::RawWriterFT0 m2r;
-  m2r.setFileFor(fileFor);
-  m2r.setFlpName(flpName);
-  m2r.setVerbosity(verbosity);
-  if (ccdbUrl != "") {
-    m2r.setCCDBurl(ccdbUrl);
-  }
-  if (lutPath != "") {
-    m2r.setLUTpath(lutPath);
-  }
-  auto& wr = m2r.getWriter();
-  std::string inputGRP = o2::base::NameConf::getGRPFileName();
-  const auto grp = o2::parameters::GRPObject::loadFrom(inputGRP);
-  wr.setContinuousReadout(grp->isDetContinuousReadOut(o2::detectors::DetID::FT0)); // must be set explicitly
-  wr.setSuperPageSize(superPageSizeInB);
-  wr.useRDHVersion(rdhV);
-  wr.setDontFillEmptyHBF(noEmptyHBF);
-
-  o2::raw::assertOutputDirectory(outDir);
-
-  std::string outDirName(outDir);
-  if (outDirName.back() != '/') {
-    outDirName += '/';
-  }
-
-  m2r.convertDigitsToRaw(outDirName, inpName);
-  wr.writeConfFile(wr.getOrigin().str, "RAWDATA", o2::utils::Str::concat_string(outDirName, wr.getOrigin().str, "raw.cfg"));
-  //
-  swTot.Stop();
-  swTot.Print();
-}
diff --git a/Detectors/FIT/FT0/workflow/CMakeLists.txt b/Detectors/FIT/FT0/workflow/CMakeLists.txt
index 0e904bfff6e52..123a29293e2fb 100644
--- a/Detectors/FIT/FT0/workflow/CMakeLists.txt
+++ b/Detectors/FIT/FT0/workflow/CMakeLists.txt
@@ -35,6 +35,7 @@ o2_add_library(FT0Decoder
                        PUBLIC_LINK_LIBRARIES O2::DataFormatsFT0
                                      O2::Framework
                                      O2::DetectorsCommonDataFormats
+                                     O2::DetectorsRaw
                                      O2::DPLUtils
                                      TARGETVARNAME targetDecoderAVX512)
 target_compile_options(${targetDecoderAVX512} PRIVATE -mavx512f -mavx512vl -mavx512bw -mavx512dq)
@@ -96,6 +97,27 @@ o2_add_executable(recpoints-reader-workflow
                   SOURCES src/recpoints-reader-workflow.cxx
                   COMPONENT_NAME ft0
                   PUBLIC_LINK_LIBRARIES O2::FT0Workflow)
+
+o2_add_executable(recpoints-writer-workflow
+                  SOURCES src/recpoints-writer-workflow.cxx
+                  COMPONENT_NAME ft0
+                  PUBLIC_LINK_LIBRARIES O2::FT0Workflow)
+
+o2_add_executable(integrate-cluster-workflow
+                  SOURCES src/cluster-integrator.cxx
+                  COMPONENT_NAME ft0
+                  PUBLIC_LINK_LIBRARIES O2::FT0Workflow O2::FITWorkflow)
+
+o2_add_executable(integrate-cluster-reader-workflow
+                  SOURCES src/cluster-integrator-reader.cxx
+                  COMPONENT_NAME ft0
+                  PUBLIC_LINK_LIBRARIES O2::FT0Workflow O2::FITWorkflow)
+
+o2_add_executable(merge-integrate-cluster-workflow
+                  SOURCES src/cluster-merge-integrator.cxx
+                  COMPONENT_NAME ft0
+                  PUBLIC_LINK_LIBRARIES O2::FT0Workflow O2::FITWorkflow)
+
 if(NOT APPLE)
 
  set_property(TARGET ${fitrecoexe} PROPERTY LINK_WHAT_YOU_USE ON)
diff --git a/Detectors/FIT/FT0/workflow/include/FT0Workflow/EntropyDecoderSpec.h b/Detectors/FIT/FT0/workflow/include/FT0Workflow/EntropyDecoderSpec.h
index 4f8e8b5e9be63..d6009accfa45b 100644
--- a/Detectors/FIT/FT0/workflow/include/FT0Workflow/EntropyDecoderSpec.h
+++ b/Detectors/FIT/FT0/workflow/include/FT0Workflow/EntropyDecoderSpec.h
@@ -28,7 +28,7 @@ namespace ft0
 class EntropyDecoderSpec : public o2::framework::Task
 {
  public:
-  EntropyDecoderSpec(int verbosity);
+  EntropyDecoderSpec(int verbosity, const std::string& ctfdictOpt = "none");
   ~EntropyDecoderSpec() override = default;
   void run(o2::framework::ProcessingContext& pc) final;
   void init(o2::framework::InitContext& ic) final;
@@ -41,7 +41,7 @@ class EntropyDecoderSpec : public o2::framework::Task
 };
 
 /// create a processor spec
-framework::DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec);
+framework::DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec, const std::string& ctfdictOpt);
 
 } // namespace ft0
 } // namespace o2
diff --git a/Detectors/FIT/FT0/workflow/include/FT0Workflow/EntropyEncoderSpec.h b/Detectors/FIT/FT0/workflow/include/FT0Workflow/EntropyEncoderSpec.h
index 1f537ff9e6070..a1b3714fdbb26 100644
--- a/Detectors/FIT/FT0/workflow/include/FT0Workflow/EntropyEncoderSpec.h
+++ b/Detectors/FIT/FT0/workflow/include/FT0Workflow/EntropyEncoderSpec.h
@@ -28,7 +28,7 @@ namespace ft0
 class EntropyEncoderSpec : public o2::framework::Task
 {
  public:
-  EntropyEncoderSpec();
+  EntropyEncoderSpec(bool selIR, const std::string& ctfdictOpt = "none");
   ~EntropyEncoderSpec() override = default;
   void run(o2::framework::ProcessingContext& pc) final;
   void init(o2::framework::InitContext& ic) final;
@@ -37,11 +37,12 @@ class EntropyEncoderSpec : public o2::framework::Task
 
  private:
   o2::ft0::CTFCoder mCTFCoder;
+  bool mSelIR = false;
   TStopwatch mTimer;
 };
 
 /// create a processor spec
-framework::DataProcessorSpec getEntropyEncoderSpec();
+framework::DataProcessorSpec getEntropyEncoderSpec(bool selIR = false, const std::string& ctfdictOpt = "none");
 
 } // namespace ft0
 } // namespace o2
diff --git a/Detectors/FIT/FT0/workflow/include/FT0Workflow/FT0DataDecoderDPLSpec.h b/Detectors/FIT/FT0/workflow/include/FT0Workflow/FT0DataDecoderDPLSpec.h
index e039f18c47bc9..b78d0435d3636 100644
--- a/Detectors/FIT/FT0/workflow/include/FT0Workflow/FT0DataDecoderDPLSpec.h
+++ b/Detectors/FIT/FT0/workflow/include/FT0Workflow/FT0DataDecoderDPLSpec.h
@@ -110,7 +110,7 @@ framework::DataProcessorSpec getFT0DataDecoderDPLSpec(bool askSTFDist)
   std::vector outputSpec;
   outputSpec.emplace_back(o2::header::gDataOriginFT0, "DIGITSBC", 0, Lifetime::Timeframe);
   outputSpec.emplace_back(o2::header::gDataOriginFT0, "DIGITSCH", 0, Lifetime::Timeframe);
-  std::vector inputSpec{{"STF", ConcreteDataTypeMatcher{"FT0", "RAWDATA"}, Lifetime::Optional}};
+  std::vector inputSpec{{"STF", ConcreteDataTypeMatcher{"FT0", "RAWDATA"}, Lifetime::Timeframe}};
   if (askSTFDist) {
     inputSpec.emplace_back("STFDist", "FLP", "DISTSUBTIMEFRAME", 0, Lifetime::Timeframe);
   }
diff --git a/Detectors/FIT/FT0/workflow/include/FT0Workflow/FT0DataProcessDPLSpec.h b/Detectors/FIT/FT0/workflow/include/FT0Workflow/FT0DataProcessDPLSpec.h
deleted file mode 100644
index 7b7e98d50368e..0000000000000
--- a/Detectors/FIT/FT0/workflow/include/FT0Workflow/FT0DataProcessDPLSpec.h
+++ /dev/null
@@ -1,61 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-/// @file   FT0DataProcessDPLSpec.h
-
-#ifndef O2_FT0DATAPROCESSDPLSPEC_H
-#define O2_FT0DATAPROCESSDPLSPEC_H
-
-#include "Framework/CallbackService.h"
-#include "Framework/ConfigParamRegistry.h"
-#include "Framework/ControlService.h"
-#include "Framework/Lifetime.h"
-#include "Framework/Output.h"
-#include "Framework/WorkflowSpec.h"
-#include "Framework/SerializationMethods.h"
-#include "Framework/DataProcessorSpec.h"
-#include "Framework/Task.h"
-
-#include "FT0Raw/DigitBlockFT0.h"
-#include "DataFormatsFT0/Digit.h"
-#include "DataFormatsFT0/ChannelData.h"
-
-#include 
-#include 
-#include 
-
-using namespace o2::framework;
-
-namespace o2
-{
-namespace ft0
-{
-
-class FT0DataProcessDPLSpec : public Task
-{
- public:
-  FT0DataProcessDPLSpec(bool dumpEventBlocks) : mDumpEventBlocks(dumpEventBlocks) {}
-  ~FT0DataProcessDPLSpec() override = default;
-  void init(InitContext& ic) final;
-  void run(ProcessingContext& pc) final;
-
- private:
-  bool mDumpEventBlocks;
-
-  o2::header::DataOrigin mOrigin = o2::header::gDataOriginFT0;
-};
-
-framework::DataProcessorSpec getFT0DataProcessDPLSpec(bool dumpProcessor);
-
-} // namespace ft0
-} // namespace o2
-
-#endif /* O2_FT0DATAPROCESSDPL_H */
diff --git a/Detectors/FIT/FT0/workflow/include/FT0Workflow/FT0DataReaderDPLSpec.h b/Detectors/FIT/FT0/workflow/include/FT0Workflow/FT0DataReaderDPLSpec.h
deleted file mode 100644
index 32677c989da7b..0000000000000
--- a/Detectors/FIT/FT0/workflow/include/FT0Workflow/FT0DataReaderDPLSpec.h
+++ /dev/null
@@ -1,110 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-/// @file   FT0DataReaderDPLSpec.h
-
-#ifndef O2_FT0DATAREADERDPLSPEC_H
-#define O2_FT0DATAREADERDPLSPEC_H
-#include "DataFormatsFT0/LookUpTable.h"
-#include "Framework/DataProcessorSpec.h"
-#include "Framework/Task.h"
-#include "Framework/CallbackService.h"
-#include "Framework/ConfigParamRegistry.h"
-#include "Framework/ControlService.h"
-#include "Framework/Lifetime.h"
-#include "Framework/Output.h"
-#include "Framework/WorkflowSpec.h"
-#include "Framework/SerializationMethods.h"
-#include "DPLUtils/DPLRawParser.h"
-#include "Framework/InputRecordWalker.h"
-#include 
-#include 
-#include 
-#include "CommonUtils/VerbosityConfig.h"
-
-using namespace o2::framework;
-
-namespace o2
-{
-namespace ft0
-{
-template 
-class FT0DataReaderDPLSpec : public Task
-{
- public:
-  FT0DataReaderDPLSpec(const RawReader& rawReader) : mRawReader(rawReader) {}
-  FT0DataReaderDPLSpec() = default;
-  ~FT0DataReaderDPLSpec() override = default;
-  typedef RawReader RawReader_t;
-  void init(InitContext& ic) final { o2::ft0::SingleLUT::Instance().printFullMap(); }
-  void run(ProcessingContext& pc) final
-  {
-    // if we see requested data type input with 0xDEADBEEF subspec and 0 payload this means that the "delayed message"
-    // mechanism created it in absence of real data from upstream. Processor should send empty output to not block the workflow
-    {
-      static size_t contDeadBeef = 0; // number of times 0xDEADBEEF was seen continuously
-      std::vector dummy{InputSpec{"dummy", ConcreteDataMatcher{o2::header::gDataOriginFT0, o2::header::gDataDescriptionRawData, 0xDEADBEEF}}};
-      for (const auto& ref : InputRecordWalker(pc.inputs(), dummy)) {
-        const auto dh = o2::framework::DataRefUtils::getHeader(ref);
-        auto payloadSize = DataRefUtils::getPayloadSize(ref);
-        if (payloadSize == 0) {
-          auto maxWarn = o2::conf::VerbosityConfig::Instance().maxWarnDeadBeef;
-          if (++contDeadBeef <= maxWarn) {
-            LOGP(alarm, "Found input [{}/{}/{:#x}] TF#{} 1st_orbit:{} Payload {} : assuming no payload for all links in this TF{}",
-                 dh->dataOrigin.str, dh->dataDescription.str, dh->subSpecification, dh->tfCounter, dh->firstTForbit, payloadSize,
-                 contDeadBeef == maxWarn ? fmt::format(". {} such inputs in row received, stopping reporting", contDeadBeef) : "");
-          }
-          mRawReader.makeSnapshot(pc); // send empty output
-          return;
-        }
-      }
-      contDeadBeef = 0; // if good data, reset the counter
-    }
-    std::vector filter{InputSpec{"filter", ConcreteDataTypeMatcher{o2::header::gDataOriginFT0, o2::header::gDataDescriptionRawData}, Lifetime::Timeframe}};
-    DPLRawParser parser(pc.inputs(), filter);
-    std::size_t count = 0;
-    for (auto it = parser.begin(), end = parser.end(); it != end; ++it) {
-      //Proccessing each page
-      count++;
-      auto rdhPtr = it.get_if();
-      gsl::span payload(it.data(), it.size());
-      mRawReader.process(payload, rdhPtr->linkID, rdhPtr->endPointID);
-    }
-    LOG(info) << "Pages: " << count;
-    mRawReader.accumulateDigits();
-    mRawReader.makeSnapshot(pc);
-    mRawReader.clear();
-  }
-  RawReader_t mRawReader;
-};
-
-template 
-framework::DataProcessorSpec getFT0DataReaderDPLSpec(const RawReader& rawReader, bool askSTFDist)
-{
-  LOG(info) << "DataProcessorSpec initDataProcSpec() for RawReaderFT0";
-  std::vector outputSpec;
-  RawReader::prepareOutputSpec(outputSpec);
-  std::vector inputSpec{{"STF", ConcreteDataTypeMatcher{o2::header::gDataOriginFT0, "RAWDATA"}, Lifetime::Optional}};
-  if (askSTFDist) {
-    inputSpec.emplace_back("STFDist", "FLP", "DISTSUBTIMEFRAME", 0, Lifetime::Timeframe);
-  }
-  return DataProcessorSpec{
-    "ft0-datareader-dpl",
-    inputSpec,
-    outputSpec,
-    adaptFromTask>(rawReader),
-    Options{}};
-}
-
-} // namespace ft0
-} // namespace o2
-
-#endif /* O2_FT0DATAREADERDPL_H */
diff --git a/Detectors/FIT/FT0/workflow/include/FT0Workflow/FT0Workflow.h b/Detectors/FIT/FT0/workflow/include/FT0Workflow/FT0Workflow.h
deleted file mode 100644
index a4988b2c18fc7..0000000000000
--- a/Detectors/FIT/FT0/workflow/include/FT0Workflow/FT0Workflow.h
+++ /dev/null
@@ -1,28 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-#ifndef O2_FIT_FT0WORKFLOW_H
-#define O2_FIT_FT0WORKFLOW_H
-
-/// @file   FT0Workflow.h
-
-#include "Framework/WorkflowSpec.h"
-
-namespace o2
-{
-namespace ft0
-{
-framework::WorkflowSpec getFT0Workflow(bool isExtendedMode, bool useProcess,
-                                       bool dumpProcessor, bool dumpReader,
-                                       bool disableRootOut, bool askSTFDist);
-} // namespace ft0
-} // namespace o2
-#endif
diff --git a/Detectors/FIT/FT0/workflow/include/FT0Workflow/RawReaderFT0.h b/Detectors/FIT/FT0/workflow/include/FT0Workflow/RawReaderFT0.h
deleted file mode 100644
index f50b3d224b32c..0000000000000
--- a/Detectors/FIT/FT0/workflow/include/FT0Workflow/RawReaderFT0.h
+++ /dev/null
@@ -1,156 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-//
-//file RawReaderFT0.h class  for RAW data reading
-//
-// Artur.Furs
-// afurs@cern.ch
-//
-//Main purpuse is to decode FT0 data blocks and push them to DigitBlockFT0 for proccess
-//TODO: prepare wrappers for containers with digits and combine classes below into one template class?
-#ifndef ALICEO2_FIT_RAWREADERFT0_H_
-#define ALICEO2_FIT_RAWREADERFT0_H_
-#include 
-#include 
-#include 
-#include "FT0Raw/RawReaderFT0Base.h"
-
-#include "DataFormatsFT0/Digit.h"
-#include "DataFormatsFT0/ChannelData.h"
-
-#include "Framework/ProcessingContext.h"
-#include "Framework/DataAllocator.h"
-#include "Framework/OutputSpec.h"
-#include 
-
-namespace o2
-{
-namespace ft0
-{
-//Normal TCM mode
-template 
-class RawReaderFT0 : public RawReaderFT0BaseNorm
-{
- public:
-  RawReaderFT0(bool dumpData) : mDumpData(dumpData) {}
-  RawReaderFT0(const RawReaderFT0&) = default;
-
-  RawReaderFT0() = default;
-  ~RawReaderFT0() = default;
-  static constexpr bool sUseTrgInput = useTrgInput;
-  void clear()
-  {
-    mVecDigits.clear();
-    if constexpr (sUseTrgInput) {
-      mVecTriggerInput.clear();
-    }
-    mVecChannelData.clear();
-  }
-  void accumulateDigits()
-  {
-    if constexpr (sUseTrgInput) {
-      getDigits(mVecDigits, mVecChannelData, mVecTriggerInput);
-    } else {
-      getDigits(mVecDigits, mVecChannelData);
-    }
-    LOG(info) << "Number of Digits: " << mVecDigits.size();
-    LOG(info) << "Number of ChannelData: " << mVecChannelData.size();
-    if constexpr (sUseTrgInput) {
-      LOG(info) << "Number of TriggerInput: " << mVecTriggerInput.size();
-    }
-    if (mDumpData) {
-      DigitBlockFT0::print(mVecDigits, mVecChannelData);
-    }
-  }
-  static void prepareOutputSpec(std::vector& outputSpec)
-  {
-    outputSpec.emplace_back(o2::header::gDataOriginFT0, "DIGITSBC", 0, o2::framework::Lifetime::Timeframe);
-    outputSpec.emplace_back(o2::header::gDataOriginFT0, "DIGITSCH", 0, o2::framework::Lifetime::Timeframe);
-    if constexpr (sUseTrgInput) {
-      outputSpec.emplace_back(o2::header::gDataOriginFT0, "TRIGGERINPUT", 0, o2::framework::Lifetime::Timeframe);
-    }
-  }
-  void makeSnapshot(o2::framework::ProcessingContext& pc)
-  {
-    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "DIGITSBC", 0, o2::framework::Lifetime::Timeframe}, mVecDigits);
-    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "DIGITSCH", 0, o2::framework::Lifetime::Timeframe}, mVecChannelData);
-    if constexpr (sUseTrgInput) {
-      pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "TRIGGERINPUT", 0, o2::framework::Lifetime::Timeframe}, mVecTriggerInput);
-    }
-  }
-  bool mDumpData;
-  std::vector mVecDigits;
-  std::vector mVecTriggerInput;
-  std::vector mVecChannelData;
-};
-
-//Extended TCM mode (additional raw data struct)
-template 
-class RawReaderFT0ext : public RawReaderFT0BaseExt
-{
- public:
-  RawReaderFT0ext(bool dumpData) : mDumpData(dumpData) {}
-  RawReaderFT0ext(const RawReaderFT0ext&) = default;
-  static constexpr bool sUseTrgInput = useTrgInput;
-  RawReaderFT0ext() = default;
-  ~RawReaderFT0ext() = default;
-  void clear()
-  {
-    mVecDigits.clear();
-    mVecChannelData.clear();
-    mVecTrgExt.clear();
-    if constexpr (sUseTrgInput) {
-      mVecTriggerInput.clear();
-    }
-  }
-  void accumulateDigits()
-  {
-    if constexpr (sUseTrgInput) {
-      getDigits(mVecDigits, mVecChannelData, mVecTrgExt, mVecTriggerInput);
-    } else {
-      getDigits(mVecDigits, mVecChannelData, mVecTrgExt);
-    }
-    LOG(info) << "Number of Digits: " << mVecDigits.size();
-    LOG(info) << "Number of ChannelData: " << mVecChannelData.size();
-    LOG(info) << "Number of TriggerExt: " << mVecTrgExt.size();
-    if (mDumpData) {
-      DigitBlockFT0ext::print(mVecDigits, mVecChannelData, mVecTrgExt);
-    }
-  }
-  static void prepareOutputSpec(std::vector& outputSpec)
-  {
-    outputSpec.emplace_back(o2::header::gDataOriginFT0, "DIGITSBC", 0, o2::framework::Lifetime::Timeframe);
-    outputSpec.emplace_back(o2::header::gDataOriginFT0, "DIGITSCH", 0, o2::framework::Lifetime::Timeframe);
-    outputSpec.emplace_back(o2::header::gDataOriginFT0, "DIGITSTRGEXT", 0, o2::framework::Lifetime::Timeframe);
-    if constexpr (sUseTrgInput) {
-      outputSpec.emplace_back(o2::header::gDataOriginFT0, "TRIGGERINPUT", 0, o2::framework::Lifetime::Timeframe);
-    }
-  }
-  void makeSnapshot(o2::framework::ProcessingContext& pc)
-  {
-    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "DIGITSBC", 0, o2::framework::Lifetime::Timeframe}, mVecDigits);
-    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "DIGITSCH", 0, o2::framework::Lifetime::Timeframe}, mVecChannelData);
-    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "DIGITSTRGEXT", 0, o2::framework::Lifetime::Timeframe}, mVecTrgExt);
-    if constexpr (sUseTrgInput) {
-      pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "TRIGGERINPUT", 0, o2::framework::Lifetime::Timeframe}, mVecTriggerInput);
-    }
-  }
-  bool mDumpData;
-  std::vector mVecDigits;
-  std::vector mVecChannelData;
-  std::vector mVecTrgExt;
-  std::vector mVecTriggerInput;
-};
-
-} // namespace ft0
-} // namespace o2
-
-#endif
\ No newline at end of file
diff --git a/Detectors/FIT/FT0/workflow/include/FT0Workflow/RecoQCworkflow.h b/Detectors/FIT/FT0/workflow/include/FT0Workflow/RecoQCworkflow.h
index e40f9f808052a..729626f47819c 100644
--- a/Detectors/FIT/FT0/workflow/include/FT0Workflow/RecoQCworkflow.h
+++ b/Detectors/FIT/FT0/workflow/include/FT0Workflow/RecoQCworkflow.h
@@ -16,7 +16,7 @@
 #ifndef O2_RECOQC_WORKFLOW
 #define O2_RECOQC_WORKFLOW
 
-#include 
+#include 
 #include 
 #include 
 #include "Framework/DeviceSpec.h"
diff --git a/Detectors/FIT/FT0/workflow/include/FT0Workflow/RecoWorkflow.h b/Detectors/FIT/FT0/workflow/include/FT0Workflow/RecoWorkflow.h
index 0df02634472ee..6de23a1c66bfd 100644
--- a/Detectors/FIT/FT0/workflow/include/FT0Workflow/RecoWorkflow.h
+++ b/Detectors/FIT/FT0/workflow/include/FT0Workflow/RecoWorkflow.h
@@ -20,7 +20,7 @@ namespace o2
 {
 namespace ft0
 {
-framework::WorkflowSpec getRecoWorkflow(bool useMC, std::string ccdbpath, bool disableRootInp, bool disableRootOut);
+framework::WorkflowSpec getRecoWorkflow(bool useMC, std::string ccdbpath, bool useTimeOffsetCalib, bool useSlewingCalib, bool disableRootInp, bool disableRootOut, bool useDeadChannelMap = true);
 } // namespace ft0
 } // namespace o2
 #endif
diff --git a/Detectors/FIT/FT0/workflow/include/FT0Workflow/ReconstructionSpec.h b/Detectors/FIT/FT0/workflow/include/FT0Workflow/ReconstructionSpec.h
index 4c3bbd9e0a0a9..307b2109fe35f 100644
--- a/Detectors/FIT/FT0/workflow/include/FT0Workflow/ReconstructionSpec.h
+++ b/Detectors/FIT/FT0/workflow/include/FT0Workflow/ReconstructionSpec.h
@@ -18,7 +18,6 @@
 #include "Framework/Task.h"
 #include "FT0Reconstruction/CollisionTimeRecoTask.h"
 #include "DataFormatsFT0/RecPoints.h"
-#include "CCDB/BasicCCDBManager.h"
 #include "FT0Base/Geometry.h"
 #include "TStopwatch.h"
 #include "CommonUtils/NameConf.h"
@@ -35,7 +34,7 @@ class ReconstructionDPL : public Task
   static constexpr int NCHANNELS = o2::ft0::Geometry::Nchannels;
 
  public:
-  ReconstructionDPL(bool useMC, const std::string ccdbpath) : mUseMC(useMC), mCCDBpath(ccdbpath) {}
+  ReconstructionDPL(bool useMC, const std::string& ccdbpath, bool useTimeOffsetCalib, bool useSlewingCalib, bool useDeadChannelMap) : mUseMC(useMC), mCCDBpath(ccdbpath), mUseTimeOffsetCalib(useTimeOffsetCalib), mUseSlewingCalib(useSlewingCalib), mUseDeadChannelMap(useDeadChannelMap) {}
   ~ReconstructionDPL() override = default;
   void init(InitContext& ic) final;
   void run(ProcessingContext& pc) final;
@@ -45,6 +44,10 @@ class ReconstructionDPL : public Task
  private:
   bool mUseMC = false;
   bool mUpdateCCDB = true;
+  bool mUseTimeOffsetCalib = true;
+  bool mUseSlewingCalib = true;
+  bool mUseDeadChannelMap = true;
+  bool mUpdateDeadChannelMap = true;
   const std::string mCCDBpath = o2::base::NameConf::getCCDBServer();
   std::vector mRecPoints;
   std::vector mRecChData;
@@ -54,7 +57,7 @@ class ReconstructionDPL : public Task
 };
 
 /// create a processor spec
-framework::DataProcessorSpec getReconstructionSpec(bool useMC = false, const std::string ccdbpath = "http://alice-ccdb.cern.ch");
+framework::DataProcessorSpec getReconstructionSpec(bool useMC = false, const std::string ccdbpath = "http://alice-ccdb.cern.ch", bool useTimeOffsetCalib = true, bool useSlewingCalib = true, bool useDeadChannelMap = true);
 
 } // namespace ft0
 } // namespace o2
diff --git a/Detectors/FIT/FT0/workflow/src/DigitReaderSpec.cxx b/Detectors/FIT/FT0/workflow/src/DigitReaderSpec.cxx
index e0ddedbe0c98b..09586d778ac15 100644
--- a/Detectors/FIT/FT0/workflow/src/DigitReaderSpec.cxx
+++ b/Detectors/FIT/FT0/workflow/src/DigitReaderSpec.cxx
@@ -34,7 +34,7 @@ void DigitReader::init(InitContext& ic)
 {
   auto filename = o2::utils::Str::concat_string(o2::utils::Str::rectifyDirectory(ic.options().get("input-dir")),
                                                 ic.options().get("ft0-digit-infile"));
-  mFile = std::make_unique(filename.c_str(), "OLD");
+  mFile.reset(TFile::Open(filename.c_str()));
   if (!mFile->IsOpen()) {
     LOG(error) << "Cannot open the " << filename.c_str() << " file !";
     throw std::runtime_error("cannot open input digits file");
@@ -64,13 +64,13 @@ void DigitReader::run(ProcessingContext& pc)
   assert(ent < mTree->GetEntries()); // this should not happen
   mTree->GetEntry(ent);
   LOG(debug) << "FT0DigitReader pushed " << channels.size() << " channels in " << digits.size() << " digits";
-  pc.outputs().snapshot(Output{"FT0", "DIGITSBC", 0, Lifetime::Timeframe}, digits);
-  pc.outputs().snapshot(Output{"FT0", "DIGITSCH", 0, Lifetime::Timeframe}, channels);
+  pc.outputs().snapshot(Output{"FT0", "DIGITSBC", 0}, digits);
+  pc.outputs().snapshot(Output{"FT0", "DIGITSCH", 0}, channels);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{"FT0", "DIGITSMCTR", 0, Lifetime::Timeframe}, labels);
+    pc.outputs().snapshot(Output{"FT0", "DIGITSMCTR", 0}, labels);
   }
   if (mUseTrgInput) {
-    pc.outputs().snapshot(Output{"FT0", "TRIGGERINPUT", 0, Lifetime::Timeframe}, trgInput);
+    pc.outputs().snapshot(Output{"FT0", "TRIGGERINPUT", 0}, trgInput);
   }
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get().endOfStream();
diff --git a/Detectors/FIT/FT0/workflow/src/EntropyDecoderSpec.cxx b/Detectors/FIT/FT0/workflow/src/EntropyDecoderSpec.cxx
index 43948d3aadb3f..066c5cc547c2e 100644
--- a/Detectors/FIT/FT0/workflow/src/EntropyDecoderSpec.cxx
+++ b/Detectors/FIT/FT0/workflow/src/EntropyDecoderSpec.cxx
@@ -24,12 +24,12 @@ namespace o2
 {
 namespace ft0
 {
-
-EntropyDecoderSpec::EntropyDecoderSpec(int verbosity) : mCTFCoder(o2::ctf::CTFCoderBase::OpType::Decoder)
+EntropyDecoderSpec::EntropyDecoderSpec(int verbosity, const std::string& ctfdictOpt) : mCTFCoder(o2::ctf::CTFCoderBase::OpType::Decoder, ctfdictOpt)
 {
   mTimer.Stop();
   mTimer.Reset();
   mCTFCoder.setVerbosity(verbosity);
+  mCTFCoder.setDictBinding("ctfdict_FT0");
 }
 
 void EntropyDecoderSpec::finaliseCCDB(o2::framework::ConcreteDataMatcher& matcher, void* obj)
@@ -50,8 +50,8 @@ void EntropyDecoderSpec::run(ProcessingContext& pc)
   mTimer.Start(false);
   o2::ctf::CTFIOSize iosize;
 
-  mCTFCoder.updateTimeDependentParams(pc);
-  auto buff = pc.inputs().get>("ctf");
+  mCTFCoder.updateTimeDependentParams(pc, true);
+  auto buff = pc.inputs().get>("ctf_FT0");
 
   auto& digits = pc.outputs().make>(OutputRef{"digits"});
   auto& channels = pc.outputs().make>(OutputRef{"channels"});
@@ -72,7 +72,7 @@ void EntropyDecoderSpec::endOfStream(EndOfStreamContext& ec)
        mTimer.CpuTime(), mTimer.RealTime(), mTimer.Counter() - 1);
 }
 
-DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec)
+DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec, const std::string& ctfdictOpt)
 {
   std::vector outputs{
     OutputSpec{{"digits"}, "FT0", "DIGITSBC", 0, Lifetime::Timeframe},
@@ -80,16 +80,19 @@ DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec)
     OutputSpec{{"ctfrep"}, "FT0", "CTFDECREP", 0, Lifetime::Timeframe}};
 
   std::vector inputs;
-  inputs.emplace_back("ctf", "FT0", "CTFDATA", sspec, Lifetime::Timeframe);
-  inputs.emplace_back("ctfdict", "FT0", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("FT0/Calib/CTFDictionary"));
+  inputs.emplace_back("ctf_FT0", "FT0", "CTFDATA", sspec, Lifetime::Timeframe);
+
+  if (ctfdictOpt.empty() || ctfdictOpt == "ccdb") {
+    inputs.emplace_back("ctfdict_FT0", "FT0", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("FT0/Calib/CTFDictionaryTree"));
+  }
+  inputs.emplace_back("trigoffset", "CTP", "Trig_Offset", 0, Lifetime::Condition, ccdbParamSpec("CTP/Config/TriggerOffsets"));
 
   return DataProcessorSpec{
     "ft0-entropy-decoder",
     inputs,
     outputs,
-    AlgorithmSpec{adaptFromTask(verbosity)},
-    Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}}};
+    AlgorithmSpec{adaptFromTask(verbosity, ctfdictOpt)},
+    Options{{"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}};
 }
-
 } // namespace ft0
 } // namespace o2
diff --git a/Detectors/FIT/FT0/workflow/src/EntropyEncoderSpec.cxx b/Detectors/FIT/FT0/workflow/src/EntropyEncoderSpec.cxx
index f27c546f8a684..7be6618a61103 100644
--- a/Detectors/FIT/FT0/workflow/src/EntropyEncoderSpec.cxx
+++ b/Detectors/FIT/FT0/workflow/src/EntropyEncoderSpec.cxx
@@ -25,8 +25,7 @@ namespace o2
 {
 namespace ft0
 {
-
-EntropyEncoderSpec::EntropyEncoderSpec() : mCTFCoder(o2::ctf::CTFCoderBase::OpType::Encoder)
+EntropyEncoderSpec::EntropyEncoderSpec(bool selIR, const std::string& ctfdictOpt) : mCTFCoder(o2::ctf::CTFCoderBase::OpType::Encoder, ctfdictOpt), mSelIR(selIR)
 {
   mTimer.Stop();
   mTimer.Reset();
@@ -48,12 +47,18 @@ void EntropyEncoderSpec::run(ProcessingContext& pc)
 {
   auto cput = mTimer.CpuTime();
   mTimer.Start(false);
-  mCTFCoder.updateTimeDependentParams(pc);
+  mCTFCoder.updateTimeDependentParams(pc, true);
   auto digits = pc.inputs().get>("digits");
   auto channels = pc.inputs().get>("channels");
+  if (mSelIR) {
+    mCTFCoder.setSelectedIRFrames(pc.inputs().get>("selIRFrames"));
+  }
 
-  auto& buffer = pc.outputs().make>(Output{"FT0", "CTFDATA", 0, Lifetime::Timeframe});
+  auto& buffer = pc.outputs().make>(Output{"FT0", "CTFDATA", 0});
   auto iosize = mCTFCoder.encode(buffer, digits, channels);
+  if (mSelIR) {
+    mCTFCoder.getIRFramesSelector().clear();
+  }
   mTimer.Stop();
   LOG(info) << iosize.asString() << " in " << mTimer.CpuTime() - cput << " s";
 }
@@ -64,21 +69,27 @@ void EntropyEncoderSpec::endOfStream(EndOfStreamContext& ec)
        mTimer.CpuTime(), mTimer.RealTime(), mTimer.Counter() - 1);
 }
 
-DataProcessorSpec getEntropyEncoderSpec()
+DataProcessorSpec getEntropyEncoderSpec(bool selIR, const std::string& ctfdictOpt)
 {
   std::vector inputs;
   inputs.emplace_back("digits", "FT0", "DIGITSBC", 0, Lifetime::Timeframe);
   inputs.emplace_back("channels", "FT0", "DIGITSCH", 0, Lifetime::Timeframe);
-  inputs.emplace_back("ctfdict", "FT0", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("FT0/Calib/CTFDictionary"));
 
+  if (ctfdictOpt.empty() || ctfdictOpt == "ccdb") {
+    inputs.emplace_back("ctfdict", "FT0", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("FT0/Calib/CTFDictionaryTree"));
+  }
+  if (selIR) {
+    inputs.emplace_back("selIRFrames", "CTF", "SELIRFRAMES", 0, Lifetime::Timeframe);
+  }
   return DataProcessorSpec{
     "ft0-entropy-encoder",
     inputs,
     Outputs{{"FT0", "CTFDATA", 0, Lifetime::Timeframe}},
-    AlgorithmSpec{adaptFromTask()},
-    Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}},
-            {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}}}};
+    AlgorithmSpec{adaptFromTask(selIR, ctfdictOpt)},
+    Options{{"irframe-margin-bwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame lower boundary when selection is requested"}},
+            {"irframe-margin-fwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame upper boundary when selection is requested"}},
+            {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}},
+            {"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}};
 }
-
 } // namespace ft0
 } // namespace o2
diff --git a/Detectors/FIT/FT0/workflow/src/FT0DataDecoderDPLSpec.cxx b/Detectors/FIT/FT0/workflow/src/FT0DataDecoderDPLSpec.cxx
index c0df0def41d51..6d81740dd2173 100644
--- a/Detectors/FIT/FT0/workflow/src/FT0DataDecoderDPLSpec.cxx
+++ b/Detectors/FIT/FT0/workflow/src/FT0DataDecoderDPLSpec.cxx
@@ -12,6 +12,7 @@
 /// @file   FITDataDecoderDPLSpec.cxx
 
 #include "FT0Workflow/FT0DataDecoderDPLSpec.h"
+#include "DetectorsRaw/RDHUtils.h"
 #include 
 #include 
 #include 
@@ -27,6 +28,12 @@ namespace ft0
 void FT0DataDecoderDPLSpec::run(ProcessingContext& pc)
 {
   auto t1 = std::chrono::high_resolution_clock::now();
+  auto dummyOutput = [&pc, this]() {
+    this->mVecDigits.resize(0);
+    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "DIGITSBC", 0}, mVecDigits);
+    this->mVecChannelData.resize(0);
+    pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "DIGITSCH", 0}, mVecChannelData);
+  };
 
   // if we see requested data type input with 0xDEADBEEF subspec and 0 payload this means that the "delayed message"
   // mechanism created it in absence of real data from upstream. Processor should send empty output to not block the workflow
@@ -43,10 +50,7 @@ void FT0DataDecoderDPLSpec::run(ProcessingContext& pc)
                dh->dataOrigin.str, dh->dataDescription.str, dh->subSpecification, dh->tfCounter, dh->firstTForbit, payloadSize,
                contDeadBeef == maxWarn ? fmt::format(". {} such inputs in row received, stopping reporting", contDeadBeef) : "");
         }
-        mVecDigits.resize(0);
-        pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "DIGITSBC", 0, o2::framework::Lifetime::Timeframe}, mVecDigits);
-        mVecChannelData.resize(0);
-        pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "DIGITSCH", 0, o2::framework::Lifetime::Timeframe}, mVecChannelData);
+        dummyOutput();
         return;
       }
     }
@@ -70,21 +74,34 @@ void FT0DataDecoderDPLSpec::run(ProcessingContext& pc)
     if (!it.size()) {
       continue; // excluding pages without payload
     }
-    auto rdhPtr = it.get_if();
-    const uint16_t orbitTF = (rdhPtr->orbit) % 256;
+    auto* rdhPtr = reinterpret_cast(it.raw());
+    try {
+      int verRDH = o2::raw::RDHUtils::getVersion(rdhPtr);
+      if (verRDH < 5 || verRDH > o2::raw::RDHUtils::getVersion()) {
+        LOGP(alarm, "Invalid RDH version {}, abandoning TF sending dummy output", verRDH);
+        dummyOutput();
+        return;
+      }
+    } catch (std::exception& e) {
+      LOG(alarm) << "Failed to extract RDH, abandoning TF sending dummy output, exception was: " << e.what();
+      dummyOutput();
+      return;
+    }
+    auto orb = o2::raw::RDHUtils::getHeartBeatOrbit(rdhPtr);
+    const uint16_t orbitTF = (orb) % 256;
     // const uint16_t feeID=rdhPtr->feeId;
-    arrOrbit[orbitTF] = rdhPtr->orbit;
-    const auto& linkID = rdhPtr->linkID;
-    const auto& endPoint = rdhPtr->endPointID;
+    arrOrbit[orbitTF] = orb;
+    const auto& linkID = o2::raw::RDHUtils::getLinkID(rdhPtr);
+    const auto& endPoint = o2::raw::RDHUtils::getEndPointID(rdhPtr);
     const uint16_t feeID = linkID + 12 * endPoint;
     if (feeID == mFEEID_TCM) {
       // Iterator is noncopyable, preparing RDH pointers and span objects
       arrOrbitSizePagesTCM[orbitTF] += it.size();
-      arrRdhTCMperOrbit[orbitTF].push_back(it.get_if());
+      arrRdhTCMperOrbit[orbitTF].push_back(reinterpret_cast(rdhPtr));
       arrDataTCMperOrbit[orbitTF].emplace_back(it.data(), it.size());
     } else {
       arrOrbitSizePages[orbitTF] += it.size();
-      arrRdhPtrPerOrbit[orbitTF][feeID].push_back(it.get_if());
+      arrRdhPtrPerOrbit[orbitTF][feeID].push_back(reinterpret_cast(rdhPtr));
       arrDataPerOrbit[orbitTF][feeID].emplace_back(it.data(), it.size());
     }
   }
@@ -511,8 +528,8 @@ void FT0DataDecoderDPLSpec::run(ProcessingContext& pc)
     // Due to empty Digit container this dummy object will never participate in any further tasks.
     mVecChannelData.emplace_back();
   }
-  pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "DIGITSBC", 0, o2::framework::Lifetime::Timeframe}, mVecDigits);
-  pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "DIGITSCH", 0, o2::framework::Lifetime::Timeframe}, mVecChannelData);
+  pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "DIGITSBC", 0}, mVecDigits);
+  pc.outputs().snapshot(o2::framework::Output{o2::header::gDataOriginFT0, "DIGITSCH", 0}, mVecChannelData);
   auto t2 = std::chrono::high_resolution_clock::now();
   auto delay = std::chrono::duration_cast(t2 - t1);
   LOG(debug) << "Decoder delay: " << delay.count();
diff --git a/Detectors/FIT/FT0/workflow/src/FT0DataProcessDPLSpec.cxx b/Detectors/FIT/FT0/workflow/src/FT0DataProcessDPLSpec.cxx
deleted file mode 100644
index d7a7a689d402f..0000000000000
--- a/Detectors/FIT/FT0/workflow/src/FT0DataProcessDPLSpec.cxx
+++ /dev/null
@@ -1,52 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-/// @file   FT0DataProcessDPLSpec.cxx
-
-#include "FT0Workflow/FT0DataProcessDPLSpec.h"
-
-using namespace o2::framework;
-
-namespace o2
-{
-namespace ft0
-{
-using namespace std;
-void FT0DataProcessDPLSpec::init(InitContext& ic)
-{
-}
-
-void FT0DataProcessDPLSpec::run(ProcessingContext& pc)
-{
-  LOG(info) << "FT0DataProcessDPLSpec running...";
-  auto vecDigits = pc.inputs().get>("digits");
-  auto vecChannelData = pc.inputs().get>("digch");
-  if (mDumpEventBlocks) {
-    DigitBlockFT0::print(vecDigits, vecChannelData);
-  }
-}
-
-DataProcessorSpec getFT0DataProcessDPLSpec(bool dumpProcessor)
-{
-  std::vector inputSpec;
-  inputSpec.emplace_back("digits", o2::header::gDataOriginFT0, "DIGITSBC", 0, Lifetime::Timeframe);
-  inputSpec.emplace_back("digch", o2::header::gDataOriginFT0, "DIGITSCH", 0, Lifetime::Timeframe);
-  LOG(info) << "DataProcessorSpec getFT0DataProcessDPLSpec";
-  return DataProcessorSpec{
-    "ft0-dataprocess-dpl-flp",
-    inputSpec,
-    Outputs{},
-    AlgorithmSpec{adaptFromTask(dumpProcessor)},
-    Options{}};
-}
-
-} // namespace ft0
-} // namespace o2
diff --git a/Detectors/FIT/FT0/workflow/src/FT0DataReaderDPLSpec.cxx b/Detectors/FIT/FT0/workflow/src/FT0DataReaderDPLSpec.cxx
deleted file mode 100644
index caa642794b561..0000000000000
--- a/Detectors/FIT/FT0/workflow/src/FT0DataReaderDPLSpec.cxx
+++ /dev/null
@@ -1,24 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-/// @file   FT0DataReaderDPLSpec.cxx
-
-#include "FT0Workflow/FT0DataReaderDPLSpec.h"
-
-using namespace o2::framework;
-
-namespace o2
-{
-namespace ft0
-{
-
-} // namespace ft0
-} // namespace o2
diff --git a/Detectors/FIT/FT0/workflow/src/FT0Workflow.cxx b/Detectors/FIT/FT0/workflow/src/FT0Workflow.cxx
deleted file mode 100644
index 156feb7dd3e2f..0000000000000
--- a/Detectors/FIT/FT0/workflow/src/FT0Workflow.cxx
+++ /dev/null
@@ -1,45 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-/// @file   FT0Workflow.cxx
-
-#include "FT0Workflow/FT0Workflow.h"
-#include "FT0Workflow/FT0DataProcessDPLSpec.h"
-#include "FT0Workflow/FT0DataReaderDPLSpec.h"
-#include "FT0Workflow/FT0DigitWriterSpec.h"
-#include "FT0Workflow/RawReaderFT0.h"
-namespace o2
-{
-namespace ft0
-{
-
-framework::WorkflowSpec getFT0Workflow(bool isExtendedMode, bool useProcess,
-                                       bool dumpProcessor, bool dumpReader,
-                                       bool disableRootOut, bool askSTFDist)
-{
-  LOG(info) << "framework::WorkflowSpec getFT0Workflow";
-  framework::WorkflowSpec specs;
-  if (isExtendedMode) {
-    specs.emplace_back(o2::ft0::getFT0DataReaderDPLSpec(RawReaderFT0ext{dumpReader}, askSTFDist));
-  } else {
-    specs.emplace_back(o2::ft0::getFT0DataReaderDPLSpec(RawReaderFT0{dumpReader}, askSTFDist));
-  }
-  if (useProcess) {
-    specs.emplace_back(o2::ft0::getFT0DataProcessDPLSpec(dumpProcessor));
-  }
-  if (!disableRootOut) {
-    specs.emplace_back(o2::ft0::getFT0DigitWriterSpec(false, false));
-  }
-  return specs;
-}
-
-} // namespace ft0
-} // namespace o2
diff --git a/Detectors/FIT/FT0/workflow/src/RawReaderFT0.cxx b/Detectors/FIT/FT0/workflow/src/RawReaderFT0.cxx
deleted file mode 100644
index b2ef17e540112..0000000000000
--- a/Detectors/FIT/FT0/workflow/src/RawReaderFT0.cxx
+++ /dev/null
@@ -1,13 +0,0 @@
-// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
-// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
-// All rights not expressly granted are reserved.
-//
-// This software is distributed under the terms of the GNU General Public
-// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
-//
-// In applying this license CERN does not waive the privileges and immunities
-// granted to it by virtue of its status as an Intergovernmental Organization
-// or submit itself to any jurisdiction.
-
-#include "FT0Workflow/RawReaderFT0.h"
-using namespace o2::ft0;
diff --git a/Detectors/FIT/FT0/workflow/src/RecPointReaderSpec.cxx b/Detectors/FIT/FT0/workflow/src/RecPointReaderSpec.cxx
index 69f21abc369c6..ba5ae4aa1356c 100644
--- a/Detectors/FIT/FT0/workflow/src/RecPointReaderSpec.cxx
+++ b/Detectors/FIT/FT0/workflow/src/RecPointReaderSpec.cxx
@@ -49,8 +49,8 @@ void RecPointReader::run(ProcessingContext& pc)
   mTree->GetEntry(ent);
 
   LOG(debug) << "FT0 RecPointReader pushes " << mRecPoints->size() << " recpoints with " << mChannelData->size() << " channels at entry " << ent;
-  pc.outputs().snapshot(Output{mOrigin, "RECPOINTS", 0, Lifetime::Timeframe}, *mRecPoints);
-  pc.outputs().snapshot(Output{mOrigin, "RECCHDATA", 0, Lifetime::Timeframe}, *mChannelData);
+  pc.outputs().snapshot(Output{mOrigin, "RECPOINTS", 0}, *mRecPoints);
+  pc.outputs().snapshot(Output{mOrigin, "RECCHDATA", 0}, *mChannelData);
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get().endOfStream();
diff --git a/Detectors/FIT/FT0/workflow/src/RecoQCworkflow.cxx b/Detectors/FIT/FT0/workflow/src/RecoQCworkflow.cxx
index 67744948417da..6993c46197dac 100644
--- a/Detectors/FIT/FT0/workflow/src/RecoQCworkflow.cxx
+++ b/Detectors/FIT/FT0/workflow/src/RecoQCworkflow.cxx
@@ -13,7 +13,7 @@
 ///\ brief QC for  reconstructed data
 /// \author Alla.Maevskaya@cern.ch
 
-#include 
+#include 
 #include 
 #include "Framework/DeviceSpec.h"
 #include "Framework/WorkflowSpec.h"
@@ -132,7 +132,7 @@ DataProcessorSpec getRecoQCworkflow(GID::mask_t src)
 {
   auto dataRequest = std::make_shared();
   LOG(info) << "@@ request primary vertex";
-  dataRequest->requestPrimaryVertertices(false);
+  dataRequest->requestPrimaryVertices(false);
   dataRequest->requestFT0RecPoints(false);
   LOG(info) << "@@@ requested T0";
   std::vector outputs; // empty
diff --git a/Detectors/FIT/FT0/workflow/src/RecoWorkflow.cxx b/Detectors/FIT/FT0/workflow/src/RecoWorkflow.cxx
index 8c5ab34166760..2231011febd7f 100644
--- a/Detectors/FIT/FT0/workflow/src/RecoWorkflow.cxx
+++ b/Detectors/FIT/FT0/workflow/src/RecoWorkflow.cxx
@@ -22,13 +22,13 @@ namespace o2
 namespace ft0
 {
 
-framework::WorkflowSpec getRecoWorkflow(bool useMC, std::string ccdbpath, bool disableRootInp, bool disableRootOut)
+framework::WorkflowSpec getRecoWorkflow(bool useMC, std::string ccdbpath, bool useTimeOffsetCalib, bool useSlewingCalib, bool disableRootInp, bool disableRootOut, bool useDeadChannelMap)
 {
   framework::WorkflowSpec specs;
   if (!disableRootInp) {
     specs.emplace_back(o2::ft0::getDigitReaderSpec(useMC));
   }
-  specs.emplace_back(o2::ft0::getReconstructionSpec(useMC, ccdbpath));
+  specs.emplace_back(o2::ft0::getReconstructionSpec(useMC, ccdbpath, useTimeOffsetCalib, useSlewingCalib, useDeadChannelMap));
   if (!disableRootOut) {
     specs.emplace_back(o2::ft0::getRecPointWriterSpec(useMC));
   }
diff --git a/Detectors/FIT/FT0/workflow/src/ReconstructionSpec.cxx b/Detectors/FIT/FT0/workflow/src/ReconstructionSpec.cxx
index 62ccc9a5b408a..bc5217c8d7471 100644
--- a/Detectors/FIT/FT0/workflow/src/ReconstructionSpec.cxx
+++ b/Detectors/FIT/FT0/workflow/src/ReconstructionSpec.cxx
@@ -19,8 +19,10 @@
 #include "FT0Workflow/ReconstructionSpec.h"
 #include "DataFormatsFT0/Digit.h"
 #include "DataFormatsFT0/ChannelData.h"
+#include "DataFormatsFT0/DigitFilterParam.h"
+#include "DataFormatsFT0/CalibParam.h"
 #include "DataFormatsFT0/MCLabel.h"
-#include "DataFormatsFT0/FT0ChannelTimeCalibrationObject.h"
+#include "DataFormatsFT0/SpectraInfoObject.h"
 #include "Framework/CCDBParamSpec.h"
 
 using namespace o2::framework;
@@ -34,66 +36,71 @@ void ReconstructionDPL::init(InitContext& ic)
 {
   mTimer.Stop();
   mTimer.Reset();
-  LOG(info) << "ReconstructionDPL::init";
+  o2::ft0::ChannelFilterParam::Instance().printKeyValues();
+  o2::ft0::TimeFilterParam::Instance().printKeyValues();
+  // Parameters which are used in reco, too many will be printed if use printKeyValues()
+  LOG(info) << "FT0 param mMinEntriesThreshold: " << CalibParam::Instance().mMinEntriesThreshold;
+  LOG(info) << "FT0 param mMaxEntriesThreshold:" << CalibParam::Instance().mMaxEntriesThreshold;
+  LOG(info) << "FT0 param mMinRMS: " << CalibParam::Instance().mMinRMS;
+  LOG(info) << "FT0 param mMaxSigma: " << CalibParam::Instance().mMaxSigma;
+  LOG(info) << "FT0 param mMaxDiffMean: " << CalibParam::Instance().mMaxDiffMean;
+  LOG(info) << "FT0 dead channel map will be applied " << mUseDeadChannelMap;
 }
 
 void ReconstructionDPL::run(ProcessingContext& pc)
 {
-  /*
-  auto creationTime = pc.services().get().creation;
-  auto& mCCDBManager = o2::ccdb::BasicCCDBManager::instance();
-  mCCDBManager.setURL(mCCDBpath);
-  mCCDBManager.setTimestamp(creationTime);
-  LOG(debug) << " set-up CCDB " << mCCDBpath << " creationTime " << creationTime;
-  */
   mTimer.Start(false);
   mRecPoints.clear();
+  mRecChData.clear();
   auto digits = pc.inputs().get>("digits");
-  auto digch = pc.inputs().get>("digch");
+  auto channels = pc.inputs().get>("digch");
   // RS: if we need to process MC truth, uncomment lines below
   // std::unique_ptr> labels;
   // const o2::dataformats::MCTruthContainer* lblPtr = nullptr;
   if (mUseMC) {
     LOG(info) << "Ignoring MC info";
   }
-  if (mUpdateCCDB) {
-    auto caliboffsets = pc.inputs().get("ft0offsets");
-    mReco.SetChannelOffset(caliboffsets.get());
-    LOG(info) << "RecoSpec  mReco.SetChannelOffset(&caliboffsets)";
+  if (mUseTimeOffsetCalib) {
+    auto timeOffsetCalibObject = pc.inputs().get("ft0_timespectra");
+    mReco.SetTimeCalibObject(timeOffsetCalibObject.get());
   }
-  /*
-  auto calibslew = mCCDBManager.get>("FT0/SlewingCorr");
-  LOG(debug) << " calibslew " << calibslew;
-  if (calibslew) {
-    mReco.SetSlew(calibslew);
-    LOG(info) << " calibslew set slew " << calibslew;
+
+  if (mUseSlewingCalib) {
+    auto slewingCalibObject = pc.inputs().get("ft0_slewing_coef");
+    mReco.SetSlewingCalibObject(slewingCalibObject.get());
   }
-  */
-  int nDig = digits.size();
-  LOG(debug) << " nDig " << nDig;
-  mRecPoints.reserve(nDig);
-  mRecChData.resize(digch.size());
-  for (int id = 0; id < nDig; id++) {
-    const auto& digit = digits[id];
-    LOG(debug) << " ndig " << id << " bc " << digit.getBC() << " orbit " << digit.getOrbit();
-    auto channels = digit.getBunchChannelData(digch);
-    gsl::span out_ch(mRecChData);
-    out_ch = out_ch.subspan(digit.ref.getFirstEntry(), digit.ref.getEntries());
-    mRecPoints.emplace_back(mReco.process(digit, channels, out_ch));
+
+  if (mUseDeadChannelMap && mUpdateDeadChannelMap) {
+    LOG(debug) << "Applying dead channel map";
+    auto deadChannelMap = pc.inputs().get("deadChannelMap");
+    mReco.SetDeadChannelMap(deadChannelMap.get());
   }
-  // do we ignore MC in this task?
 
+  mRecPoints.reserve(digits.size());
+  mRecChData.reserve(channels.size());
+  mReco.processTF(digits, channels, mRecPoints, mRecChData);
+  // do we ignore MC in this task?
   LOG(debug) << "FT0 reconstruction pushes " << mRecPoints.size() << " RecPoints";
-  pc.outputs().snapshot(Output{mOrigin, "RECPOINTS", 0, Lifetime::Timeframe}, mRecPoints);
-  pc.outputs().snapshot(Output{mOrigin, "RECCHDATA", 0, Lifetime::Timeframe}, mRecChData);
+  pc.outputs().snapshot(Output{mOrigin, "RECPOINTS", 0}, mRecPoints);
+  pc.outputs().snapshot(Output{mOrigin, "RECCHDATA", 0}, mRecChData);
 
   mTimer.Stop();
 }
 //_______________________________________
 void ReconstructionDPL::finaliseCCDB(ConcreteDataMatcher& matcher, void* obj)
 {
-  if (matcher == ConcreteDataMatcher("FT0", "TimeOffset", 0)) {
-    mUpdateCCDB = false;
+  if (matcher == ConcreteDataMatcher("FT0", "TimeSpectraInfo", 0)) {
+    LOG(debug) << "New TimeSpectraInfo is uploaded";
+    return;
+  }
+  if (matcher == ConcreteDataMatcher("FT0", "SlewingCoef", 0)) {
+    LOG(debug) << "New SlewingCoef is uploaded";
+    mUseSlewingCalib = false; // upload only once, slewing should be stable during the run
+    return;
+  }
+  if (matcher == ConcreteDataMatcher("FT0", "DeadChannelMap", 0)) {
+    LOG(debug) << "New DeadChannelMap is uploaded";
+    mUpdateDeadChannelMap = false;
     return;
   }
 }
@@ -104,19 +111,33 @@ void ReconstructionDPL::endOfStream(EndOfStreamContext& ec)
        mTimer.CpuTime(), mTimer.RealTime(), mTimer.Counter() - 1);
 }
 
-DataProcessorSpec getReconstructionSpec(bool useMC, const std::string ccdbpath)
+DataProcessorSpec getReconstructionSpec(bool useMC, const std::string ccdbpath, bool useTimeOffsetCalib, bool useSlewingCalib, bool useDeadChannelMap)
 {
   std::vector inputSpec;
   std::vector outputSpec;
   inputSpec.emplace_back("digits", o2::header::gDataOriginFT0, "DIGITSBC", 0, Lifetime::Timeframe);
   inputSpec.emplace_back("digch", o2::header::gDataOriginFT0, "DIGITSCH", 0, Lifetime::Timeframe);
+
   if (useMC) {
     LOG(info) << "Currently Reconstruction does not consume and provide MC truth";
     inputSpec.emplace_back("labels", o2::header::gDataOriginFT0, "DIGITSMCTR", 0, Lifetime::Timeframe);
   }
-  inputSpec.emplace_back("ft0offsets", "FT0", "TimeOffset", 0,
-                         Lifetime::Condition,
-                         ccdbParamSpec("FT0/Calib/ChannelTimeOffset"));
+  if (useTimeOffsetCalib) {
+    inputSpec.emplace_back("ft0_timespectra", "FT0", "TimeSpectraInfo", 0,
+                           Lifetime::Condition,
+                           ccdbParamSpec("FT0/Calib/TimeSpectraInfo", {}, 1));
+  }
+
+  if (useSlewingCalib) {
+    inputSpec.emplace_back("ft0_slewing_coef", "FT0", "SlewingCoef", 0,
+                           Lifetime::Condition,
+                           ccdbParamSpec("FT0/Calib/SlewingCoef"));
+  }
+
+  if (useDeadChannelMap) {
+    LOG(info) << "Dead channel map will be applied during reconstruction";
+    inputSpec.emplace_back("deadChannelMap", o2::header::gDataOriginFT0, "DeadChannelMap", 0, Lifetime::Condition, ccdbParamSpec("FT0/Calib/DeadChannelMap"));
+  }
 
   outputSpec.emplace_back(o2::header::gDataOriginFT0, "RECPOINTS", 0, Lifetime::Timeframe);
   outputSpec.emplace_back(o2::header::gDataOriginFT0, "RECCHDATA", 0, Lifetime::Timeframe);
@@ -125,7 +146,7 @@ DataProcessorSpec getReconstructionSpec(bool useMC, const std::string ccdbpath)
     "ft0-reconstructor",
     inputSpec,
     outputSpec,
-    AlgorithmSpec{adaptFromTask(useMC, ccdbpath)},
+    AlgorithmSpec{adaptFromTask(useMC, ccdbpath, useTimeOffsetCalib, useSlewingCalib, useDeadChannelMap)},
     Options{}};
 }
 
diff --git a/Detectors/FIT/FT0/workflow/src/cluster-integrator-reader.cxx b/Detectors/FIT/FT0/workflow/src/cluster-integrator-reader.cxx
new file mode 100644
index 0000000000000..fc1d04be2ca82
--- /dev/null
+++ b/Detectors/FIT/FT0/workflow/src/cluster-integrator-reader.cxx
@@ -0,0 +1,23 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "FITWorkflow/FITIntegrateClusterReaderSpec.h"
+#include "DataFormatsFT0/RecPoints.h"
+#include "Framework/runDataProcessing.h"
+
+using namespace o2::framework;
+
+WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
+{
+  WorkflowSpec wf;
+  wf.emplace_back(o2::fit::getFITIntegrateClusterReaderSpec());
+  return wf;
+}
diff --git a/Detectors/FIT/FT0/workflow/src/cluster-integrator.cxx b/Detectors/FIT/FT0/workflow/src/cluster-integrator.cxx
new file mode 100644
index 0000000000000..474c9d9d477c2
--- /dev/null
+++ b/Detectors/FIT/FT0/workflow/src/cluster-integrator.cxx
@@ -0,0 +1,44 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "FITWorkflow/FITIntegrateClusterSpec.h"
+#include "DataFormatsFT0/RecPoints.h"
+#include "FITWorkflow/FITIntegrateClusterWriterSpec.h"
+#include "CommonUtils/ConfigurableParam.h"
+#include "Framework/ConfigParamSpec.h"
+
+using namespace o2::framework;
+
+void customize(std::vector& workflowOptions)
+{
+  std::vector options{
+    ConfigParamSpec{"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}},
+    {"min-NChan", VariantType::Int, 0, {"Minimum NChan signal required to avoid noise"}},
+    {"min-Ampl", VariantType::Int, 0, {"Minimum Ampl signal required to avoid noise"}},
+    {"disable-root-output", VariantType::Bool, false, {"disable root-files output writers"}}};
+  std::swap(workflowOptions, options);
+}
+
+#include "Framework/runDataProcessing.h"
+
+WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
+{
+  WorkflowSpec wf;
+  o2::conf::ConfigurableParam::updateFromString(cfgc.options().get("configKeyValues"));
+  const bool disableWriter = cfgc.options().get("disable-root-output");
+  const int minNChan = cfgc.options().get("min-NChan");
+  const int minAmpl = cfgc.options().get("min-Ampl");
+  wf.emplace_back(o2::fit::getFITIntegrateClusterSpec(disableWriter, minNChan, minAmpl));
+  if (!disableWriter) {
+    wf.emplace_back(o2::fit::getFITIntegrateClusterWriterSpec());
+  }
+  return wf;
+}
diff --git a/Detectors/FIT/FT0/workflow/src/cluster-merge-integrator.cxx b/Detectors/FIT/FT0/workflow/src/cluster-merge-integrator.cxx
new file mode 100644
index 0000000000000..cf7fdbd071094
--- /dev/null
+++ b/Detectors/FIT/FT0/workflow/src/cluster-merge-integrator.cxx
@@ -0,0 +1,35 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "FITWorkflow/FITMergeIntegrateClusterSpec.h"
+#include "DataFormatsFT0/RecPoints.h"
+#include "CommonUtils/ConfigurableParam.h"
+#include "Framework/ConfigParamSpec.h"
+
+using namespace o2::framework;
+
+void customize(std::vector& workflowOptions)
+{
+  std::vector options{
+    ConfigParamSpec{"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}},
+  };
+  std::swap(workflowOptions, options);
+}
+
+#include "Framework/runDataProcessing.h"
+
+WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
+{
+  WorkflowSpec wf;
+  o2::conf::ConfigurableParam::updateFromString(cfgc.options().get("configKeyValues"));
+  wf.emplace_back(o2::fit::getFITMergeIntegrateClusterSpec());
+  return wf;
+}
diff --git a/Detectors/FIT/FT0/workflow/src/entropy-encoder-workflow.cxx b/Detectors/FIT/FT0/workflow/src/entropy-encoder-workflow.cxx
index 25271680222cc..144d27abeda9c 100644
--- a/Detectors/FIT/FT0/workflow/src/entropy-encoder-workflow.cxx
+++ b/Detectors/FIT/FT0/workflow/src/entropy-encoder-workflow.cxx
@@ -21,7 +21,10 @@ using namespace o2::framework;
 void customize(std::vector& workflowOptions)
 {
   // option allowing to set parameters
-  std::vector options{ConfigParamSpec{"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}}};
+  std::vector options{
+    ConfigParamSpec{"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}},
+    ConfigParamSpec{"ctf-dict", VariantType::String, "none", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}},
+    ConfigParamSpec{"select-ir-frames", VariantType::Bool, false, {"Subscribe and filter according to external IR Frames"}}};
 
   std::swap(workflowOptions, options);
 }
@@ -35,6 +38,6 @@ WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
   WorkflowSpec wf;
   // Update the (declared) parameters if changed from the command line
   o2::conf::ConfigurableParam::updateFromString(cfgc.options().get("configKeyValues"));
-  wf.emplace_back(o2::ft0::getEntropyEncoderSpec());
+  wf.emplace_back(o2::ft0::getEntropyEncoderSpec(cfgc.options().get("select-ir-frames"), cfgc.options().get("ctf-dict")));
   return wf;
 }
diff --git a/Detectors/FIT/FT0/workflow/src/ft0-flp-workflow.cxx b/Detectors/FIT/FT0/workflow/src/ft0-flp-workflow.cxx
index c067a8ed9d7cc..df48a145508fd 100644
--- a/Detectors/FIT/FT0/workflow/src/ft0-flp-workflow.cxx
+++ b/Detectors/FIT/FT0/workflow/src/ft0-flp-workflow.cxx
@@ -63,6 +63,16 @@ void customize(std::vector& workflowOptions)
                     o2::framework::VariantType::Bool,
                     false,
                     {"do not subscribe to FLP/DISTSUBTIMEFRAME/0 message (no lost TF recovery)"}});
+  workflowOptions.push_back(
+    ConfigParamSpec{"input-sub-sampled",
+                    o2::framework::VariantType::Bool,
+                    false,
+                    {"SUB_RAWDATA DPL channel will be used as input, in case of dispatcher usage"}});
+  workflowOptions.push_back(
+    ConfigParamSpec{"disable-dpl-ccdb-fetcher",
+                    o2::framework::VariantType::Bool,
+                    false,
+                    {"Disable DPL CCDB fetcher, channel map will be uploaded during initialization by taking last entry in CCDB"}});
 #if defined(FT0_NEW_DECOER_ON)
   workflowOptions.push_back(
     ConfigParamSpec{"new-decoder",
@@ -83,6 +93,8 @@ WorkflowSpec defineDataProcessing(ConfigContext const& configcontext)
   auto isExtendedMode = configcontext.options().get("tcm-extended-mode");
   auto disableRootOut = configcontext.options().get("disable-root-output");
   auto askSTFDist = !configcontext.options().get("ignore-dist-stf");
+  const auto isSubSampled = configcontext.options().get("input-sub-sampled");
+  const auto disableDplCcdbFetcher = configcontext.options().get("disable-dpl-ccdb-fetcher");
   bool isNewDecoder = false;
 #if defined(FT0_NEW_DECOER_ON)
   isNewDecoder = configcontext.options().get("new-decoder");
@@ -100,12 +112,12 @@ WorkflowSpec defineDataProcessing(ConfigContext const& configcontext)
   WorkflowSpec specs;
   if (!isNewDecoder) {
     if (isExtendedMode) {
-      specs.emplace_back(o2::fit::getFITDataReaderDPLSpec(RawReaderFT0ext{dataOrigin, dumpReader}, askSTFDist));
+      specs.emplace_back(o2::fit::getFITDataReaderDPLSpec(RawReaderFT0ext{dataOrigin, dumpReader}, askSTFDist, isSubSampled, disableDplCcdbFetcher));
       if (!disableRootOut) {
         specs.emplace_back(o2::fit::FITDigitWriterSpecHelper::getFITDigitWriterSpec(false, false, dataOrigin));
       }
     } else {
-      specs.emplace_back(o2::fit::getFITDataReaderDPLSpec(RawReaderFT0{dataOrigin, dumpReader}, askSTFDist));
+      specs.emplace_back(o2::fit::getFITDataReaderDPLSpec(RawReaderFT0{dataOrigin, dumpReader}, askSTFDist, isSubSampled, disableDplCcdbFetcher));
       if (!disableRootOut) {
         specs.emplace_back(o2::fit::FITDigitWriterSpecHelper::getFITDigitWriterSpec(false, false, dataOrigin));
       }
diff --git a/Detectors/FIT/FT0/workflow/src/ft0-reco-workflow.cxx b/Detectors/FIT/FT0/workflow/src/ft0-reco-workflow.cxx
index ee84c8b35a5f6..ab39068aedb38 100644
--- a/Detectors/FIT/FT0/workflow/src/ft0-reco-workflow.cxx
+++ b/Detectors/FIT/FT0/workflow/src/ft0-reco-workflow.cxx
@@ -39,7 +39,10 @@ void customize(std::vector& workflowOptions)
     {"disable-mc", o2::framework::VariantType::Bool, false, {"disable MC propagation even if available"}},
     {"disable-root-input", o2::framework::VariantType::Bool, false, {"disable root-files input readers"}},
     {"disable-root-output", o2::framework::VariantType::Bool, false, {"disable root-files output writers"}},
-    {"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}}};
+    {"disable-time-offset-calib", o2::framework::VariantType::Bool, false, {"disable timeoffset calibration"}},
+    {"disable-slewing-calib", o2::framework::VariantType::Bool, false, {"disable slewing calibration"}},
+    {"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}},
+    {"disable-dead-channel-map", VariantType::Bool, false, {"disable dead channel map"}}};
   o2::raw::HBFUtilsInitializer::addConfigOption(options);
   std::swap(workflowOptions, options);
 }
@@ -60,9 +63,12 @@ WorkflowSpec defineDataProcessing(ConfigContext const& configcontext)
   auto ccdbpath = o2::base::NameConf::getCCDBServer();
   auto disableRootInp = configcontext.options().get("disable-root-input");
   auto disableRootOut = configcontext.options().get("disable-root-output");
+  const auto useTimeOffsetCalib = !configcontext.options().get("disable-time-offset-calib");
+  const auto useSlewingCalib = !configcontext.options().get("disable-slewing-calib");
+  const auto useDeadChannelMap = !configcontext.options().get("disable-dead-channel-map");
 
   LOG(info) << "WorkflowSpec getRecoWorkflow useMC " << useMC << " CCDB  " << ccdbpath;
-  auto wf = o2::ft0::getRecoWorkflow(useMC, ccdbpath, disableRootInp, disableRootOut);
+  auto wf = o2::ft0::getRecoWorkflow(useMC, ccdbpath, useTimeOffsetCalib, useSlewingCalib, disableRootInp, disableRootOut, useDeadChannelMap);
 
   // configure dpl timer to inject correct firstTForbit: start from the 1st orbit of TF containing 1st sampled orbit
   o2::raw::HBFUtilsInitializer hbfIni(configcontext, wf);
diff --git a/Detectors/FIT/FT0/workflow/src/recpoints-reader-workflow.cxx b/Detectors/FIT/FT0/workflow/src/recpoints-reader-workflow.cxx
index 376a8385941ce..b1d824e10687e 100644
--- a/Detectors/FIT/FT0/workflow/src/recpoints-reader-workflow.cxx
+++ b/Detectors/FIT/FT0/workflow/src/recpoints-reader-workflow.cxx
@@ -9,29 +9,35 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 
-/// \file recpoints-reader-workflow.cxx
-/// \brief Implementation of FT0 digits reader
+/// \file  recpoints-reader-workflow.cxx
+/// \brief FT0 RecPoints reader workflow
 ///
-/// \author ruben.shahoyan@cern.ch
+/// \author ruben.shahoyan@cern.ch, Andreas Molander andreas.molander@cern.ch
 
-#include "Framework/CallbackService.h"
-#include "Framework/ControlService.h"
-#include "Framework/ConfigParamRegistry.h"
-#include "Framework/Task.h"
-#include "FT0Workflow/RecPointReaderSpec.h"
 #include "CommonUtils/ConfigurableParam.h"
+#include "DetectorsRaw/HBFUtilsInitializer.h"
+#include "Framework/CallbacksPolicy.h"
+#include "Framework/ConfigParamSpec.h"
+#include "Framework/Variant.h"
+
+#include "FT0Workflow/RecPointReaderSpec.h"
+
+#include 
 
 using namespace o2::framework;
 
+void customize(std::vector& policies)
+{
+  o2::raw::HBFUtilsInitializer::addNewTimeSliceCallback(policies);
+}
+
 // we need to add workflow options before including Framework/runDataProcessing
 void customize(std::vector& workflowOptions)
 {
-  // option allowing to set parameters
-
-  std::vector options{
-    {"disable-mc", o2::framework::VariantType::Bool, false, {"disable MC propagation even if available"}}};
-  std::string keyvaluehelp("Semicolon separated key=value strings");
-  options.push_back(ConfigParamSpec{"configKeyValues", VariantType::String, "", {keyvaluehelp}});
+  std::vector options{
+    {"disable-mc", VariantType::Bool, false, {"disable MC propagation even if available"}},
+    {"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}}};
+  o2::raw::HBFUtilsInitializer::addConfigOption(options);
   std::swap(workflowOptions, options);
 }
 
@@ -39,9 +45,14 @@ void customize(std::vector& workflowOptions)
 
 WorkflowSpec defineDataProcessing(const ConfigContext& ctx)
 {
-  WorkflowSpec specs;
   o2::conf::ConfigurableParam::updateFromString(ctx.options().get("configKeyValues"));
-  DataProcessorSpec producer = o2::ft0::getRecPointReaderSpec(ctx.options().get("disable-mc"));
+  bool disableMC = ctx.options().get("disable-mc");
+
+  WorkflowSpec specs;
+  DataProcessorSpec producer = o2::ft0::getRecPointReaderSpec(!disableMC);
   specs.push_back(producer);
+
+  // configure dpl timer to inject correct firstTForbit: start from the 1st orbit of TF containing 1st sampled orbit
+  o2::raw::HBFUtilsInitializer hbfIni(ctx, specs);
   return specs;
 }
diff --git a/Detectors/FIT/FT0/workflow/src/recpoints-writer-workflow.cxx b/Detectors/FIT/FT0/workflow/src/recpoints-writer-workflow.cxx
new file mode 100644
index 0000000000000..c8baef76b5ccd
--- /dev/null
+++ b/Detectors/FIT/FT0/workflow/src/recpoints-writer-workflow.cxx
@@ -0,0 +1,47 @@
+// Copyright 2019-2024 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file  recpoints-writer-workflow.cxx
+/// \brief FT0 RecPoints writer workflow
+///
+/// \author Andreas Molander andreas.molander@cern.ch
+
+#include "CommonUtils/ConfigurableParam.h"
+#include "Framework/ConfigParamSpec.h"
+#include "Framework/Variant.h"
+
+#include "FT0Workflow/RecPointWriterSpec.h"
+
+#include 
+
+using namespace o2::framework;
+
+// we need to add workflow options before including Framework/runDataProcessing
+void customize(std::vector& workflowOptions)
+{
+  std::vector options{
+    {"disable-mc", VariantType::Bool, false, {"disable MC propagation even if available"}},
+    {"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}}};
+  workflowOptions.insert(workflowOptions.end(), options.begin(), options.end());
+}
+
+#include "Framework/runDataProcessing.h"
+
+WorkflowSpec defineDataProcessing(const ConfigContext& ctx)
+{
+  o2::conf::ConfigurableParam::updateFromString(ctx.options().get("configKeyValues"));
+  bool disableMC = ctx.options().get("disable-mc");
+
+  WorkflowSpec specs;
+  DataProcessorSpec producer = o2::ft0::getRecPointWriterSpec(!disableMC);
+  specs.push_back(producer);
+  return specs;
+}
diff --git a/Detectors/FIT/FV0/CMakeLists.txt b/Detectors/FIT/FV0/CMakeLists.txt
index fa31a5f6b65c7..cd76f3a24fc28 100644
--- a/Detectors/FIT/FV0/CMakeLists.txt
+++ b/Detectors/FIT/FV0/CMakeLists.txt
@@ -10,6 +10,10 @@
 # or submit itself to any jurisdiction.
 
 add_subdirectory(base)
+add_subdirectory(dcsmonitoring)
+if(BUILD_TESTING)
+    add_subdirectory(dcsmonitoring/macros)
+endif()
 add_subdirectory(raw)
 add_subdirectory(reconstruction)
 add_subdirectory(simulation)
diff --git a/Detectors/FIT/FV0/base/README.md b/Detectors/FIT/FV0/base/README.md
new file mode 100644
index 0000000000000..1fba33c47b690
--- /dev/null
+++ b/Detectors/FIT/FV0/base/README.md
@@ -0,0 +1,17 @@
+
+
+# FV0 Base
+
+## Geometry
+
+The `o2::fv0::Geometry` class represents the geometry of the FV0 detector as used in simulation. It also provides utility methods for retrieving the center locations of the detector cells. See the below example for how to use the `Geometry` class to query the FV0 cell locations. Note that these are the ideal locations, and that any misalignment is not considered.
+
+```cpp
+o2::fv0::Geometry* fv0Geometry = o2::fv0::Geometry::instance(o2::fv0::Geometry::eUninitialized);
+o2::fv0::Point3Dsimple cellPosition = fv0Geometry->getReadoutCenter(chId);
+float x = cellPosition.x;
+float y = cellPosition.y;
+float z = cellPosition.z;
+```
diff --git a/Detectors/FIT/FV0/base/include/FV0Base/Constants.h b/Detectors/FIT/FV0/base/include/FV0Base/Constants.h
index b8342c54306d8..c91c000394856 100644
--- a/Detectors/FIT/FV0/base/include/FV0Base/Constants.h
+++ b/Detectors/FIT/FV0/base/include/FV0Base/Constants.h
@@ -30,6 +30,7 @@ struct Constants {
   static constexpr int nTcms = 1;           // Number of trigger and clock modules (TCMs)
   static constexpr int nGbtLinks = nPms + nTcms;
   static constexpr int nFv0Channels = Geometry::getNumberOfReadoutChannels();
+  static constexpr int nFv0ChannelsPlusRef = nFv0Channels + 1;
 };
 
 } // namespace fv0
diff --git a/Detectors/FIT/FV0/base/include/FV0Base/Geometry.h b/Detectors/FIT/FV0/base/include/FV0Base/Geometry.h
index 3b50be7441ec2..ec87c07c57c45 100644
--- a/Detectors/FIT/FV0/base/include/FV0Base/Geometry.h
+++ b/Detectors/FIT/FV0/base/include/FV0Base/Geometry.h
@@ -133,6 +133,16 @@ class Geometry
     return o2::base::GeometryManager::getPNEntry(getDetID(), index);
   }
 
+  static std::string getDetectorRightSymName()
+  {
+    return sDetectorRightName + "_0";
+  }
+
+  static std::string getDetectorLeftSymName()
+  {
+    return sDetectorLeftName + "_1";
+  }
+
   /// Get the density of the PMTs.
   static constexpr float getPmtDensity()
   {
@@ -143,6 +153,8 @@ class Geometry
   explicit Geometry(EGeoType initType);
 
   inline static const std::string sDetectorName = "FV0";
+  inline static const std::string sDetectorRightName = sDetectorName + "RIGHT";
+  inline static const std::string sDetectorLeftName = sDetectorName + "LEFT";
 
   // General geometry constants
   static constexpr float sEpsilon = 0.01;     ///< Used to make one spatial dimension infinitesimally larger than other
diff --git a/Detectors/FIT/FV0/base/src/Geometry.cxx b/Detectors/FIT/FV0/base/src/Geometry.cxx
index 2f357c263c8a6..4eec725ea1abc 100644
--- a/Detectors/FIT/FV0/base/src/Geometry.cxx
+++ b/Detectors/FIT/FV0/base/src/Geometry.cxx
@@ -19,7 +19,7 @@
 
 #include 
 
-#include 
+#include 
 
 #include 
 #include 
diff --git a/Detectors/FIT/FV0/calibration/include/FV0Calibration/FV0CalibCollector.h b/Detectors/FIT/FV0/calibration/include/FV0Calibration/FV0CalibCollector.h
index 6324d8ef30104..722bf8b07f71e 100644
--- a/Detectors/FIT/FV0/calibration/include/FV0Calibration/FV0CalibCollector.h
+++ b/Detectors/FIT/FV0/calibration/include/FV0Calibration/FV0CalibCollector.h
@@ -58,7 +58,7 @@ class FV0CalibInfoSlot
   ClassDefNV(FV0CalibInfoSlot, 1);
 };
 
-class FV0CalibCollector final : public o2::calibration::TimeSlotCalibration
+class FV0CalibCollector final : public o2::calibration::TimeSlotCalibration
 {
   using TFType = o2::calibration::TFType;
   using Slot = o2::calibration::TimeSlot;
diff --git a/Detectors/FIT/FV0/calibration/include/FV0Calibration/FV0ChannelTimeOffsetSlotContainer.h b/Detectors/FIT/FV0/calibration/include/FV0Calibration/FV0ChannelTimeOffsetSlotContainer.h
index 35bd3edf00195..9127cb2819e62 100644
--- a/Detectors/FIT/FV0/calibration/include/FV0Calibration/FV0ChannelTimeOffsetSlotContainer.h
+++ b/Detectors/FIT/FV0/calibration/include/FV0Calibration/FV0ChannelTimeOffsetSlotContainer.h
@@ -56,7 +56,7 @@ class FV0ChannelTimeOffsetSlotContainer final
   {
     return mFirstCreation;
   }
-  FV0ChannelTimeCalibrationObject generateCalibrationObject() const;
+  FV0ChannelTimeCalibrationObject generateCalibrationObject(long, long, const std::string&) const;
 
  private:
   std::size_t mMinEntries;
diff --git a/Detectors/FIT/FV0/calibration/macros/CMakeLists.txt b/Detectors/FIT/FV0/calibration/macros/CMakeLists.txt
index 4723d4f98d3e5..36f3df6a7dd84 100644
--- a/Detectors/FIT/FV0/calibration/macros/CMakeLists.txt
+++ b/Detectors/FIT/FV0/calibration/macros/CMakeLists.txt
@@ -1,3 +1,14 @@
+# Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+# See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+# All rights not expressly granted are reserved.
+#
+# This software is distributed under the terms of the GNU General Public
+# License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+#
+# In applying this license CERN does not waive the privileges and immunities
+# granted to it by virtue of its status as an Intergovernmental Organization
+# or submit itself to any jurisdiction.
+
 o2_add_test_root_macro(
   makeChannelTimeOffsetFV0CalibObjectInCCDB.C
   PUBLIC_LINK_LIBRARIES O2::DetectorsCommonDataFormats
@@ -21,7 +32,7 @@ o2_add_test_root_macro(
 )
 
 install(
-  FILES makeChannelTimeOffsetFV0CalibObjectInCCDB.C 
+  FILES makeChannelTimeOffsetFV0CalibObjectInCCDB.C
         readChannelTimeOffsetFV0CalibObjectFromCCDB.C
   DESTINATION share/macro/
 )
diff --git a/Detectors/FIT/FV0/calibration/macros/readChannelTimeOffsetFV0CalibObjectFromCCDB.C b/Detectors/FIT/FV0/calibration/macros/readChannelTimeOffsetFV0CalibObjectFromCCDB.C
index 06b86e3c5015d..3f42c0219b101 100644
--- a/Detectors/FIT/FV0/calibration/macros/readChannelTimeOffsetFV0CalibObjectFromCCDB.C
+++ b/Detectors/FIT/FV0/calibration/macros/readChannelTimeOffsetFV0CalibObjectFromCCDB.C
@@ -22,8 +22,8 @@ int readChannelTimeOffsetFV0CalibObjectFromCCDB(const std::string url = "http://
 {
   o2::ccdb::CcdbApi api;
   api.init(url);
-  map metadata;
-  map headers;
+  std::map metadata;
+  std::map headers;
   auto retrieved = api.retrieveFromTFileAny("FV0/Calib/ChannelTimeOffset", metadata, -1, &headers);
 
   std::cout << "--- HEADERS ---" << std::endl;
diff --git a/Detectors/FIT/FV0/calibration/src/FV0CalibrationLinkDef.h b/Detectors/FIT/FV0/calibration/src/FV0CalibrationLinkDef.h
index 290fb62d207a6..9474ca1375fd9 100644
--- a/Detectors/FIT/FV0/calibration/src/FV0CalibrationLinkDef.h
+++ b/Detectors/FIT/FV0/calibration/src/FV0CalibrationLinkDef.h
@@ -17,7 +17,9 @@
 
 #pragma link C++ class o2::fv0::FV0ChannelTimeOffsetSlotContainer + ;
 #pragma link C++ class o2::fv0::FV0CalibCollector + ;
-#pragma link C++ class o2::calibration::TimeSlotCalibration < o2::fv0::FV0CalibrationInfoObject, o2::fv0::FV0CalibInfoSlot>;
-#pragma link C++ class o2::calibration::TimeSlotCalibration < o2::fv0::FV0CalibrationInfoObject, o2::fv0::FV0ChannelTimeOffsetSlotContainer>;
+#pragma link C++ class o2::calibration::TimeSlot < o2::fv0::FV0CalibInfoSlot>;
+#pragma link C++ class o2::calibration::TimeSlotCalibration < o2::fv0::FV0CalibInfoSlot>;
+#pragma link C++ class o2::calibration::TimeSlot < o2::fv0::FV0ChannelTimeOffsetSlotContainer>;
+#pragma link C++ class o2::calibration::TimeSlotCalibration < o2::fv0::FV0ChannelTimeOffsetSlotContainer>;
 
 #endif
diff --git a/Detectors/FIT/FV0/calibration/src/FV0ChannelTimeOffsetSlotContainer.cxx b/Detectors/FIT/FV0/calibration/src/FV0ChannelTimeOffsetSlotContainer.cxx
index 4c7bb973f249e..c66a94d4c6b8d 100644
--- a/Detectors/FIT/FV0/calibration/src/FV0ChannelTimeOffsetSlotContainer.cxx
+++ b/Detectors/FIT/FV0/calibration/src/FV0ChannelTimeOffsetSlotContainer.cxx
@@ -103,7 +103,7 @@ int16_t FV0ChannelTimeOffsetSlotContainer::getMeanGaussianFitValue(std::size_t c
 
   return static_cast(std::round(outputGaussianFitValues[MEAN_VALUE_INDEX_IN_OUTPUT_VECTOR]));
 }
-FV0ChannelTimeCalibrationObject FV0ChannelTimeOffsetSlotContainer::generateCalibrationObject() const
+FV0ChannelTimeCalibrationObject FV0ChannelTimeOffsetSlotContainer::generateCalibrationObject(long, long, const std::string&) const
 {
   FV0ChannelTimeCalibrationObject calibrationObject;
 
diff --git a/Detectors/FIT/FV0/calibration/workflow/FV0ChannelTimeCalibrationSpec.h b/Detectors/FIT/FV0/calibration/workflow/FV0ChannelTimeCalibrationSpec.h
index d209afdf3670e..f71a464854ad2 100644
--- a/Detectors/FIT/FV0/calibration/workflow/FV0ChannelTimeCalibrationSpec.h
+++ b/Detectors/FIT/FV0/calibration/workflow/FV0ChannelTimeCalibrationSpec.h
@@ -25,15 +25,12 @@ o2::framework::DataProcessorSpec getFV0ChannelTimeCalibrationSpec()
 {
   using CalibrationDeviceType = o2::fit::FITCalibrationDevice;
-
-  std::vector outputs;
-  outputs.emplace_back(o2::framework::ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBPayload, "FIT_CALIB"}, o2::framework::Lifetime::Sporadic);
-  outputs.emplace_back(o2::framework::ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBWrapper, "FIT_CALIB"}, o2::framework::Lifetime::Sporadic);
-
-  constexpr const char* DEFAULT_INPUT_LABEL = "calib";
-
   std::vector inputs;
-  inputs.emplace_back(DEFAULT_INPUT_LABEL, "FV0", "CALIB_INFO");
+  std::vector outputs;
+  const o2::header::DataDescription inputDataDescriptor{"CALIB_INFO"};
+  const o2::header::DataDescription outputDataDescriptor{"FV0_TIME_CALIB"};
+  CalibrationDeviceType::prepareVecInputSpec(inputs, o2::header::gDataOriginFV0, inputDataDescriptor);
+  CalibrationDeviceType::prepareVecOutputSpec(outputs, outputDataDescriptor);
   auto ccdbRequest = std::make_shared(true,                           // orbitResetTime
                                                                 true,                           // GRPECS=true
                                                                 false,                          // GRPLHCIF
@@ -45,11 +42,12 @@ o2::framework::DataProcessorSpec getFV0ChannelTimeCalibrationSpec()
     "calib-fv0-channel-time",
     inputs,
     outputs,
-    o2::framework::AlgorithmSpec{o2::framework::adaptFromTask(DEFAULT_INPUT_LABEL, ccdbRequest)},
+    o2::framework::AlgorithmSpec{o2::framework::adaptFromTask(ccdbRequest, outputDataDescriptor)},
     o2::framework::Options{
       {"tf-per-slot", o2::framework::VariantType::UInt32, 5u, {""}},
       {"max-delay", o2::framework::VariantType::UInt32, 3u, {""}},
-      {"updateInterval", o2::framework::VariantType::UInt32, 10u, {""}}}};
+      {"updateInterval", o2::framework::VariantType::UInt32, 10u, {""}},
+      {"extra-info-per-slot", o2::framework::VariantType::String, "", {"Extra info for time slot(usually for debugging)"}}}};
 }
 } // namespace o2::fv0
 
diff --git a/Detectors/FIT/FV0/dcsmonitoring/CMakeLists.txt b/Detectors/FIT/FV0/dcsmonitoring/CMakeLists.txt
new file mode 100644
index 0000000000000..7508e24fc069c
--- /dev/null
+++ b/Detectors/FIT/FV0/dcsmonitoring/CMakeLists.txt
@@ -0,0 +1,33 @@
+# Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+# See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+# All rights not expressly granted are reserved.
+#
+# This software is distributed under the terms of the GNU General Public
+# License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+#
+# In applying this license CERN does not waive the privileges and immunities
+# granted to it by virtue of its status as an Intergovernmental Organization
+# or submit itself to any jurisdiction.
+
+o2_add_library(FV0DCSMonitoring
+               SOURCES src/FV0DCSDataProcessor.cxx
+               PUBLIC_LINK_LIBRARIES O2::FITDCSMonitoring)
+
+o2_add_executable(dcs-sim-workflow
+                  COMPONENT_NAME fv0
+                  SOURCES workflow/fv0-dcs-sim-workflow.cxx
+                  PUBLIC_LINK_LIBRARIES O2::DCStestWorkflow)
+
+o2_add_executable(dcs-workflow
+                  COMPONENT_NAME fv0
+                  SOURCES workflow/fv0-dcs-data-workflow.cxx
+                  PUBLIC_LINK_LIBRARIES O2::DetectorsDCS
+                                        O2::Framework
+                                        O2::FV0DCSMonitoring)
+
+o2_add_executable(dcs-config-workflow
+                  COMPONENT_NAME fv0
+                  SOURCES workflow/fv0-dcs-config-workflow.cxx
+                  PUBLIC_LINK_LIBRARIES O2::DetectorsCalibration
+                                        O2::FITDCSMonitoring
+                                        O2::Framework)
diff --git a/Detectors/FIT/FV0/dcsmonitoring/include/FV0DCSMonitoring/FV0DCSDataProcessor.h b/Detectors/FIT/FV0/dcsmonitoring/include/FV0DCSMonitoring/FV0DCSDataProcessor.h
new file mode 100644
index 0000000000000..01cc27a8d036d
--- /dev/null
+++ b/Detectors/FIT/FV0/dcsmonitoring/include/FV0DCSMonitoring/FV0DCSDataProcessor.h
@@ -0,0 +1,44 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// @file  FV0DCSDataProcessor.h
+/// @brief Task for processing FV0 DCS data
+///
+/// \author Andreas Molander , University of Jyvaskyla, Finland
+
+#ifndef O2_FV0_DATAPROCESSOR_H
+#define O2_FV0_DATAPROCESSOR_H
+
+#include "DetectorsDCS/DataPointIdentifier.h"
+#include "FITDCSMonitoring/FITDCSDataProcessor.h"
+
+#include 
+#include 
+
+namespace o2
+{
+namespace fv0
+{
+
+class FV0DCSDataProcessor : public o2::fit::FITDCSDataProcessor
+{
+ public:
+  FV0DCSDataProcessor(const std::string detectorName, const o2::header::DataDescription& dataDescription)
+    : o2::fit::FITDCSDataProcessor(detectorName, dataDescription) {}
+
+ protected:
+  std::vector getHardCodedDPIDs() override;
+}; // end class
+
+} // namespace fv0
+} // namespace o2
+
+#endif // O2_FV0_DATAPROCESSOR_H
diff --git a/Detectors/FIT/FV0/dcsmonitoring/macros/CMakeLists.txt b/Detectors/FIT/FV0/dcsmonitoring/macros/CMakeLists.txt
new file mode 100644
index 0000000000000..446f01dde8ffe
--- /dev/null
+++ b/Detectors/FIT/FV0/dcsmonitoring/macros/CMakeLists.txt
@@ -0,0 +1,15 @@
+# Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+# See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+# All rights not expressly granted are reserved.
+#
+# This software is distributed under the terms of the GNU General Public
+# License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+#
+# In applying this license CERN does not waive the privileges and immunities
+# granted to it by virtue of its status as an Intergovernmental Organization
+# or submit itself to any jurisdiction.
+
+o2_add_test_root_macro(makeFV0CCDBEntryForDCS.C
+                       PUBLIC_LINK_LIBRARIES O2::CCDB
+                                             O2::DetectorsDCS
+                       LABELS fv0)
diff --git a/Detectors/FIT/FV0/dcsmonitoring/macros/makeFV0CCDBEntryForDCS.C b/Detectors/FIT/FV0/dcsmonitoring/macros/makeFV0CCDBEntryForDCS.C
new file mode 100644
index 0000000000000..ccbc45f90ba0e
--- /dev/null
+++ b/Detectors/FIT/FV0/dcsmonitoring/macros/makeFV0CCDBEntryForDCS.C
@@ -0,0 +1,98 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file makeFV0CCDBEntryForDCS.C
+/// \brief Macro for uploading a DCS data point definition object to CCDB
+///
+/// \author Andreas Molander , University of Jyvaskyla, Finland
+
+#include "CCDB/CcdbApi.h"
+#include "CCDB/CCDBTimeStampUtils.h"
+#include "DetectorsDCS/AliasExpander.h"
+#include "DetectorsDCS/DeliveryType.h"
+#include "DetectorsDCS/DataPointIdentifier.h"
+#include "TFile.h"
+
+#include 
+#include 
+#include 
+#include 
+
+using DPID = o2::dcs::DataPointIdentifier;
+
+int makeFV0CCDBEntryForDCS(const std::string ccdbUrl = "http://localhost:8080",
+                           const std::string fileName = "")
+{
+  // FV0/PM/??/actual/ADC0_BASELINE
+  // FV0/PM/??/actual/ADC1_BASELINE
+  // FV0/HV/??/actual/iMon
+
+  // where ?? can be SX1,SX2,SX3,SX4,SX51,SX52 - where X={A,B,C,D,E,F,G,H}
+  //                 or SREF
+  std::unordered_map dpid2DataDesc;
+  std::vector aliasesHV = {"FV0/HV/S[A,B,C,D,E,F,G,H][1..4]/actual/iMon",
+                                        "FV0/HV/S[A,B,C,D,E,F,G,H][51,52]/actual/iMon",
+                                        "FV0/HV/SREF/actual/iMon"};
+  std::vector aliasesADC = {"FV0/PM/S[A,B,C,D,E,F,G,H][1..4]/actual/ADC[0,1]_BASELINE",
+                                         "FV0/PM/S[A,B,C,D,E,F,G,H][51,52]/actual/ADC[0,1]_BASELINE",
+                                         "FV0/PM/SREF/actual/ADC[0,1]_BASELINE"};
+
+  std::vector aliasesRates = {"FV0/Trigger1_Charge/CNT_RATE",
+                                           "FV0/Trigger2_Nchannels/CNT_RATE",
+                                           "FV0/Trigger3_InnerRings/CNT_RATE",
+                                           "FV0/Trigger4_OuterRings/CNT_RATE",
+                                           "FV0/Trigger5_OrA/CNT_RATE",
+                                           "FV0/Background/[0..9]/CNT_RATE",
+                                           "FV0/Background/[A,B,C,D,E,F,G,H]/CNT_RATE"};
+  std::vector expAliasesHV = o2::dcs::expandAliases(aliasesHV);
+  std::vector expAliasesADC = o2::dcs::expandAliases(aliasesADC);
+  std::vector expAliasesRates = o2::dcs::expandAliases(aliasesRates);
+
+  LOG(info) << "DCS DP IDs:";
+
+  DPID dpIdTmp;
+  for (size_t i = 0; i < expAliasesHV.size(); i++) {
+    DPID::FILL(dpIdTmp, expAliasesHV[i], o2::dcs::DeliveryType::DPVAL_DOUBLE);
+    dpid2DataDesc[dpIdTmp] = "FV0DATAPOINTS";
+    LOG(info) << dpIdTmp;
+  }
+  for (size_t i = 0; i < expAliasesADC.size(); i++) {
+    DPID::FILL(dpIdTmp, expAliasesADC[i], o2::dcs::DeliveryType::DPVAL_UINT);
+    dpid2DataDesc[dpIdTmp] = "FV0DATAPOINTS";
+    LOG(info) << dpIdTmp;
+  }
+  for (size_t i = 0; i < expAliasesRates.size(); i++) {
+    DPID::FILL(dpIdTmp, expAliasesRates[i], o2::dcs::DeliveryType::DPVAL_DOUBLE);
+    dpid2DataDesc[dpIdTmp] = "FV0DATAPOINTS";
+    LOG(info) << dpIdTmp;
+  }
+
+  LOG(info) << "Total number of DPs: " << dpid2DataDesc.size();
+
+  if (!ccdbUrl.empty()) {
+    const std::string ccdbPath = "FV0/Config/DCSDPconfig";
+    LOGP(info, "Storing DCS DP definition object on {}/{}", ccdbUrl, ccdbPath);
+    o2::ccdb::CcdbApi api;
+    api.init(ccdbUrl);
+    std::map metadata;
+    long ts = o2::ccdb::getCurrentTimestamp();
+    api.storeAsTFileAny(&dpid2DataDesc, ccdbPath, metadata, ts, 99999999999999);
+  }
+
+  if (!fileName.empty()) {
+    LOG(info) << "Storing DCS DP definitions locally in " << fileName;
+    TFile file(fileName.c_str(), "recreate");
+    file.WriteObjectAny(&dpid2DataDesc, "std::unordered_map", "DCSDPconfig");
+    file.Close();
+  }
+
+  return 0;
+}
diff --git a/Detectors/FIT/FV0/dcsmonitoring/src/FV0DCSDataProcessor.cxx b/Detectors/FIT/FV0/dcsmonitoring/src/FV0DCSDataProcessor.cxx
new file mode 100644
index 0000000000000..c9750377df07e
--- /dev/null
+++ b/Detectors/FIT/FV0/dcsmonitoring/src/FV0DCSDataProcessor.cxx
@@ -0,0 +1,54 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// @file  FV0DCSDataProcessor.cxx
+/// @brief Task for processing FV0 DCS data
+///
+/// \author Andreas Molander , University of Jyvaskyla, Finland
+
+#include "FV0DCSMonitoring/FV0DCSDataProcessor.h"
+
+#include "DetectorsDCS/AliasExpander.h"
+#include "DetectorsDCS/DataPointIdentifier.h"
+
+#include 
+#include 
+
+std::vector o2::fv0::FV0DCSDataProcessor::getHardCodedDPIDs()
+{
+  std::vector vect;
+  std::vector aliasesHV = {"FV0/HV/S[A,B,C,D,E,F,G,H][1..4]/actual/iMon",
+                                        "FV0/HV/S[A,B,C,D,E,F,G,H][51,52]/actual/iMon",
+                                        "FV0/HV/SREF/actual/iMon"};
+  std::vector aliasesADC = {"FV0/PM/S[A,B,C,D,E,F,G,H][1..4]/actual/ADC[0,1]_BASELINE",
+                                         "FV0/PM/S[A,B,C,D,E,F,G,H][51,52]/actual/ADC[0,1]_BASELINE",
+                                         "FV0/PM/SREF/actual/ADC[0,1]_BASELINE"};
+  std::vector aliasesRates = {"FV0/Trigger1_Charge/CNT_RATE",
+                                           "FV0/Trigger2_Nchannels/CNT_RATE",
+                                           "FV0/Trigger3_InnerRings/CNT_RATE",
+                                           "FV0/Trigger4_OuterRings/CNT_RATE",
+                                           "FV0/Trigger5_OrA/CNT_RATE",
+                                           "FV0/Background/[0..9]/CNT_RATE",
+                                           "FV0/Background/[A,B,C,D,E,F,G,H]/CNT_RATE"};
+  std::vector expAliasesHV = o2::dcs::expandAliases(aliasesHV);
+  std::vector expAliasesADC = o2::dcs::expandAliases(aliasesADC);
+  std::vector expAliasesRates = o2::dcs::expandAliases(aliasesRates);
+  for (const auto& i : expAliasesHV) {
+    vect.emplace_back(i, o2::dcs::DPVAL_DOUBLE);
+  }
+  for (const auto& i : expAliasesADC) {
+    vect.emplace_back(i, o2::dcs::DPVAL_UINT);
+  }
+  for (const auto& i : expAliasesRates) {
+    vect.emplace_back(i, o2::dcs::DPVAL_DOUBLE);
+  }
+  return vect;
+}
diff --git a/Detectors/FIT/FV0/dcsmonitoring/workflow/FV0DCSConfigProcessorSpec.h b/Detectors/FIT/FV0/dcsmonitoring/workflow/FV0DCSConfigProcessorSpec.h
new file mode 100644
index 0000000000000..7c5a888312713
--- /dev/null
+++ b/Detectors/FIT/FV0/dcsmonitoring/workflow/FV0DCSConfigProcessorSpec.h
@@ -0,0 +1,56 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file FV0DCSConfigProcessorSpec.cxx
+/// \brief FV0 processor spec for DCS configurations
+///
+/// \author Andreas Molander , University of Jyvaskyla, Finland
+
+#ifndef O2_FV0_DCSCONFIGPROCESSOR_H
+#define O2_FV0_DCSCONFIGPROCESSOR_H
+
+#include "FITDCSMonitoring/FITDCSConfigProcessorSpec.h"
+#include "DetectorsCalibration/Utils.h"
+#include "Framework/WorkflowSpec.h"
+#include "Headers/DataHeader.h"
+
+#include 
+#include 
+
+namespace o2
+{
+
+namespace framework
+{
+
+DataProcessorSpec getFV0DCSConfigProcessorSpec()
+{
+  o2::header::DataDescription ddDChM = "FV0_DCHM";
+  std::vector outputs;
+  outputs.emplace_back(ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBPayload, ddDChM}, Lifetime::Sporadic);
+  outputs.emplace_back(ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBWrapper, ddDChM}, Lifetime::Sporadic);
+
+  return DataProcessorSpec{
+    "fv0-dcs-config-processor",
+    Inputs{{"inputConfig", o2::header::gDataOriginFV0, "DCS_CONFIG_FILE", Lifetime::Sporadic},
+           {"inputConfigFileName", o2::header::gDataOriginFV0, "DCS_CONFIG_NAME", Lifetime::Sporadic}},
+    outputs,
+    AlgorithmSpec{adaptFromTask("FV0", ddDChM)},
+    Options{{"use-verbose-mode", VariantType::Bool, false, {"Use verbose mode"}},
+            {"filename-dchm", VariantType::String, "FV0-deadchannels.txt", {"Dead channel map file name"}},
+            {"valid-days-dchm", VariantType::UInt32, 180u, {"Dead channel map validity in days"}},
+            {"no-validate", VariantType::Bool, false, {"Don't validate the CCDB uploads"}}}};
+}
+
+} // namespace framework
+} // namespace o2
+
+#endif // O2_FV0_DCSCONFIGPROCESSOR_H
diff --git a/Detectors/FIT/FV0/dcsmonitoring/workflow/FV0DCSDataProcessorSpec.h b/Detectors/FIT/FV0/dcsmonitoring/workflow/FV0DCSDataProcessorSpec.h
new file mode 100644
index 0000000000000..2a94825c3b3ee
--- /dev/null
+++ b/Detectors/FIT/FV0/dcsmonitoring/workflow/FV0DCSDataProcessorSpec.h
@@ -0,0 +1,51 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// @file FV0DCSDataProcessorSpec.h
+/// @brief DataProcessorSpec for FV0 DCS data
+///
+/// \author Andreas Molander , University of Jyvaskyla, Finland
+
+#ifndef O2_FV0_DATAPROCESSORSPEC_H
+#define O2_FV0_DATAPROCESSORSPEC_H
+
+#include "DetectorsCalibration/Utils.h"
+#include "FV0DCSMonitoring/FV0DCSDataProcessor.h"
+#include "Framework/Task.h"
+#include "Framework/WorkflowSpec.h"
+
+namespace o2
+{
+namespace framework
+{
+
+DataProcessorSpec getFV0DCSDataProcessorSpec()
+{
+  o2::header::DataDescription ddDCSDPs = "FV0_DCSDPs";
+  std::vector outputs;
+  outputs.emplace_back(ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBPayload, ddDCSDPs}, Lifetime::Sporadic);
+  outputs.emplace_back(ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBWrapper, ddDCSDPs}, Lifetime::Sporadic);
+
+  return DataProcessorSpec{
+    "fv0-dcs-data-processor",
+    Inputs{{"input", "DCS", "FV0DATAPOINTS"}},
+    outputs,
+    AlgorithmSpec{adaptFromTask("FV0", ddDCSDPs)},
+    Options{{"ccdb-path", VariantType::String, "http://localhost:8080", {"Path to CCDB"}},
+            {"use-ccdb-to-configure", VariantType::Bool, false, {"Use CCDB to configure"}},
+            {"use-verbose-mode", VariantType::Bool, false, {"Use verbose mode"}},
+            {"DPs-update-interval", VariantType::Int64, 600ll, {"Interval (in s) after which to update the DPs CCDB entry"}}}};
+}
+
+} // namespace framework
+} // namespace o2
+
+#endif // O2_FV0_DATAPROCESSORSPEC_H
diff --git a/Detectors/FIT/FV0/dcsmonitoring/workflow/fv0-dcs-config-workflow.cxx b/Detectors/FIT/FV0/dcsmonitoring/workflow/fv0-dcs-config-workflow.cxx
new file mode 100644
index 0000000000000..e66803515087d
--- /dev/null
+++ b/Detectors/FIT/FV0/dcsmonitoring/workflow/fv0-dcs-config-workflow.cxx
@@ -0,0 +1,35 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file fv0-dcs-config-workflow.cxx
+/// \brief Workflow for FV0 DCS configuration processing
+///
+/// \author Andreas Molander , University of Jyvaskyla, Finland
+
+#include "Framework/DataProcessorSpec.h"
+#include "FV0DCSConfigProcessorSpec.h"
+
+using namespace o2::framework;
+
+// we need to add workflow options before including Framework/runDataProcessing.h
+void customize(std::vector& workflowOptions)
+{
+  // option allowing to set parameters
+}
+
+#include "Framework/runDataProcessing.h"
+
+WorkflowSpec defineDataProcessing(o2::framework::ConfigContext const& configcontext)
+{
+  WorkflowSpec specs;
+  specs.emplace_back(getFV0DCSConfigProcessorSpec());
+  return specs;
+}
diff --git a/Detectors/FIT/FV0/dcsmonitoring/workflow/fv0-dcs-data-workflow.cxx b/Detectors/FIT/FV0/dcsmonitoring/workflow/fv0-dcs-data-workflow.cxx
new file mode 100644
index 0000000000000..3d6b838915f28
--- /dev/null
+++ b/Detectors/FIT/FV0/dcsmonitoring/workflow/fv0-dcs-data-workflow.cxx
@@ -0,0 +1,42 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file fv0-dcs-data-workflow.cxx
+/// \brief Workflow for FV0 DCS data processing
+///
+/// \author Andreas Molander , University of Jyvaskyla, Finland
+
+#include "DetectorsDCS/DataPointIdentifier.h"
+#include "DetectorsDCS/DataPointValue.h"
+#include "FV0DCSDataProcessorSpec.h"
+#include "Framework/TypeTraits.h"
+
+#include 
+
+namespace o2::framework
+{
+template <>
+struct has_root_dictionary, void> : std::true_type {
+};
+} // namespace o2::framework
+
+#include "Framework/DataProcessorSpec.h"
+
+using namespace o2::framework;
+
+#include "Framework/runDataProcessing.h"
+
+WorkflowSpec defineDataProcessing(ConfigContext const& configcontext)
+{
+  WorkflowSpec specs;
+  specs.emplace_back(getFV0DCSDataProcessorSpec());
+  return specs;
+}
diff --git a/Detectors/FIT/FV0/dcsmonitoring/workflow/fv0-dcs-sim-workflow.cxx b/Detectors/FIT/FV0/dcsmonitoring/workflow/fv0-dcs-sim-workflow.cxx
new file mode 100644
index 0000000000000..d6ec973bcfc35
--- /dev/null
+++ b/Detectors/FIT/FV0/dcsmonitoring/workflow/fv0-dcs-sim-workflow.cxx
@@ -0,0 +1,40 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file fv0-dcs-sim-workflow.cxx
+/// \brief Simulate DCS data for FV0
+///
+/// \author Andreas Molander , University of Jyvaskyla, Finland
+
+#include "DCStestWorkflow/DCSRandomDataGeneratorSpec.h"
+#include "Framework/runDataProcessing.h"
+
+o2::framework::WorkflowSpec defineDataProcessing(o2::framework::ConfigContext const& configcontext)
+{
+  std::vector dphints;
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FV0/HV/S[A,B,C,D,E,F,G,H][1..4]/actual/iMon", 250, 350});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FV0/HV/S[A,B,C,D,E,F,G,H][51,52]/actual/iMon", 250, 350});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FV0/HV/SREF/actual/iMon", 250, 350});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FV0/PM/S[A,B,C,D,E,F,G,H][1..4]/actual/ADC[0,1]_BASELINE", 30, 150});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FV0/PM/S[A,B,C,D,E,F,G,H][51,52]/actual/ADC[0,1]_BASELINE", 30, 150});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FV0/PM/SREF/actual/ADC[0,1]_BASELINE", 30, 150});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FV0/Trigger1_Charge/CNT_RATE", 0, 5000000});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FV0/Trigger2_Nchannels/CNT_RATE", 0, 5000000});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FV0/Trigger3_InnerRings/CNT_RATE", 0, 5000000});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FV0/Trigger4_OuterRings/CNT_RATE", 0, 5000000});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FV0/Trigger5_OrA/CNT_RATE", 0, 5000000});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FV0/Background/[0..9]/CNT_RATE", 0, 50000});
+  dphints.emplace_back(o2::dcs::test::DataPointHint{"FV0/Background/[A,B,C,D,E,F,G,H]/CNT_RATE", 0, 50000});
+
+  o2::framework::WorkflowSpec specs;
+  specs.emplace_back(o2::dcs::test::getDCSRandomDataGeneratorSpec(dphints, "FV0"));
+  return specs;
+}
diff --git a/Detectors/FIT/FV0/macros/FV0Misaligner.C b/Detectors/FIT/FV0/macros/FV0Misaligner.C
index 500bdaf565965..61be50b48dede 100644
--- a/Detectors/FIT/FV0/macros/FV0Misaligner.C
+++ b/Detectors/FIT/FV0/macros/FV0Misaligner.C
@@ -1,13 +1,32 @@
+// Copyright 2021-2025 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file  FV0Misaligner.C
+/// \brief ROOT macro for creating an FV0 geometry alignment object. The alignment object will align both
+///        detector halves in the same way. Based on ITSMisaligner.C
+///
+/// \author Andreas Molander andreas.molander@cern.ch, Alla Maevskaya
+
 #if !defined(__CLING__) || defined(__ROOTCLING__)
+
+#include "CCDB/CcdbApi.h"
 #include "DetectorsCommonDataFormats/DetID.h"
 #include "DetectorsCommonDataFormats/DetectorNameConf.h"
 #include "DetectorsCommonDataFormats/AlignParam.h"
-#include "DetectorsBase/GeometryManager.h"
-#include "CCDB/CcdbApi.h"
-#include 
+#include "FV0Base/Geometry.h"
+
 #include 
 #include 
 #include 
+
 #endif
 
 using AlgPar = std::array;
@@ -20,16 +39,14 @@ void FV0Misaligner(const std::string& ccdbHost = "http://ccdb-test.cern.ch:8080"
                    const std::string& fileName = "FV0Alignment.root")
 {
   std::vector params;
-  o2::base::GeometryManager::loadGeometry("", false);
   AlgPar pars;
   bool glo = true;
 
   o2::detectors::DetID detFV0("FV0");
 
-  // FV0 detector
-  for (int ihalf = 1; ihalf < 3; ihalf++) {
-    std::string symName = Form("FV0half_%i", ihalf);
-    pars = generateMisalignment(x, y, z, psi, theta, phi);
+  pars = generateMisalignment(x, y, z, psi, theta, phi);
+
+  for (auto& symName : {o2::fv0::Geometry::getDetectorRightSymName(), o2::fv0::Geometry::getDetectorLeftSymName()}) {
     params.emplace_back(symName.c_str(), -1, pars[0], pars[1], pars[2], pars[3], pars[4], pars[5], glo);
   }
 
@@ -37,7 +54,7 @@ void FV0Misaligner(const std::string& ccdbHost = "http://ccdb-test.cern.ch:8080"
     std::string path = objectPath.empty() ? o2::base::DetectorNameConf::getAlignmentPath(detFV0) : objectPath;
     LOGP(info, "Storing alignment object on {}/{}", ccdbHost, path);
     o2::ccdb::CcdbApi api;
-    map metadata; // can be empty
+    std::map metadata; // can be empty
     api.init(ccdbHost.c_str());   // or http://localhost:8080 for a local installation
     // store abitrary user object in strongly typed manner
     api.storeAsTFileAny(¶ms, path, metadata, tmin, tmax);
@@ -50,14 +67,15 @@ void FV0Misaligner(const std::string& ccdbHost = "http://ccdb-test.cern.ch:8080"
     algFile.Close();
   }
 }
+
 AlgPar generateMisalignment(double x, double y, double z, double psi, double theta, double phi)
 {
   AlgPar pars;
-  pars[0] = gRandom->Gaus(0, x);
-  pars[1] = gRandom->Gaus(0, y);
-  pars[2] = gRandom->Gaus(0, z);
-  pars[3] = gRandom->Gaus(0, psi);
-  pars[4] = gRandom->Gaus(0, theta);
-  pars[5] = gRandom->Gaus(0, phi);
+  pars[0] = x;
+  pars[1] = y;
+  pars[2] = z;
+  pars[3] = psi;
+  pars[4] = theta;
+  pars[5] = phi;
   return std::move(pars);
 }
diff --git a/Detectors/FIT/FV0/raw/include/FV0Raw/DataBlockFV0.h b/Detectors/FIT/FV0/raw/include/FV0Raw/DataBlockFV0.h
index 8bbcadf45b301..658535599b457 100644
--- a/Detectors/FIT/FV0/raw/include/FV0Raw/DataBlockFV0.h
+++ b/Detectors/FIT/FV0/raw/include/FV0Raw/DataBlockFV0.h
@@ -9,7 +9,7 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 //
-//file DataBlockFV0.h class  for RAW data format data blocks at FV0
+// file DataBlockFV0.h class  for RAW data format data blocks at FV0
 //
 // Artur.Furs
 // afurs@cern.ch
@@ -23,17 +23,17 @@ namespace o2
 {
 namespace fv0
 {
-//Raw event data for FV0
+// Raw event data for FV0
 using RawHeaderPM = o2::fv0::EventHeader;
 using RawDataPM = o2::fv0::EventData;
 using RawHeaderTCM = o2::fv0::EventHeader;
 using RawDataTCM = o2::fv0::TCMdata;
 using RawHeaderTCMext = o2::fv0::EventHeader;
 using RawDataTCMext = o2::fv0::TCMdataExtended;
-//Data block for FV0 modules
-using DataBlockPM = o2::fit::DataBlockPM;
-using DataBlockTCM = o2::fit::DataBlockTCM;
-using DataBlockTCMext = o2::fit::DataBlockTCMext;
+// Data block for FV0 modules, no padding - DataBlockConfig
+using DataBlockPM = o2::fit::DataBlockPM, RawHeaderPM, RawDataPM>;
+using DataBlockTCM = o2::fit::DataBlockTCM, RawHeaderTCM, RawDataTCM>;
+using DataBlockTCMext = o2::fit::DataBlockTCMext, RawHeaderTCMext, RawDataTCM, RawDataTCMext>;
 } // namespace fv0
 } // namespace o2
 #endif
diff --git a/Detectors/FIT/FV0/raw/include/FV0Raw/DigitBlockFV0.h b/Detectors/FIT/FV0/raw/include/FV0Raw/DigitBlockFV0.h
index 081ebac1c9a40..bfd95d5ff5c29 100644
--- a/Detectors/FIT/FV0/raw/include/FV0Raw/DigitBlockFV0.h
+++ b/Detectors/FIT/FV0/raw/include/FV0Raw/DigitBlockFV0.h
@@ -9,7 +9,7 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 //
-//file DigitBlockFV0.h class  for proccessing RAW data into Digits
+// file DigitBlockFV0.h class  for proccessing RAW data into Digits
 //
 // Artur.Furs
 // afurs@cern.ch
@@ -25,10 +25,10 @@ namespace o2
 {
 namespace fv0
 {
-//Normal data taking mode
-using DigitBlockFV0 = DigitBlockFIT;
-//TCM extended data taking mode
-using DigitBlockFV0ext = DigitBlockFIText;
+// Normal data taking mode
+using DigitBlockFV0 = o2::fit::DigitBlockFIT;
+// TCM extended data taking mode
+using DigitBlockFV0ext = o2::fit::DigitBlockFIText;
 } // namespace fv0
 } // namespace o2
 #endif
diff --git a/Detectors/FIT/FV0/raw/include/FV0Raw/RawWriterFV0.h b/Detectors/FIT/FV0/raw/include/FV0Raw/RawWriterFV0.h
index 2c25075721693..386f30e8a4e9f 100644
--- a/Detectors/FIT/FV0/raw/include/FV0Raw/RawWriterFV0.h
+++ b/Detectors/FIT/FV0/raw/include/FV0Raw/RawWriterFV0.h
@@ -9,7 +9,7 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 //
-//file RawWriterFV0.h Raw writer class for FV0
+// file RawWriterFV0.h Raw writer class for FV0
 //
 // Artur.Furs
 // afurs@cern.ch
@@ -24,10 +24,12 @@ namespace o2
 {
 namespace fv0
 {
-//Normal TCM mode
+// Normal TCM mode
 using RawWriterFV0 = o2::fit::RawWriterFIT;
-//Extended TCM mode
-//using RawWriterFV0ext = o2::fit::RawWriterFIT;
+using RawWriterFV0_padded = o2::fit::RawWriterFIT;
+
+// Extended TCM mode
+// using RawWriterFV0ext = o2::fit::RawWriterFIT;
 } // namespace fv0
 } // namespace o2
 
diff --git a/Detectors/FIT/FV0/raw/src/DataBlockFV0.cxx b/Detectors/FIT/FV0/raw/src/DataBlockFV0.cxx
index 4f53af0c80f79..37027f629550e 100644
--- a/Detectors/FIT/FV0/raw/src/DataBlockFV0.cxx
+++ b/Detectors/FIT/FV0/raw/src/DataBlockFV0.cxx
@@ -10,3 +10,7 @@
 // or submit itself to any jurisdiction.
 
 #include "FV0Raw/DataBlockFV0.h"
+
+template class o2::fit::DataBlockPM, o2::fv0::RawHeaderPM, o2::fv0::RawDataPM>;
+template class o2::fit::DataBlockTCM, o2::fv0::RawHeaderTCM, o2::fv0::RawDataTCM>;
+template class o2::fit::DataBlockTCMext, o2::fv0::RawHeaderTCMext, o2::fv0::RawDataTCM, o2::fv0::RawDataTCMext>;
diff --git a/Detectors/FIT/FV0/raw/src/DigitBlockFV0.cxx b/Detectors/FIT/FV0/raw/src/DigitBlockFV0.cxx
index 8da75f3bdbcff..367c57686ab95 100644
--- a/Detectors/FIT/FV0/raw/src/DigitBlockFV0.cxx
+++ b/Detectors/FIT/FV0/raw/src/DigitBlockFV0.cxx
@@ -10,3 +10,6 @@
 // or submit itself to any jurisdiction.
 
 #include "FV0Raw/DigitBlockFV0.h"
+
+template class o2::fit::DigitBlockFIT;
+template class o2::fit::DigitBlockFIText;
\ No newline at end of file
diff --git a/Detectors/FIT/FV0/raw/src/RawReaderFV0Base.cxx b/Detectors/FIT/FV0/raw/src/RawReaderFV0Base.cxx
index 7bba6ea3937c9..bbf2a780de0a1 100644
--- a/Detectors/FIT/FV0/raw/src/RawReaderFV0Base.cxx
+++ b/Detectors/FIT/FV0/raw/src/RawReaderFV0Base.cxx
@@ -10,3 +10,6 @@
 // or submit itself to any jurisdiction.
 
 #include "FV0Raw/RawReaderFV0Base.h"
+
+template class o2::fit::RawReaderBaseFIT;
+template class o2::fit::RawReaderBaseFIT;
diff --git a/Detectors/FIT/FV0/raw/src/RawWriterFV0.cxx b/Detectors/FIT/FV0/raw/src/RawWriterFV0.cxx
index 64d75091cbe80..d52e1117460a8 100644
--- a/Detectors/FIT/FV0/raw/src/RawWriterFV0.cxx
+++ b/Detectors/FIT/FV0/raw/src/RawWriterFV0.cxx
@@ -10,3 +10,6 @@
 // or submit itself to any jurisdiction.
 
 #include "FV0Raw/RawWriterFV0.h"
+
+template class o2::fit::RawWriterFIT;
+template class o2::fit::RawWriterFIT;
diff --git a/Detectors/FIT/FV0/reconstruction/README.md b/Detectors/FIT/FV0/reconstruction/README.md
new file mode 100644
index 0000000000000..46de9800fda92
--- /dev/null
+++ b/Detectors/FIT/FV0/reconstruction/README.md
@@ -0,0 +1,36 @@
+
+
+# FV0 reconstruction
+
+Please note that this readme shows how the reconstruction is done _at the moment_. The current reconstruction is not necessarily done the way it _should be_. The algorithm is under review and subject to change.
+
+The FV0 reconstruction workflow is
+
+- [O2/Detectors/FIT/FV0/workflow/src/fv0-reco-workflow.cxx](../workflow/src/fv0-reco-workflow.cxx)
+
+and the actual reconstruction happens in
+
+- [src/BaseRecoTask.cxx](src/BaseRecoTask.cxx) ([include/FV0Reconstruction/BaseRecoTask.h](include/BaseRecoTask.h))
+
+## Channel data reconstruction
+
+Currently, there's no channel data filtering applied in the FV0 reconstruction. All digit channel data are propagated to RecPoints. The following conversions are made:
+
+- Channel ID (`uint8_t`) -> Channel ID (`int`)
+- CFD time (TDC units, `int16_t`) -> Time in ns (`double`)
+    - Furthermore, if available and enabled, a time offset correction is applied. For FV0 it is not available nor enabled at the moment.
+- Amplitude (ADC channels, `int16_t`) -> Ampltidue (ADC channels, `double`)
+- Channel bits (`uint8_t`) -> Channel bits (`int`)
+
+## RecPoint reconstruction
+
+The reconstructed RecPoints contain:
+
+- BC information (`o2::InteractionRecord`)
+- Channel data
+- Three different times:
+    - "TimeFirst": time of first signal (w/ charge > `FV0DigParam::chargeThrForMeanTime`)
+    - "TimeGlobalMean": average of all signals passing some filtering (including CFD in gate)
+    - "TimeSelectedMean": average of all signals passing a bit stricter filtering (includling CFD in gate)
diff --git a/Detectors/FIT/FV0/reconstruction/include/FV0Reconstruction/BaseRecoTask.h b/Detectors/FIT/FV0/reconstruction/include/FV0Reconstruction/BaseRecoTask.h
index 20740a46a77ee..c5cb5b0da6d05 100644
--- a/Detectors/FIT/FV0/reconstruction/include/FV0Reconstruction/BaseRecoTask.h
+++ b/Detectors/FIT/FV0/reconstruction/include/FV0Reconstruction/BaseRecoTask.h
@@ -18,6 +18,7 @@
 #include "DataFormatsFV0/ChannelData.h"
 #include "DataFormatsFV0/RecPoints.h"
 #include "DataFormatsFV0/FV0ChannelTimeCalibrationObject.h"
+#include "DataFormatsFIT/DeadChannelMap.h"
 #include 
 
 namespace o2
@@ -33,14 +34,15 @@ class BaseRecoTask
   ~BaseRecoTask() = default;
   o2::fv0::RecPoints process(o2::fv0::Digit const& bcd,
                              gsl::span inChData,
-                             gsl::span outChData);
+                             std::vector& outChData);
   void FinishTask();
-  void setChannelOffset(o2::fv0::FV0ChannelTimeCalibrationObject* caliboffsets) { mCalibOffset = caliboffsets; };
-  int getChannelOffset(int channel);
+  void SetChannelOffset(o2::fv0::FV0ChannelTimeCalibrationObject const* caliboffsets) { mCalibOffset = caliboffsets; };
+  void SetDeadChannelMap(o2::fit::DeadChannelMap const* deadChannelMap) { mDeadChannelMap = deadChannelMap; }
+  int getOffset(int channel);
 
  private:
-  o2::fv0::FV0ChannelTimeCalibrationObject* mCalibOffset = nullptr;
-
+  o2::fv0::FV0ChannelTimeCalibrationObject const* mCalibOffset = nullptr;
+  o2::fit::DeadChannelMap const* mDeadChannelMap = nullptr;
   ClassDefNV(BaseRecoTask, 3);
 };
 } // namespace fv0
diff --git a/Detectors/FIT/FV0/reconstruction/include/FV0Reconstruction/CTFCoder.h b/Detectors/FIT/FV0/reconstruction/include/FV0Reconstruction/CTFCoder.h
index f4699b3f9735b..80dcd6060455b 100644
--- a/Detectors/FIT/FV0/reconstruction/include/FV0Reconstruction/CTFCoder.h
+++ b/Detectors/FIT/FV0/reconstruction/include/FV0Reconstruction/CTFCoder.h
@@ -30,10 +30,10 @@ namespace o2
 namespace fv0
 {
 
-class CTFCoder : public o2::ctf::CTFCoderBase
+class CTFCoder final : public o2::ctf::CTFCoderBase
 {
  public:
-  CTFCoder(o2::ctf::CTFCoderBase::OpType op) : o2::ctf::CTFCoderBase(op, CTF::getNBlocks(), o2::detectors::DetID::FV0) {}
+  CTFCoder(o2::ctf::CTFCoderBase::OpType op, const std::string& ctfdictOpt = "none") : o2::ctf::CTFCoderBase(op, CTF::getNBlocks(), o2::detectors::DetID::FV0, 1.f, ctfdictOpt) {}
   ~CTFCoder() final = default;
 
   /// entropy-encode digits to buffer with CTF
@@ -68,15 +68,15 @@ o2::ctf::CTFIOSize CTFCoder::encode(VEC& buff, const gsl::span& dig
   using MD = o2::ctf::Metadata::OptStore;
   // what to do which each field: see o2::ctd::Metadata explanation
   constexpr MD optField[CTF::getNBlocks()] = {
-    MD::EENCODE, // BLC_bcInc
-    MD::EENCODE, // BLC_orbitInc
-    MD::EENCODE, // BLC_nChan
-    MD::EENCODE, // BLC_idChan
-    MD::EENCODE, // BLC_cfdTime
-    MD::EENCODE, // BLC_qtcAmpl
+    MD::EENCODE_OR_PACK, // BLC_bcInc
+    MD::EENCODE_OR_PACK, // BLC_orbitInc
+    MD::EENCODE_OR_PACK, // BLC_nChan
+    MD::EENCODE_OR_PACK, // BLC_idChan
+    MD::EENCODE_OR_PACK, // BLC_cfdTime
+    MD::EENCODE_OR_PACK, // BLC_qtcAmpl
     // extra slot was added in the end
-    MD::EENCODE, // BLC_trigger
-    MD::EENCODE  // BLC_qtcChain
+    MD::EENCODE_OR_PACK, // BLC_trigger
+    MD::EENCODE_OR_PACK  // BLC_qtcChain
   };
   CompressedDigits cd;
   if (mExtHeader.isValidDictTimeStamp()) {
@@ -97,11 +97,10 @@ o2::ctf::CTFIOSize CTFCoder::encode(VEC& buff, const gsl::span& dig
 
   ec->setHeader(cd.header);
   assignDictVersion(static_cast(ec->getHeader()));
-  ec->getANSHeader().majorVersion = 0;
-  ec->getANSHeader().minorVersion = 1;
+  ec->setANSHeader(mANSVersion);
   // at every encoding the buffer might be autoexpanded, so we don't work with fixed pointer ec
   o2::ctf::CTFIOSize iosize;
-#define ENCODEFV0(part, slot, bits) CTF::get(buff.data())->encode(part, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)].get(), getMemMarginFactor());
+#define ENCODEFV0(part, slot, bits) CTF::get(buff.data())->encode(part, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)], getMemMarginFactor());
   // clang-format off
   iosize += ENCODEFV0(cd.bcInc,     CTF::BLC_bcInc,    0);
   iosize += ENCODEFV0(cd.orbitInc,  CTF::BLC_orbitInc, 0);
@@ -129,7 +128,7 @@ o2::ctf::CTFIOSize CTFCoder::decode(const CTF::base& ec, VDIG& digitVec, VCHAN&
   checkDictVersion(hd);
   ec.print(getPrefix(), mVerbosity);
   o2::ctf::CTFIOSize iosize;
-#define DECODEFV0(part, slot) ec.decode(part, int(slot), mCoders[int(slot)].get())
+#define DECODEFV0(part, slot) ec.decode(part, int(slot), mCoders[int(slot)])
   // clang-format off
   iosize += DECODEFV0(cd.bcInc,     CTF::BLC_bcInc);
   iosize += DECODEFV0(cd.orbitInc,  CTF::BLC_orbitInc);
@@ -219,12 +218,30 @@ void CTFCoder::compress(CompressedDigits& cd, const gsl::span& digi
 {
   // convert digits/channel to their compressed version
   cd.clear();
+  cd.header.det = mDet;
   if (!digitVec.size()) {
     return;
   }
-  const auto& dig0 = digitVec[0];
-  cd.header.det = mDet;
-  cd.header.nTriggers = digitVec.size();
+  uint32_t firstDig = digitVec.size(), nDigSel = digitVec.size(), nChanSel = channelVec.size();
+  std::vector reject(digitVec.size());
+  if (mIRFrameSelector.isSet()) {
+    for (size_t id = 0; id < digitVec.size(); id++) {
+      if (mIRFrameSelector.check(digitVec[id].mIntRecord) < 0) {
+        reject[id] = true;
+        nDigSel--;
+        nChanSel -= digitVec[id].ref.getEntries();
+      } else if (firstDig == digitVec.size()) {
+        firstDig = id;
+      }
+    }
+  } else {
+    firstDig = 0;
+  }
+  if (nDigSel == 0) { // nothing is selected
+    return;
+  }
+  const auto& dig0 = digitVec[firstDig];
+  cd.header.nTriggers = nDigSel;
   cd.header.firstOrbit = dig0.getOrbit();
   cd.header.firstBC = dig0.getBC();
   cd.header.triggerGate = FV0DigParam::Instance().mTime_trg_gate;
@@ -234,37 +251,41 @@ void CTFCoder::compress(CompressedDigits& cd, const gsl::span& digi
   cd.orbitInc.resize(cd.header.nTriggers);
   cd.nChan.resize(cd.header.nTriggers);
 
-  cd.idChan.resize(channelVec.size());
-  cd.qtcChain.resize(channelVec.size());
-  cd.cfdTime.resize(channelVec.size());
-  cd.qtcAmpl.resize(channelVec.size());
+  cd.idChan.resize(nChanSel);
+  cd.qtcChain.resize(nChanSel);
+  cd.cfdTime.resize(nChanSel);
+  cd.qtcAmpl.resize(nChanSel);
 
   uint16_t prevBC = cd.header.firstBC;
   uint32_t prevOrbit = cd.header.firstOrbit;
-  uint32_t ccount = 0;
-  for (uint32_t idig = 0; idig < cd.header.nTriggers; idig++) {
+  uint32_t ccount = 0, dcount = 0;
+  for (uint32_t idig = 0; idig < digitVec.size(); idig++) {
+    if (reject[idig]) {
+      continue;
+    }
     const auto& digit = digitVec[idig];
     const auto chanels = digit.getBunchChannelData(channelVec); // we assume the channels are sorted
 
     // fill trigger info
-    cd.trigger[idig] = digit.getTriggers().getTriggersignals();
+    cd.trigger[dcount] = digit.getTriggers().getTriggersignals();
     if (prevOrbit == digit.getOrbit()) {
-      cd.bcInc[idig] = digit.getBC() - prevBC;
-      cd.orbitInc[idig] = 0;
+      cd.bcInc[dcount] = digit.getBC() - prevBC;
+      cd.orbitInc[dcount] = 0;
     } else {
-      cd.bcInc[idig] = digit.getBC();
-      cd.orbitInc[idig] = digit.getOrbit() - prevOrbit;
+      cd.bcInc[dcount] = digit.getBC();
+      cd.orbitInc[dcount] = digit.getOrbit() - prevOrbit;
     }
     prevBC = digit.getBC();
     prevOrbit = digit.getOrbit();
     // fill channels info
-    cd.nChan[idig] = chanels.size();
-    if (!cd.nChan[idig]) {
+    cd.nChan[dcount] = chanels.size();
+    if (!cd.nChan[dcount]) {
       LOG(debug) << "Digits with no channels";
+      dcount++;
       continue;
     }
     uint8_t prevChan = 0;
-    for (uint8_t ic = 0; ic < cd.nChan[idig]; ic++) {
+    for (uint8_t ic = 0; ic < cd.nChan[dcount]; ic++) {
       if constexpr (MINOR_VERSION == 0 && MAJOR_VERSION == 1) {
         cd.idChan[ccount] = chanels[ic].ChId - prevChan; // Old method, lets keep it for a while
       } else {
@@ -276,6 +297,7 @@ void CTFCoder::compress(CompressedDigits& cd, const gsl::span& digi
       prevChan = chanels[ic].ChId;
       ccount++;
     }
+    dcount++;
   }
 }
 
diff --git a/Detectors/FIT/FV0/reconstruction/src/BaseRecoTask.cxx b/Detectors/FIT/FV0/reconstruction/src/BaseRecoTask.cxx
index 90198ccc8119e..8032220f8996d 100644
--- a/Detectors/FIT/FV0/reconstruction/src/BaseRecoTask.cxx
+++ b/Detectors/FIT/FV0/reconstruction/src/BaseRecoTask.cxx
@@ -27,14 +27,14 @@ using RP = o2::fv0::RecPoints;
 
 RP BaseRecoTask::process(o2::fv0::Digit const& bcd,
                          gsl::span inChData,
-                         gsl::span outChData)
+                         std::vector& outChData)
 {
   LOG(debug) << "Running reconstruction on new event";
 
-  Float_t sideAtimeFirst = 1e10;
   Int_t ndigitsA = 0;
-  Float_t sideAtimeAvg = 0;
   Int_t ndigitsASelected = 0;
+  Float_t sideAtimeFirst = 1e10;
+  Float_t sideAtimeAvg = 0;
   Float_t sideAtimeAvgSelected = 0;
 
   auto timeStamp = o2::InteractionRecord::bc2ns(bcd.getIntRecord().bc, bcd.getIntRecord().orbit);
@@ -44,28 +44,29 @@ RP BaseRecoTask::process(o2::fv0::Digit const& bcd,
   int nch = inChData.size();
   for (int ich = 0; ich < nch; ich++) {
     LOG(debug) << "  channel " << ich << " / " << nch;
-    int offsetChannel = getChannelOffset(inChData[ich].ChId);
-
-    outChData[ich] = o2::fv0::ChannelDataFloat{inChData[ich].ChId,
-                                               (inChData[ich].CFDTime - offsetChannel) * DigitizationConstant::TIME_PER_TDCCHANNEL,
-                                               (double)inChData[ich].QTCAmpl,
-                                               inChData[ich].ChainQTC};
-    //  only signals with amplitude participate in collision time
-    if (outChData[ich].charge > 0) {
-      sideAtimeFirst = std::min(static_cast(sideAtimeFirst), outChData[ich].time);
-      sideAtimeAvg += outChData[ich].time;
-      ndigitsA++;
+    if (mDeadChannelMap && !mDeadChannelMap->isChannelAlive(inChData[ich].ChId)) {
+      LOG(debug) << "Channel " << ich << " is dead - discarding data";
+      continue;
     }
-    if (outChData[ich].charge > o2::fv0::FV0DigParam::Instance().chargeThrForMeanTime) {
-      if (!inChData[ich].getFlag(ChannelData::kIsDoubleEvent) &&
-          !inChData[ich].getFlag(ChannelData::kIsTimeInfoNOTvalid) &&
-          inChData[ich].getFlag(ChannelData::kIsCFDinADCgate) &&
-          !inChData[ich].getFlag(ChannelData::kIsTimeInfoLate) &&
-          !inChData[ich].getFlag(ChannelData::kIsAmpHigh) &&
-          inChData[ich].getFlag(ChannelData::kIsEventInTVDC) &&
-          !inChData[ich].getFlag(ChannelData::kIsTimeInfoLost)) {
-        sideAtimeAvgSelected += outChData[ich].time;
-        ndigitsASelected++;
+    int offsetChannel = getOffset(int(inChData[ich].ChId));
+    outChData.emplace_back(o2::fv0::ChannelDataFloat{inChData[ich].ChId,
+                                                     (inChData[ich].CFDTime - offsetChannel) * DigitizationConstant::TIME_PER_TDCCHANNEL,
+                                                     (float)inChData[ich].QTCAmpl,
+                                                     inChData[ich].ChainQTC});
+    const auto& currentOutCh = outChData.back();
+
+    // Conditions for reconstructing collision time (3 variants: first, average-relaxed and average-tight)
+    if (currentOutCh.charge > FV0DigParam::Instance().chargeThrForMeanTime) {
+      sideAtimeFirst = std::min(static_cast(sideAtimeFirst), currentOutCh.time);
+      if (inChData[ich].areAllFlagsGood()) {
+        if (std::abs(currentOutCh.time) < FV0DigParam::Instance().mTimeThresholdForReco) {
+          sideAtimeAvg += currentOutCh.time;
+          ndigitsA++;
+        }
+        if (currentOutCh.charge > FV0DigParam::Instance().mAmpThresholdForReco && std::abs(currentOutCh.time) < FV0DigParam::Instance().mTimeThresholdForReco) {
+          sideAtimeAvgSelected += currentOutCh.time;
+          ndigitsASelected++;
+        }
       }
     }
   }
@@ -81,13 +82,14 @@ RP BaseRecoTask::process(o2::fv0::Digit const& bcd,
 void BaseRecoTask::FinishTask()
 {
   // finalize digitization, if needed, flash remaining digits
-  //if (!mContinuous)   return;
+  // if (!mContinuous)   return;
 }
 //______________________________________________________
-int BaseRecoTask::getChannelOffset(int channel)
+int BaseRecoTask::getOffset(int channel)
 {
   if (!mCalibOffset) {
     return 0;
   }
-  return mCalibOffset->mTimeOffsets[channel];
+  int offsetChannel = mCalibOffset->mTimeOffsets[channel];
+  return offsetChannel;
 }
diff --git a/Detectors/FIT/FV0/reconstruction/src/CTFCoder.cxx b/Detectors/FIT/FV0/reconstruction/src/CTFCoder.cxx
index e718dbd3c07f0..f2e4f59e10c97 100644
--- a/Detectors/FIT/FV0/reconstruction/src/CTFCoder.cxx
+++ b/Detectors/FIT/FV0/reconstruction/src/CTFCoder.cxx
@@ -53,7 +53,7 @@ void CTFCoder::createCoders(const std::vector& bufVec, o2::ctf::CTFCoderBa
 {
   const auto ctf = CTF::getImage(bufVec.data());
   CompressedDigits cd; // just to get member types
-#define MAKECODER(part, slot) createCoder(op, ctf.getFrequencyTable(slot), int(slot))
+#define MAKECODER(part, slot) createCoder(op, std::get>(ctf.getDictionary(slot, mANSVersion)), int(slot))
   // clang-format off
   MAKECODER(cd.bcInc,        CTF::BLC_bcInc);
   MAKECODER(cd.orbitInc,     CTF::BLC_orbitInc);
@@ -73,22 +73,17 @@ void CTFCoder::createCoders(const std::vector& bufVec, o2::ctf::CTFCoderBa
 size_t CTFCoder::estimateCompressedSize(const CompressedDigits& cd)
 {
   size_t sz = 0;
-  // clang-format off
   // RS FIXME this is very crude estimate, instead, an empirical values should be used
-#define VTP(vec) typename std::remove_reference::type::value_type
-#define ESTSIZE(vec, slot) mCoders[int(slot)] ?                         \
-  rans::calculateMaxBufferSize(vec.size(), reinterpret_cast*>(mCoders[int(slot)].get())->getAlphabetRangeBits(), sizeof(VTP(vec)) ) : vec.size()*sizeof(VTP(vec))
-  sz += ESTSIZE(cd.trigger,      CTF::BLC_trigger);
-  sz += ESTSIZE(cd.bcInc,        CTF::BLC_bcInc);
-  sz += ESTSIZE(cd.orbitInc,     CTF::BLC_orbitInc);
-  sz += ESTSIZE(cd.nChan,        CTF::BLC_nChan);
+  sz += estimateBufferSize(static_cast(CTF::BLC_trigger), cd.trigger);
+  sz += estimateBufferSize(static_cast(CTF::BLC_bcInc), cd.bcInc);
+  sz += estimateBufferSize(static_cast(CTF::BLC_orbitInc), cd.orbitInc);
+  sz += estimateBufferSize(static_cast(CTF::BLC_nChan), cd.nChan);
 
-  sz += ESTSIZE(cd.idChan,       CTF::BLC_idChan);
-  sz += ESTSIZE(cd.qtcChain,     CTF::BLC_qtcChain);
-  sz += ESTSIZE(cd.cfdTime,      CTF::BLC_cfdTime);
-  sz += ESTSIZE(cd.qtcAmpl,      CTF::BLC_qtcAmpl);
-  // clang-format on
+  sz += estimateBufferSize(static_cast(CTF::BLC_idChan), cd.idChan);
+  sz += estimateBufferSize(static_cast(CTF::BLC_qtcChain), cd.qtcChain);
+  sz += estimateBufferSize(static_cast(CTF::BLC_cfdTime), cd.cfdTime);
+  sz += estimateBufferSize(static_cast(CTF::BLC_qtcAmpl), cd.qtcAmpl);
 
-  LOG(info) << "Estimated output size is " << sz << " bytes";
+  LOG(debug) << "Estimated output size is " << sz << " bytes";
   return sz;
-}
+};
\ No newline at end of file
diff --git a/Detectors/FIT/FV0/simulation/CMakeLists.txt b/Detectors/FIT/FV0/simulation/CMakeLists.txt
index d14b72450ff8f..6c696c52523ae 100644
--- a/Detectors/FIT/FV0/simulation/CMakeLists.txt
+++ b/Detectors/FIT/FV0/simulation/CMakeLists.txt
@@ -29,12 +29,8 @@ o2_target_root_dictionary(FV0Simulation
 o2_add_executable(digi2raw
         COMPONENT_NAME fv0
         SOURCES src/digit2raw.cxx
-        PUBLIC_LINK_LIBRARIES O2::FV0Simulation
-                              O2::DetectorsRaw
-                              O2::DetectorsCommonDataFormats
-                              O2::CommonUtils
-                              Boost::program_options
-                              O2::FV0Raw)
+        PUBLIC_LINK_LIBRARIES O2::FV0Raw
+                              Boost::program_options)
 
 o2_data_file(COPY data DESTINATION Detectors/FV0/simulation)
 
diff --git a/Detectors/FIT/FV0/simulation/include/FV0Simulation/Digitizer.h b/Detectors/FIT/FV0/simulation/include/FV0Simulation/Digitizer.h
index 3d36f255d5094..b97893822c9d8 100644
--- a/Detectors/FIT/FV0/simulation/include/FV0Simulation/Digitizer.h
+++ b/Detectors/FIT/FV0/simulation/include/FV0Simulation/Digitizer.h
@@ -14,6 +14,7 @@
 
 #include "CommonDataFormat/InteractionRecord.h"
 #include "DataFormatsFV0/Digit.h"
+#include "DataFormatsFIT/DeadChannelMap.h"
 #include "DataFormatsFV0/ChannelData.h"
 #include "DataFormatsFV0/MCLabel.h"
 #include "FV0Simulation/Detector.h"
@@ -51,6 +52,7 @@ class Digitizer
   void setEventId(Int_t id) { mEventId = id; }
   void setSrcId(Int_t id) { mSrcId = id; }
   void setInteractionRecord(const InteractionTimeRecord& ir) { mIntRecord = ir; }
+  void setDeadChannelMap(o2::fit::DeadChannelMap const* deadChannelMap) { mDeadChannelMap = deadChannelMap; };
 
   void process(const std::vector& hits, std::vector& digitsBC,
                std::vector& digitsCh, std::vector& digitsTrig,
@@ -90,8 +92,16 @@ class Digitizer
 
  private:
   static constexpr int BCCacheMin = 0, BCCacheMax = 7, NBC2Cache = 1 + BCCacheMax - BCCacheMin;
-  void createPulse(float mipFraction, int parID, const double hitTime, std::array const& cachedIR,
-                   int nCachedIR, const int detID);
+  /// Create signal pulse based on MC hit
+  /// \param mipFraction Fraction of the MIP energy deposited in the cell
+  /// \param parID       Particle ID
+  /// \param hitTime     Time of the hit
+  /// \param hitR        Length to IP from the position of the hit
+  /// \param cachedIR    Cached interaction records
+  /// \param nCachedIR   Number of cached interaction records
+  /// \param detID       Detector cell ID
+  void createPulse(float mipFraction, int parID, const double hitTime, const float hitR,
+                   std::array const& cachedIR, int nCachedIR, const int detID);
 
   long mTimeStamp;                  // TF (run) timestamp
   InteractionTimeRecord mIntRecord; // Interaction record (orbit, bc) -> InteractionTimeRecord
@@ -124,6 +134,8 @@ class Digitizer
   BCCache mLastBCCache;                                    // buffer for the last BC
   std::array mCfdStartIndex; // start indices for the CFD detector
 
+  o2::fit::DeadChannelMap const* mDeadChannelMap = nullptr;
+
   /// Internal helper methods related to conversion of energy-deposition into el. signal
   Int_t SimulateLightYield(Int_t pmt, Int_t nPhot) const;
   Float_t SimulateTimeCfd(int& startIndex, const ChannelDigitF& pulseLast, const ChannelDigitF& pulse) const;
diff --git a/Detectors/FIT/FV0/simulation/include/FV0Simulation/FV0DigParam.h b/Detectors/FIT/FV0/simulation/include/FV0Simulation/FV0DigParam.h
index b86c863ab94af..383fa4cb494c1 100644
--- a/Detectors/FIT/FV0/simulation/include/FV0Simulation/FV0DigParam.h
+++ b/Detectors/FIT/FV0/simulation/include/FV0Simulation/FV0DigParam.h
@@ -9,7 +9,7 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 
-/// \file FV0DigParam.h
+/// \file  FV0DigParam.h
 /// \brief Configurable digitization parameters
 
 #ifndef ALICEO2_FV0_DIG_PARAM
@@ -22,9 +22,11 @@ namespace o2::fv0
 {
 // parameters of FV0 digitization / transport simulation
 struct FV0DigParam : o2::conf::ConfigurableParamHelper {
-  float photoCathodeEfficiency = 0.23;                  // quantum efficiency = nOfPhotoE_emitted_by_photocathode / nIncidentPhotons
-  float lightYield = 0.01;                              // light collection efficiency to be tuned using collision data [1%]
-  float adcChannelsPerMip = 16;                         // Default: 16 for pp and 8 for PbPb
+  float hitTimeOffset = 0.0; ///< Hit time offset [ns]
+
+  float photoCathodeEfficiency = 0.23;                                     // quantum efficiency = nOfPhotoE_emitted_by_photocathode / nIncidentPhotons
+  float lightYield = 0.01;                                                 // light collection efficiency to be tuned using collision data [1%]
+  float adcChannelsPerMip = 16;                                            // Default: 16 for pp and 8 for PbPb
   float getChannelsPerMilivolt() const { return adcChannelsPerMip / 7.5; } // Non-trivial conversion depending on the pulseshape: amplitude to charge
   float chargeThrForMeanTime = 5;                                          // Charge threshold, only above which the time is taken into account in calculating the mean time of all qualifying channels
 
@@ -55,17 +57,18 @@ struct FV0DigParam : o2::conf::ConfigurableParamHelper {
   bool isIntegrateFull = false;                                                  // Full charge integration widow in 25 ns
   float cfdCheckWindow = 2.5;                                                    // time window for the cfd in ns to trigger the charge integration
   int avgNumberPhElectronPerMip = 201;                                           // avg number of photo-electrons per MIP
-  float globalTimeOfFlight = 315.0 / o2::constants::physics::LightSpeedCm2NS;    // TODO [check the correct value for distance of FV0 to IP]
   float mCfdDeadTime = 15.6;                                                     // [ns]
   float mCFD_trsh = 3.;                                                          // [mV]
   float getCFDTrshInAdc() const { return mCFD_trsh * getChannelsPerMilivolt(); } // [ADC channels]
   /// Parameters for trigger simulation
-  bool useMaxChInAdc = true;         // default = true
-  int adcChargeCenThr = 3 * 498;     // threshold value of ADC charge for Central trigger
-  int adcChargeSCenThr = 1 * 498;    // threshold value of ADC charge for Semi-central trigger
-  int maxCountInAdc = 4095;          // to take care adc ADC overflow
-  short mTime_trg_gate = 153;        // #channels as in TCM as in Pilot beams ('OR gate' setting in TCM tab in ControlServer)
-  uint8_t defaultChainQtc = 0x48;    // only 2 flags are set by default in simulation: kIsCFDinADCgate and kIsEventInTVDC
+  bool useMaxChInAdc = true;          // default = true
+  int adcChargeCenThr = 3 * 498;      // threshold value of ADC charge for Central trigger
+  int adcChargeSCenThr = 1 * 498;     // threshold value of ADC charge for Semi-central trigger
+  int maxCountInAdc = 4095;           // to take care adc ADC overflow
+  short mTime_trg_gate = 153;         // #channels as in TCM as in Pilot beams ('OR gate' setting in TCM tab in ControlServer)
+  uint8_t defaultChainQtc = 0x48;     // only 2 flags are set by default in simulation: kIsCFDinADCgate and kIsEventInTVDC
+  float mAmpThresholdForReco = 24;    // only channels with amplitude higher will participate in calibration and collision time
+  short mTimeThresholdForReco = 1000; // only channels with time below will participate in calibration and collision time
 
   O2ParamDef(FV0DigParam, "FV0DigParam");
 };
diff --git a/Detectors/FIT/FV0/simulation/src/Detector.cxx b/Detectors/FIT/FV0/simulation/src/Detector.cxx
index 460f8fe018bae..07eb9053bf3b8 100644
--- a/Detectors/FIT/FV0/simulation/src/Detector.cxx
+++ b/Detectors/FIT/FV0/simulation/src/Detector.cxx
@@ -23,7 +23,7 @@
 #include 
 #include "FV0Simulation/Detector.h"
 #include "Framework/Logger.h"
-#include "SimulationDataFormat/Stack.h"
+#include "DetectorsBase/Stack.h"
 #include "FV0Base/Geometry.h"
 
 using namespace o2::fv0;
@@ -280,6 +280,7 @@ void Detector::ConstructGeometry()
   // mGeometry->enableComponent(Geometry::eAluminiumContainer, false);
   mGeometry->buildGeometry();
 }
+
 void Detector::addAlignableVolumes() const
 {
   //
@@ -292,19 +293,19 @@ void Detector::addAlignableVolumes() const
   LOG(info) << "FV0: Add alignable volumes";
 
   if (!gGeoManager) {
-    LOG(fatal) << "TGeoManager doesn't exist !";
+    LOG(fatal) << "TGeoManager doesn't exist!";
     return;
   }
 
-  TString volPath, symName;
-  for (auto& half : {"RIGHT_0", "LEFT_1"}) {
-    volPath = Form("/cave_1/barrel_1/FV0_1/FV0%s", half);
-    symName = Form("FV0%s", half);
-    LOG(info) << "FV0: Add alignable volume: " << symName << ": " << volPath;
-    if (!gGeoManager->SetAlignableEntry(symName.Data(), volPath.Data())) {
-      LOG(fatal) << "FV0: Unable to set alignable entry! " << symName << ": " << volPath;
+  auto addAlignabelVolume = [](const std::string& volPath, const std::string& symName) -> void {
+    LOG(info) << "FV0: Add alignable volume: " << symName << " <-> " << volPath;
+    if (!gGeoManager->SetAlignableEntry(symName.c_str(), volPath.c_str())) {
+      LOG(fatal) << "FV0: Unable to set alignable entry! " << symName << " <-> " << volPath;
     }
-  }
+  };
+
+  addAlignabelVolume("/cave_1/barrel_1/FV0_1/FV0RIGHT_0", Geometry::getDetectorRightSymName());
+  addAlignabelVolume("/cave_1/barrel_1/FV0_1/FV0LEFT_1", Geometry::getDetectorLeftSymName());
 }
 
 o2::fv0::Hit* Detector::addHit(Int_t trackId, Int_t cellId,
diff --git a/Detectors/FIT/FV0/simulation/src/Digitizer.cxx b/Detectors/FIT/FV0/simulation/src/Digitizer.cxx
index 04a36515d794d..8c1d2dc8824e2 100644
--- a/Detectors/FIT/FV0/simulation/src/Digitizer.cxx
+++ b/Detectors/FIT/FV0/simulation/src/Digitizer.cxx
@@ -98,8 +98,13 @@ void Digitizer::process(const std::vector& hits,
   for (auto ids : hitIdx) {
     const auto& hit = hits[ids];
     Int_t detId = hit.GetDetectorID();
-    Double_t hitEdep = hit.GetHitValue() * 1e3; //convert to MeV
-    Float_t const hitTime = hit.GetTime() * 1e9;
+
+    if (mDeadChannelMap && !mDeadChannelMap->isChannelAlive(detId)) {
+      continue;
+    }
+
+    Double_t hitEdep = hit.GetHitValue() * 1e3;  // convert to MeV
+    Float_t const hitTime = hit.GetTime() * 1e9; // convert to ns
     // TODO: check how big is inaccuracy if more than 1 'below-threshold' particles hit the same detector cell
     if (hitEdep < FV0DigParam::Instance().singleMipThreshold || hitTime > FV0DigParam::Instance().singleHitTimeThreshold) {
       continue;
@@ -127,7 +132,7 @@ void Digitizer::process(const std::vector& hits,
       Double_t const nPhotons = hitEdep * DP::N_PHOTONS_PER_MEV;
       float const nPhE = SimulateLightYield(detId, nPhotons);
       float const mipFraction = float(nPhE / FV0DigParam::Instance().avgNumberPhElectronPerMip);
-      Float_t timeHit = hitTime;
+      Long64_t timeHit = hitTime;
       timeHit += mIntRecord.getTimeNS();
       o2::InteractionTimeRecord const irHit(timeHit);
       std::array cachedIR;
@@ -138,15 +143,17 @@ void Digitizer::process(const std::vector& hits,
         if (tNS < 0 && cachedIR[nCachedIR] > irHit) {
           continue; // don't go to negative BC/orbit (it will wrap)
         }
-        setBCCache(cachedIR[nCachedIR++]); // ensure existence of cached container
-      }                                    //BCCache loop
-      createPulse(mipFraction, hit.GetTrackID(), hitTime, cachedIR, nCachedIR, detId);
+        // ensure existence of cached container
+        setBCCache(cachedIR[nCachedIR++]);
+      } // BCCache loop
+
+      createPulse(mipFraction, hit.GetTrackID(), hitTime, hit.GetPos().R(), cachedIR, nCachedIR, detId);
 
     } //while loop
   }   //hitloop
 }
 
-void Digitizer::createPulse(float mipFraction, int parID, const double hitTime,
+void Digitizer::createPulse(float mipFraction, int parID, const double hitTime, const float hitR,
                             std::array const& cachedIR, int nCachedIR, const int detId)
 {
 
@@ -160,8 +167,9 @@ void Digitizer::createPulse(float mipFraction, int parID, const double hitTime,
     }
   }
 
-  ///Time of flight subtracted from Hit time //TODO have different TOF according to thr ring number
-  Int_t const NBinShift = std::lround((hitTime - FV0DigParam::Instance().globalTimeOfFlight) / FV0DigParam::Instance().waveformBinWidth);
+  // Subtract time-of-flight from hit time
+  const float timeOfFlight = hitR / o2::constants::physics::LightSpeedCm2NS;
+  Int_t const NBinShift = std::lround((hitTime - timeOfFlight + FV0DigParam::Instance().hitTimeOffset) / FV0DigParam::Instance().waveformBinWidth);
 
   if (NBinShift >= 0 && NBinShift < FV0DigParam::Instance().waveformNbins) {
     mPmtResponseTemp.resize(FV0DigParam::Instance().waveformNbins, 0.);
diff --git a/Detectors/FIT/FV0/simulation/src/digit2raw.cxx b/Detectors/FIT/FV0/simulation/src/digit2raw.cxx
index d6f46b397bd4f..c7441407093ab 100644
--- a/Detectors/FIT/FV0/simulation/src/digit2raw.cxx
+++ b/Detectors/FIT/FV0/simulation/src/digit2raw.cxx
@@ -13,21 +13,13 @@
 /// \author ruben.shahoyan@cern.ch afurs@cern.ch
 
 #include 
-#include 
 #include 
-#include "CommonUtils/StringUtils.h"
-#include "CommonUtils/ConfigurableParam.h"
-#include "DetectorsRaw/HBFUtils.h"
 #include "FV0Raw/RawWriterFV0.h"
-#include "DataFormatsParameters/GRPObject.h"
 
 /// MC->raw conversion for FV0
 
 namespace bpo = boost::program_options;
 
-void digit2raw(const std::string& inpName, const std::string& outDir, int verbosity, const std::string& fileFor, uint32_t rdhV, bool noEmptyHBF, const std::string& flpName,
-               const std::string& ccdbUrl, const std::string& lutPath, int superPageSizeInB = 1024 * 1024);
-
 int main(int argc, char** argv)
 {
   bpo::variables_map vm;
@@ -38,21 +30,10 @@ int main(int argc, char** argv)
   bpo::positional_options_description opt_pos;
 
   try {
-    auto add_option = opt_general.add_options();
-    add_option("help,h", "Print this help message");
-    add_option("verbosity,v", bpo::value()->default_value(0), "verbosity level");
-    //    add_option("input-file,i", bpo::value()->default_value(o2::base::NameConf::getDigitsFileName(o2::detectors::DetID::FV0)),"input FV0 digits file"); // why not used?
-    add_option("input-file,i", bpo::value()->default_value("fv0digits.root"), "input FV0 digits file");
-    add_option("flp-name", bpo::value()->default_value("alio2-cr1-flp180"), "single file per: all,flp,cru,link");
-    add_option("file-for,f", bpo::value()->default_value("all"), "single file per: all,flp,cru,link");
-    add_option("output-dir,o", bpo::value()->default_value("./"), "output directory for raw data");
-    uint32_t defRDH = o2::raw::RDHUtils::getVersion();
-    add_option("rdh-version,r", bpo::value()->default_value(defRDH), "RDH version to use");
-    add_option("no-empty-hbf,e", bpo::value()->default_value(false)->implicit_value(true), "do not create empty HBF pages (except for HBF starting TF)");
-    add_option("hbfutils-config,u", bpo::value()->default_value(std::string(o2::base::NameConf::DIGITIZATIONCONFIGFILE)), "config file for HBFUtils (or none)");
-    add_option("configKeyValues", bpo::value()->default_value(""), "comma-separated configKeyValues");
-    add_option("ccdb-path", bpo::value()->default_value(""), "CCDB url which contains LookupTable");
-    add_option("lut-path", bpo::value()->default_value(""), "LookupTable path, e.g. FV0/LookupTable");
+    opt_general.add_options()("help,h", "Print this help message");
+    // config with FV0 defaults
+    o2::fit::DigitToRawConfig::configureExecOptions(opt_general, "fv0digits.root", "alio2-cr1-flp180");
+    // common part
     opt_all.add(opt_general).add(opt_hidden);
     bpo::store(bpo::command_line_parser(argc, argv).options(opt_all).positional(opt_pos).run(), vm);
 
@@ -71,59 +52,12 @@ int main(int argc, char** argv)
     std::cerr << e.what() << ", application will now exit" << std::endl;
     exit(2);
   }
-
-  std::string confDig = vm["hbfutils-config"].as();
-  if (!confDig.empty() && confDig != "none") {
-    o2::conf::ConfigurableParam::updateFromFile(confDig, "HBFUtils");
+  const o2::fit::DigitToRawConfig cfg(vm);
+  if (!cfg.mEnablePadding) {
+    o2::fit::DigitToRawDevice::digit2raw(cfg);
+  } else {
+    o2::fit::DigitToRawDevice::digit2raw(cfg);
   }
-  o2::conf::ConfigurableParam::updateFromString(vm["configKeyValues"].as());
-  digit2raw(vm["input-file"].as(),
-            vm["output-dir"].as(),
-            vm["verbosity"].as(),
-            vm["file-for"].as(),
-            vm["rdh-version"].as(),
-            vm["no-empty-hbf"].as(),
-            vm["flp-name"].as(),
-            vm["ccdb-path"].as(),
-            vm["lut-path"].as());
-
   o2::raw::HBFUtils::Instance().print();
-
   return 0;
 }
-
-void digit2raw(const std::string& inpName, const std::string& outDir, int verbosity, const std::string& fileFor, uint32_t rdhV, bool noEmptyHBF, const std::string& flpName, const std::string& ccdbUrl, const std::string& lutPath, int superPageSizeInB)
-{
-  TStopwatch swTot;
-  swTot.Start();
-  o2::fv0::RawWriterFV0 m2r;
-  m2r.setFileFor(fileFor);
-  m2r.setFlpName(flpName);
-  m2r.setVerbosity(verbosity);
-  if (ccdbUrl != "") {
-    m2r.setCCDBurl(ccdbUrl);
-  }
-  if (lutPath != "") {
-    m2r.setLUTpath(lutPath);
-  }
-  auto& wr = m2r.getWriter();
-  std::string inputGRP = o2::base::NameConf::getGRPFileName();
-  const auto grp = o2::parameters::GRPObject::loadFrom(inputGRP);
-  wr.setContinuousReadout(grp->isDetContinuousReadOut(o2::detectors::DetID::FV0)); // must be set explicitly
-  wr.setSuperPageSize(superPageSizeInB);
-  wr.useRDHVersion(rdhV);
-  wr.setDontFillEmptyHBF(noEmptyHBF);
-
-  o2::raw::assertOutputDirectory(outDir);
-
-  std::string outDirName(outDir);
-  if (outDirName.back() != '/') {
-    outDirName += '/';
-  }
-
-  m2r.convertDigitsToRaw(outDirName, inpName);
-  wr.writeConfFile(wr.getOrigin().str, "RAWDATA", o2::utils::Str::concat_string(outDirName, wr.getOrigin().str, "raw.cfg"));
-  //
-  swTot.Stop();
-  swTot.Print();
-}
diff --git a/Detectors/FIT/FV0/workflow/CMakeLists.txt b/Detectors/FIT/FV0/workflow/CMakeLists.txt
index 5ff38e991432c..a304adc61b5fd 100644
--- a/Detectors/FIT/FV0/workflow/CMakeLists.txt
+++ b/Detectors/FIT/FV0/workflow/CMakeLists.txt
@@ -53,6 +53,31 @@ o2_add_executable(flp-dpl-workflow
                   PUBLIC_LINK_LIBRARIES O2::FV0Workflow O2::FITWorkflow O2::FV0Raw
                   TARGETVARNAME fv0flpexe)
 
+o2_add_executable(recpoints-reader-workflow
+                  SOURCES src/recpoints-reader-workflow.cxx
+                  COMPONENT_NAME fv0
+                  PUBLIC_LINK_LIBRARIES O2::FV0Workflow)
+
+o2_add_executable(recpoints-writer-workflow
+                  SOURCES src/recpoints-writer-workflow.cxx
+                  COMPONENT_NAME fv0
+                  PUBLIC_LINK_LIBRARIES O2::FV0Workflow)
+
+o2_add_executable(integrate-cluster-workflow
+                  SOURCES src/cluster-integrator.cxx
+                  COMPONENT_NAME fv0
+                  PUBLIC_LINK_LIBRARIES O2::FV0Workflow O2::FITWorkflow)
+
+o2_add_executable(integrate-cluster-reader-workflow
+                  SOURCES src/cluster-integrator-reader.cxx
+                  COMPONENT_NAME fv0
+                  PUBLIC_LINK_LIBRARIES O2::FV0Workflow O2::FITWorkflow)
+
+o2_add_executable(merge-integrate-cluster-workflow
+                  SOURCES src/cluster-merge-integrator.cxx
+                  COMPONENT_NAME fv0
+                  PUBLIC_LINK_LIBRARIES O2::FV0Workflow O2::FITWorkflow)
+
 if(NOT APPLE)
 
  set_property(TARGET ${fitrecoexe} PROPERTY LINK_WHAT_YOU_USE ON)
diff --git a/Detectors/FIT/FV0/workflow/include/FV0Workflow/EntropyDecoderSpec.h b/Detectors/FIT/FV0/workflow/include/FV0Workflow/EntropyDecoderSpec.h
index 67b74f45e42bf..76f1aae5e728d 100644
--- a/Detectors/FIT/FV0/workflow/include/FV0Workflow/EntropyDecoderSpec.h
+++ b/Detectors/FIT/FV0/workflow/include/FV0Workflow/EntropyDecoderSpec.h
@@ -28,7 +28,7 @@ namespace fv0
 class EntropyDecoderSpec : public o2::framework::Task
 {
  public:
-  EntropyDecoderSpec(int verbosity);
+  EntropyDecoderSpec(int verbosity, const std::string& ctfdictOpt = "none");
   ~EntropyDecoderSpec() override = default;
   void run(o2::framework::ProcessingContext& pc) final;
   void init(o2::framework::InitContext& ic) final;
@@ -41,7 +41,7 @@ class EntropyDecoderSpec : public o2::framework::Task
 };
 
 /// create a processor spec
-framework::DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec);
+framework::DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec, const std::string& ctfdictOpt);
 
 } // namespace fv0
 } // namespace o2
diff --git a/Detectors/FIT/FV0/workflow/include/FV0Workflow/EntropyEncoderSpec.h b/Detectors/FIT/FV0/workflow/include/FV0Workflow/EntropyEncoderSpec.h
index 7e9ff0c2b2771..0df9403a88a12 100644
--- a/Detectors/FIT/FV0/workflow/include/FV0Workflow/EntropyEncoderSpec.h
+++ b/Detectors/FIT/FV0/workflow/include/FV0Workflow/EntropyEncoderSpec.h
@@ -28,7 +28,7 @@ namespace fv0
 class EntropyEncoderSpec : public o2::framework::Task
 {
  public:
-  EntropyEncoderSpec();
+  EntropyEncoderSpec(bool selIR, const std::string& ctfdictOpt = "none");
   ~EntropyEncoderSpec() override = default;
   void run(o2::framework::ProcessingContext& pc) final;
   void init(o2::framework::InitContext& ic) final;
@@ -37,11 +37,12 @@ class EntropyEncoderSpec : public o2::framework::Task
 
  private:
   o2::fv0::CTFCoder mCTFCoder;
+  bool mSelIR = false;
   TStopwatch mTimer;
 };
 
 /// create a processor spec
-framework::DataProcessorSpec getEntropyEncoderSpec();
+framework::DataProcessorSpec getEntropyEncoderSpec(bool selIR = false, const std::string& ctfdictOpt = "none");
 
 } // namespace fv0
 } // namespace o2
diff --git a/Detectors/FIT/FV0/workflow/include/FV0Workflow/RecoWorkflow.h b/Detectors/FIT/FV0/workflow/include/FV0Workflow/RecoWorkflow.h
index 015870d9178e2..f035b2406e5ba 100644
--- a/Detectors/FIT/FV0/workflow/include/FV0Workflow/RecoWorkflow.h
+++ b/Detectors/FIT/FV0/workflow/include/FV0Workflow/RecoWorkflow.h
@@ -20,7 +20,7 @@ namespace o2
 {
 namespace fv0
 {
-framework::WorkflowSpec getRecoWorkflow(bool useMC, bool disableRootInp, bool disableRootOut);
+framework::WorkflowSpec getRecoWorkflow(bool useMC, bool disableRootInp, bool disableRootOut, bool useDeadChannelMap);
 } // namespace fv0
 } // namespace o2
 #endif
diff --git a/Detectors/FIT/FV0/workflow/include/FV0Workflow/ReconstructionSpec.h b/Detectors/FIT/FV0/workflow/include/FV0Workflow/ReconstructionSpec.h
index 32218795ca90c..934ce4d2c4a66 100644
--- a/Detectors/FIT/FV0/workflow/include/FV0Workflow/ReconstructionSpec.h
+++ b/Detectors/FIT/FV0/workflow/include/FV0Workflow/ReconstructionSpec.h
@@ -20,6 +20,7 @@
 #include "DataFormatsFV0/RecPoints.h"
 #include "FV0Base/Constants.h"
 #include "TStopwatch.h"
+#include "CommonUtils/NameConf.h"
 
 using namespace o2::framework;
 
@@ -33,16 +34,19 @@ class ReconstructionDPL : public Task
   static constexpr int NCHANNELS = o2::fv0::Constants::nFv0Channels;
 
  public:
-  ReconstructionDPL(bool useMC) : mUseMC(useMC) {}
+  ReconstructionDPL(bool useMC, bool useDeadChannelMap, const std::string ccdbpath) : mUseMC(useMC), mUseDeadChannelMap(useDeadChannelMap), mCCDBpath(ccdbpath) {}
   ~ReconstructionDPL() override = default;
   void init(InitContext& ic) final;
   void run(ProcessingContext& pc) final;
   void endOfStream(framework::EndOfStreamContext& ec) final;
-  void finaliseCCDB(o2::framework::ConcreteDataMatcher& matcher, void* obj) final;
+  void finaliseCCDB(ConcreteDataMatcher& matcher, void* obj) final;
 
  private:
-  void updateTimeDependentParams(o2::framework::ProcessingContext& pc);
   bool mUseMC = false;
+  bool mUpdateCCDB = true;
+  bool mUseDeadChannelMap = true;
+  bool mUpdateDeadChannelMap = true;
+  const std::string mCCDBpath = o2::base::NameConf::getCCDBServer();
   std::vector mRecPoints;
   std::vector mRecChData;
   o2::fv0::BaseRecoTask mReco;
@@ -51,7 +55,7 @@ class ReconstructionDPL : public Task
 };
 
 /// create a processor spec
-framework::DataProcessorSpec getReconstructionSpec(bool useMC = false);
+framework::DataProcessorSpec getReconstructionSpec(bool useMC = false, bool useDeadChannelMap = true, const std::string ccdbpath = "http://alice-ccdb.cern.ch");
 
 } // namespace fv0
 } // namespace o2
diff --git a/Detectors/FIT/FV0/workflow/src/DigitReaderSpec.cxx b/Detectors/FIT/FV0/workflow/src/DigitReaderSpec.cxx
index c81d61df19019..a49bda2cec18b 100644
--- a/Detectors/FIT/FV0/workflow/src/DigitReaderSpec.cxx
+++ b/Detectors/FIT/FV0/workflow/src/DigitReaderSpec.cxx
@@ -34,7 +34,7 @@ void DigitReader::init(InitContext& ic)
 {
   auto filename = o2::utils::Str::concat_string(o2::utils::Str::rectifyDirectory(ic.options().get("input-dir")),
                                                 ic.options().get("fv0-digit-infile"));
-  mFile = std::make_unique(filename.c_str(), "OLD");
+  mFile.reset(TFile::Open(filename.c_str()));
   if (!mFile->IsOpen()) {
     LOG(error) << "Cannot open the " << filename.c_str() << " file !";
     throw std::runtime_error("cannot open input digits file");
@@ -65,13 +65,13 @@ void DigitReader::run(ProcessingContext& pc)
   assert(ent < mTree->GetEntries()); // this should not happen
   mTree->GetEntry(ent);
   LOG(debug) << "FV0DigitReader pushed " << channels.size() << " channels in " << digits.size() << " digits";
-  pc.outputs().snapshot(Output{"FV0", "DIGITSBC", 0, Lifetime::Timeframe}, digits);
-  pc.outputs().snapshot(Output{"FV0", "DIGITSCH", 0, Lifetime::Timeframe}, channels);
+  pc.outputs().snapshot(Output{"FV0", "DIGITSBC", 0}, digits);
+  pc.outputs().snapshot(Output{"FV0", "DIGITSCH", 0}, channels);
   if (mUseMC) {
-    pc.outputs().snapshot(Output{"FV0", "DIGITSMCTR", 0, Lifetime::Timeframe}, labels);
+    pc.outputs().snapshot(Output{"FV0", "DIGITSMCTR", 0}, labels);
   }
   if (mUseTrgInput) {
-    pc.outputs().snapshot(Output{"FV0", "TRIGGERINPUT", 0, Lifetime::Timeframe}, trgInput);
+    pc.outputs().snapshot(Output{"FV0", "TRIGGERINPUT", 0}, trgInput);
   }
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get().endOfStream();
diff --git a/Detectors/FIT/FV0/workflow/src/EntropyDecoderSpec.cxx b/Detectors/FIT/FV0/workflow/src/EntropyDecoderSpec.cxx
index 506b1176a475a..7babe9fdea6ed 100644
--- a/Detectors/FIT/FV0/workflow/src/EntropyDecoderSpec.cxx
+++ b/Detectors/FIT/FV0/workflow/src/EntropyDecoderSpec.cxx
@@ -24,12 +24,12 @@ namespace o2
 {
 namespace fv0
 {
-
-EntropyDecoderSpec::EntropyDecoderSpec(int verbosity) : mCTFCoder(o2::ctf::CTFCoderBase::OpType::Decoder)
+EntropyDecoderSpec::EntropyDecoderSpec(int verbosity, const std::string& ctfdictOpt) : mCTFCoder(o2::ctf::CTFCoderBase::OpType::Decoder, ctfdictOpt)
 {
   mTimer.Stop();
   mTimer.Reset();
   mCTFCoder.setVerbosity(verbosity);
+  mCTFCoder.setDictBinding("ctfdict_FV0");
 }
 
 void EntropyDecoderSpec::finaliseCCDB(o2::framework::ConcreteDataMatcher& matcher, void* obj)
@@ -50,8 +50,8 @@ void EntropyDecoderSpec::run(ProcessingContext& pc)
   mTimer.Start(false);
   o2::ctf::CTFIOSize iosize;
 
-  mCTFCoder.updateTimeDependentParams(pc);
-  auto buff = pc.inputs().get>("ctf");
+  mCTFCoder.updateTimeDependentParams(pc, true);
+  auto buff = pc.inputs().get>("ctf_FV0");
 
   auto& digits = pc.outputs().make>(OutputRef{"digits"});
   auto& channels = pc.outputs().make>(OutputRef{"channels"});
@@ -72,7 +72,7 @@ void EntropyDecoderSpec::endOfStream(EndOfStreamContext& ec)
        mTimer.CpuTime(), mTimer.RealTime(), mTimer.Counter() - 1);
 }
 
-DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec)
+DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec, const std::string& ctfdictOpt)
 {
   std::vector outputs{
     OutputSpec{{"digits"}, "FV0", "DIGITSBC", 0, Lifetime::Timeframe},
@@ -80,16 +80,19 @@ DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec)
     OutputSpec{{"ctfrep"}, "FV0", "CTFDECREP", 0, Lifetime::Timeframe}};
 
   std::vector inputs;
-  inputs.emplace_back("ctf", "FV0", "CTFDATA", sspec, Lifetime::Timeframe);
-  inputs.emplace_back("ctfdict", "FV0", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("FV0/Calib/CTFDictionary"));
+  inputs.emplace_back("ctf_FV0", "FV0", "CTFDATA", sspec, Lifetime::Timeframe);
+
+  if (ctfdictOpt.empty() || ctfdictOpt == "ccdb") {
+    inputs.emplace_back("ctfdict_FV0", "FV0", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("FV0/Calib/CTFDictionaryTree"));
+  }
+  inputs.emplace_back("trigoffset", "CTP", "Trig_Offset", 0, Lifetime::Condition, ccdbParamSpec("CTP/Config/TriggerOffsets"));
 
   return DataProcessorSpec{
     "fv0-entropy-decoder",
     inputs,
     outputs,
-    AlgorithmSpec{adaptFromTask(verbosity)},
-    Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}}};
+    AlgorithmSpec{adaptFromTask(verbosity, ctfdictOpt)},
+    Options{{"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}};
 }
-
 } // namespace fv0
 } // namespace o2
diff --git a/Detectors/FIT/FV0/workflow/src/EntropyEncoderSpec.cxx b/Detectors/FIT/FV0/workflow/src/EntropyEncoderSpec.cxx
index 8353c1942001f..2448af09fac4e 100644
--- a/Detectors/FIT/FV0/workflow/src/EntropyEncoderSpec.cxx
+++ b/Detectors/FIT/FV0/workflow/src/EntropyEncoderSpec.cxx
@@ -25,8 +25,7 @@ namespace o2
 {
 namespace fv0
 {
-
-EntropyEncoderSpec::EntropyEncoderSpec() : mCTFCoder(o2::ctf::CTFCoderBase::OpType::Encoder)
+EntropyEncoderSpec::EntropyEncoderSpec(bool selIR, const std::string& ctfdictOpt) : mCTFCoder(o2::ctf::CTFCoderBase::OpType::Encoder, ctfdictOpt), mSelIR(selIR)
 {
   mTimer.Stop();
   mTimer.Reset();
@@ -48,12 +47,19 @@ void EntropyEncoderSpec::run(ProcessingContext& pc)
 {
   auto cput = mTimer.CpuTime();
   mTimer.Start(false);
-  mCTFCoder.updateTimeDependentParams(pc);
+  mCTFCoder.updateTimeDependentParams(pc, true);
   auto digits = pc.inputs().get>("digits");
   auto channels = pc.inputs().get>("channels");
-  auto& buffer = pc.outputs().make>(Output{"FV0", "CTFDATA", 0, Lifetime::Timeframe});
+  if (mSelIR) {
+    mCTFCoder.setSelectedIRFrames(pc.inputs().get>("selIRFrames"));
+  }
+
+  auto& buffer = pc.outputs().make>(Output{"FV0", "CTFDATA", 0});
   auto iosize = mCTFCoder.encode(buffer, digits, channels);
   pc.outputs().snapshot({"ctfrep", 0}, iosize);
+  if (mSelIR) {
+    mCTFCoder.getIRFramesSelector().clear();
+  }
   mTimer.Stop();
   LOG(info) << iosize.asString() << " in " << mTimer.CpuTime() - cput << " s";
 }
@@ -64,22 +70,28 @@ void EntropyEncoderSpec::endOfStream(EndOfStreamContext& ec)
        mTimer.CpuTime(), mTimer.RealTime(), mTimer.Counter() - 1);
 }
 
-DataProcessorSpec getEntropyEncoderSpec()
+DataProcessorSpec getEntropyEncoderSpec(bool selIR, const std::string& ctfdictOpt)
 {
   std::vector inputs;
   inputs.emplace_back("digits", "FV0", "DIGITSBC", 0, Lifetime::Timeframe);
   inputs.emplace_back("channels", "FV0", "DIGITSCH", 0, Lifetime::Timeframe);
-  inputs.emplace_back("ctfdict", "FV0", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("FV0/Calib/CTFDictionary"));
 
+  if (ctfdictOpt.empty() || ctfdictOpt == "ccdb") {
+    inputs.emplace_back("ctfdict", "FV0", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("FV0/Calib/CTFDictionaryTree"));
+  }
+  if (selIR) {
+    inputs.emplace_back("selIRFrames", "CTF", "SELIRFRAMES", 0, Lifetime::Timeframe);
+  }
   return DataProcessorSpec{
     "fv0-entropy-encoder",
     inputs,
     Outputs{{"FV0", "CTFDATA", 0, Lifetime::Timeframe},
             {{"ctfrep"}, "FV0", "CTFENCREP", 0, Lifetime::Timeframe}},
-    AlgorithmSpec{adaptFromTask()},
-    Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}},
-            {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}}}};
+    AlgorithmSpec{adaptFromTask(selIR, ctfdictOpt)},
+    Options{{"irframe-margin-bwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame lower boundary when selection is requested"}},
+            {"irframe-margin-fwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame upper boundary when selection is requested"}},
+            {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}},
+            {"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}};
 }
-
 } // namespace fv0
 } // namespace o2
diff --git a/Detectors/FIT/FV0/workflow/src/FV0DigitWriterSpec.cxx b/Detectors/FIT/FV0/workflow/src/FV0DigitWriterSpec.cxx
index eda1f5061e6f2..57ecfe368625c 100644
--- a/Detectors/FIT/FV0/workflow/src/FV0DigitWriterSpec.cxx
+++ b/Detectors/FIT/FV0/workflow/src/FV0DigitWriterSpec.cxx
@@ -51,19 +51,19 @@ o2::framework::DataProcessorSpec getFV0DigitWriterSpec(bool mctruth, bool trigIn
                                   "fv0digits.root",
                                   "o2sim",
                                   MakeRootTreeWriterSpec::CustomClose(finishWriting),
-                                  BranchDefinition>{InputSpec{"digitBCinput", "FV0", "DIGITSBC"}, "FV0DigitBC", "fv0-digits-branch-name", 1,
+                                  BranchDefinition>{InputSpec{"digitBCinput", "FV0", "DIGITSBC"}, "FV0DigitBC", 1,
                                                                                 logger},
-                                  BranchDefinition>{InputSpec{"digitChinput", "FV0", "DIGITSCH"}, "FV0DigitCh", "fv0-chhdata-branch-name"},
-                                  BranchDefinition>{InputSpec{"digitTrinput", "FV0", "TRIGGERINPUT"}, "TRIGGERINPUT", "fv0-triggerinput-branch-name"},
+                                  BranchDefinition>{InputSpec{"digitChinput", "FV0", "DIGITSCH"}, "FV0DigitCh"},
+                                  BranchDefinition>{InputSpec{"digitTrinput", "FV0", "TRIGGERINPUT"}, "TRIGGERINPUT"},
                                   std::move(labelsdef))();
   } else {
     return MakeRootTreeWriterSpec("FV0DigitWriterRaw",
                                   "o2_fv0digits.root",
                                   "o2sim",
                                   MakeRootTreeWriterSpec::CustomClose(finishWriting),
-                                  BranchDefinition>{InputSpec{"digitBCinput", "FV0", "DIGITSBC"}, "FV0DigitBC", "fv0-digits-branch-name", 1,
+                                  BranchDefinition>{InputSpec{"digitBCinput", "FV0", "DIGITSBC"}, "FV0DigitBC", 1,
                                                                                 logger},
-                                  BranchDefinition>{InputSpec{"digitChinput", "FV0", "DIGITSCH"}, "FV0DigitCh", "fv0-chhdata-branch-name"},
+                                  BranchDefinition>{InputSpec{"digitChinput", "FV0", "DIGITSCH"}, "FV0DigitCh"},
                                   std::move(labelsdef))();
   }
 }
diff --git a/Detectors/FIT/FV0/workflow/src/RecPointReaderSpec.cxx b/Detectors/FIT/FV0/workflow/src/RecPointReaderSpec.cxx
index 163beacf0c140..5997cac500ee6 100644
--- a/Detectors/FIT/FV0/workflow/src/RecPointReaderSpec.cxx
+++ b/Detectors/FIT/FV0/workflow/src/RecPointReaderSpec.cxx
@@ -49,8 +49,8 @@ void RecPointReader::run(ProcessingContext& pc)
   mTree->GetEntry(ent);
 
   LOG(debug) << "FV0 RecPointReader pushes " << mRecPoints->size() << " recpoints with " << mChannelData->size() << " channels at entry " << ent;
-  pc.outputs().snapshot(Output{mOrigin, "RECPOINTS", 0, Lifetime::Timeframe}, *mRecPoints);
-  pc.outputs().snapshot(Output{mOrigin, "RECCHDATA", 0, Lifetime::Timeframe}, *mChannelData);
+  pc.outputs().snapshot(Output{mOrigin, "RECPOINTS", 0}, *mRecPoints);
+  pc.outputs().snapshot(Output{mOrigin, "RECCHDATA", 0}, *mChannelData);
 
   if (mTree->GetReadEntry() + 1 >= mTree->GetEntries()) {
     pc.services().get().endOfStream();
diff --git a/Detectors/FIT/FV0/workflow/src/RecoWorkflow.cxx b/Detectors/FIT/FV0/workflow/src/RecoWorkflow.cxx
index 6bfc5479303d1..a0ef71b75765a 100644
--- a/Detectors/FIT/FV0/workflow/src/RecoWorkflow.cxx
+++ b/Detectors/FIT/FV0/workflow/src/RecoWorkflow.cxx
@@ -22,14 +22,13 @@ namespace o2
 namespace fv0
 {
 
-framework::WorkflowSpec getRecoWorkflow(bool useMC, bool disableRootInp, bool disableRootOut)
+framework::WorkflowSpec getRecoWorkflow(bool useMC, bool disableRootInp, bool disableRootOut, bool useDeadChannelMap)
 {
   framework::WorkflowSpec specs;
   if (!disableRootInp) {
     specs.emplace_back(o2::fv0::getDigitReaderSpec(useMC));
   }
-
-  specs.emplace_back(o2::fv0::getReconstructionSpec(useMC));
+  specs.emplace_back(o2::fv0::getReconstructionSpec(useMC, useDeadChannelMap));
   if (!disableRootOut) {
     specs.emplace_back(o2::fv0::getRecPointWriterSpec(useMC));
   }
diff --git a/Detectors/FIT/FV0/workflow/src/ReconstructionSpec.cxx b/Detectors/FIT/FV0/workflow/src/ReconstructionSpec.cxx
index 7a6b20fbbbfab..cdf297b334588 100644
--- a/Detectors/FIT/FV0/workflow/src/ReconstructionSpec.cxx
+++ b/Detectors/FIT/FV0/workflow/src/ReconstructionSpec.cxx
@@ -21,6 +21,8 @@
 #include "DataFormatsFV0/ChannelData.h"
 #include "DataFormatsFV0/MCLabel.h"
 #include "DataFormatsFV0/FV0ChannelTimeCalibrationObject.h"
+#include "DataFormatsFIT/DeadChannelMap.h"
+#include "Framework/CCDBParamSpec.h"
 
 using namespace o2::framework;
 
@@ -38,9 +40,9 @@ void ReconstructionDPL::init(InitContext& ic)
 
 void ReconstructionDPL::run(ProcessingContext& pc)
 {
-  updateTimeDependentParams(pc);
   mTimer.Start(false);
   mRecPoints.clear();
+  mRecChData.clear();
   auto digits = pc.inputs().get>("digits");
   auto digch = pc.inputs().get>("digch");
   // RS: if we need to process MC truth, uncomment lines below
@@ -49,25 +51,42 @@ void ReconstructionDPL::run(ProcessingContext& pc)
   if (mUseMC) {
     LOG(info) << "Ignoring MC info";
   }
+  if (mUpdateCCDB) {
+    auto caliboffsets = pc.inputs().get("fv0offsets");
+    mReco.SetChannelOffset(caliboffsets.get());
+  }
+  if (mUseDeadChannelMap && mUpdateDeadChannelMap) {
+    auto deadChannelMap = pc.inputs().get("deadChannelMap");
+    mReco.SetDeadChannelMap(deadChannelMap.get());
+  }
+
   int nDig = digits.size();
   LOG(debug) << " nDig " << nDig << " | ndigch " << digch.size();
   mRecPoints.reserve(nDig);
-  mRecChData.resize(digch.size());
   for (int id = 0; id < nDig; id++) {
     const auto& digit = digits[id];
-    LOG(debug) << " ndig " << id << " bc " << digit.getIntRecord().bc << " orbit " << digit.getIntRecord().orbit;
+    LOG(debug) << " ndig " << id << " bc " << digit.getBC() << " orbit " << digit.getOrbit();
     auto channels = digit.getBunchChannelData(digch);
-    gsl::span out_ch(mRecChData);
-    out_ch = out_ch.subspan(digit.ref.getFirstEntry(), digit.ref.getEntries());
-    mRecPoints.emplace_back(mReco.process(digit, channels, out_ch));
+    mRecPoints.emplace_back(mReco.process(digit, channels, mRecChData));
   }
 
   LOG(debug) << "FV0 reconstruction pushes " << mRecPoints.size() << " RecPoints";
-  pc.outputs().snapshot(Output{mOrigin, "RECPOINTS", 0, Lifetime::Timeframe}, mRecPoints);
-  pc.outputs().snapshot(Output{mOrigin, "RECCHDATA", 0, Lifetime::Timeframe}, mRecChData);
+  pc.outputs().snapshot(Output{mOrigin, "RECPOINTS", 0}, mRecPoints);
+  pc.outputs().snapshot(Output{mOrigin, "RECCHDATA", 0}, mRecChData);
 
   mTimer.Stop();
 }
+//_______________________________________
+void ReconstructionDPL::finaliseCCDB(ConcreteDataMatcher& matcher, void* obj)
+{
+  if (matcher == ConcreteDataMatcher("FT0", "TimeOffset", 0)) {
+    mUpdateCCDB = false;
+    return;
+  }
+  if (matcher == ConcreteDataMatcher(o2::header::gDataOriginFV0, "DeadChannelMap", 0)) {
+    mUpdateDeadChannelMap = false;
+  }
+}
 
 void ReconstructionDPL::endOfStream(EndOfStreamContext& ec)
 {
@@ -75,31 +94,24 @@ void ReconstructionDPL::endOfStream(EndOfStreamContext& ec)
        mTimer.CpuTime(), mTimer.RealTime(), mTimer.Counter() - 1);
 }
 
-void ReconstructionDPL::updateTimeDependentParams(ProcessingContext& pc)
-{
-  pc.inputs().get("chtimeoffs");
-}
-
-void ReconstructionDPL::finaliseCCDB(ConcreteDataMatcher& matcher, void* obj)
-{
-  if (matcher == ConcreteDataMatcher("FV0", "CHANTIMEOFFS", 0)) {
-    LOG(info) << "ChannelTimeOffset updated";
-    mReco.setChannelOffset((o2::fv0::FV0ChannelTimeCalibrationObject*)obj);
-    return;
-  }
-}
-
-DataProcessorSpec getReconstructionSpec(bool useMC)
+DataProcessorSpec getReconstructionSpec(bool useMC, bool useDeadChannelMap, const std::string ccdbpath)
 {
   std::vector inputSpec;
   std::vector outputSpec;
   inputSpec.emplace_back("digits", o2::header::gDataOriginFV0, "DIGITSBC", 0, Lifetime::Timeframe);
   inputSpec.emplace_back("digch", o2::header::gDataOriginFV0, "DIGITSCH", 0, Lifetime::Timeframe);
-  inputSpec.emplace_back("chtimeoffs", o2::header::gDataOriginFV0, "CHANTIMEOFFS", 0, Lifetime::Condition, ccdbParamSpec("FV0/Calib/ChannelTimeOffset"));
   if (useMC) {
     LOG(info) << "Currently Reconstruction does not consume and provide MC truth";
     inputSpec.emplace_back("labels", o2::header::gDataOriginFV0, "DIGITSMCTR", 0, Lifetime::Timeframe);
   }
+  if (useDeadChannelMap) {
+    LOG(info) << "Dead channel map will be applied during reconstruction";
+    inputSpec.emplace_back("deadChannelMap", o2::header::gDataOriginFV0, "DeadChannelMap", 0, Lifetime::Condition, ccdbParamSpec("FV0/Calib/DeadChannelMap"));
+  }
+  inputSpec.emplace_back("fv0offsets", "FV0", "TimeOffset", 0,
+                         Lifetime::Condition,
+                         ccdbParamSpec("FV0/Calib/ChannelTimeOffset"));
+
   outputSpec.emplace_back(o2::header::gDataOriginFV0, "RECPOINTS", 0, Lifetime::Timeframe);
   outputSpec.emplace_back(o2::header::gDataOriginFV0, "RECCHDATA", 0, Lifetime::Timeframe);
 
@@ -107,7 +119,7 @@ DataProcessorSpec getReconstructionSpec(bool useMC)
     "fv0-reconstructor",
     inputSpec,
     outputSpec,
-    AlgorithmSpec{adaptFromTask(useMC)},
+    AlgorithmSpec{adaptFromTask(useMC, useDeadChannelMap, ccdbpath)},
     Options{}};
 }
 
diff --git a/Detectors/FIT/FV0/workflow/src/cluster-integrator-reader.cxx b/Detectors/FIT/FV0/workflow/src/cluster-integrator-reader.cxx
new file mode 100644
index 0000000000000..d8568b3066384
--- /dev/null
+++ b/Detectors/FIT/FV0/workflow/src/cluster-integrator-reader.cxx
@@ -0,0 +1,23 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "FITWorkflow/FITIntegrateClusterReaderSpec.h"
+#include "DataFormatsFV0/RecPoints.h"
+#include "Framework/runDataProcessing.h"
+
+using namespace o2::framework;
+
+WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
+{
+  WorkflowSpec wf;
+  wf.emplace_back(o2::fit::getFITIntegrateClusterReaderSpec());
+  return wf;
+}
diff --git a/Detectors/FIT/FV0/workflow/src/cluster-integrator.cxx b/Detectors/FIT/FV0/workflow/src/cluster-integrator.cxx
new file mode 100644
index 0000000000000..65c87ee52f127
--- /dev/null
+++ b/Detectors/FIT/FV0/workflow/src/cluster-integrator.cxx
@@ -0,0 +1,44 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "FITWorkflow/FITIntegrateClusterSpec.h"
+#include "FITWorkflow/FITIntegrateClusterWriterSpec.h"
+#include "CommonUtils/ConfigurableParam.h"
+#include "Framework/ConfigParamSpec.h"
+#include "DataFormatsFV0/RecPoints.h"
+
+using namespace o2::framework;
+
+void customize(std::vector& workflowOptions)
+{
+  std::vector options{
+    ConfigParamSpec{"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}},
+    {"min-NChan", VariantType::Int, 2, {"Minimum NChan signal required to avoid noise"}},
+    {"min-Ampl", VariantType::Int, 2, {"Minimum Ampl signal required to avoid noise"}},
+    {"disable-root-output", VariantType::Bool, false, {"disable root-files output writers"}}};
+  std::swap(workflowOptions, options);
+}
+
+#include "Framework/runDataProcessing.h"
+
+WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
+{
+  WorkflowSpec wf;
+  o2::conf::ConfigurableParam::updateFromString(cfgc.options().get("configKeyValues"));
+  const bool disableWriter = cfgc.options().get("disable-root-output");
+  const int minNChan = cfgc.options().get("min-NChan");
+  const int minAmpl = cfgc.options().get("min-Ampl");
+  wf.emplace_back(o2::fit::getFITIntegrateClusterSpec(disableWriter, minNChan, minAmpl));
+  if (!disableWriter) {
+    wf.emplace_back(o2::fit::getFITIntegrateClusterWriterSpec());
+  }
+  return wf;
+}
diff --git a/Detectors/FIT/FV0/workflow/src/cluster-merge-integrator.cxx b/Detectors/FIT/FV0/workflow/src/cluster-merge-integrator.cxx
new file mode 100644
index 0000000000000..5ad6d7d38d384
--- /dev/null
+++ b/Detectors/FIT/FV0/workflow/src/cluster-merge-integrator.cxx
@@ -0,0 +1,35 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "FITWorkflow/FITMergeIntegrateClusterSpec.h"
+#include "DataFormatsFV0/RecPoints.h"
+#include "CommonUtils/ConfigurableParam.h"
+#include "Framework/ConfigParamSpec.h"
+
+using namespace o2::framework;
+
+void customize(std::vector& workflowOptions)
+{
+  std::vector options{
+    ConfigParamSpec{"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}},
+  };
+  std::swap(workflowOptions, options);
+}
+
+#include "Framework/runDataProcessing.h"
+
+WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
+{
+  WorkflowSpec wf;
+  o2::conf::ConfigurableParam::updateFromString(cfgc.options().get("configKeyValues"));
+  wf.emplace_back(o2::fit::getFITMergeIntegrateClusterSpec());
+  return wf;
+}
diff --git a/Detectors/FIT/FV0/workflow/src/digits-reader-workflow.cxx b/Detectors/FIT/FV0/workflow/src/digits-reader-workflow.cxx
index d3cb4f3229fb8..0dd200b16968f 100644
--- a/Detectors/FIT/FV0/workflow/src/digits-reader-workflow.cxx
+++ b/Detectors/FIT/FV0/workflow/src/digits-reader-workflow.cxx
@@ -49,7 +49,7 @@ WorkflowSpec defineDataProcessing(const ConfigContext& ctx)
 {
   WorkflowSpec specs;
   o2::conf::ConfigurableParam::updateFromString(ctx.options().get("configKeyValues"));
-  DataProcessorSpec producer = o2::fv0::getDigitReaderSpec(ctx.options().get("disable-mc"), ctx.options().get("disable-trigger-input"));
+  DataProcessorSpec producer = o2::fv0::getDigitReaderSpec(!ctx.options().get("disable-mc"), ctx.options().get("disable-trigger-input"));
   specs.push_back(producer);
 
   // configure dpl timer to inject correct firstTForbit: start from the 1st orbit of TF containing 1st sampled orbit
diff --git a/Detectors/FIT/FV0/workflow/src/entropy-encoder-workflow.cxx b/Detectors/FIT/FV0/workflow/src/entropy-encoder-workflow.cxx
index 06800f6fa0453..932e0a37ee376 100644
--- a/Detectors/FIT/FV0/workflow/src/entropy-encoder-workflow.cxx
+++ b/Detectors/FIT/FV0/workflow/src/entropy-encoder-workflow.cxx
@@ -21,7 +21,10 @@ using namespace o2::framework;
 void customize(std::vector& workflowOptions)
 {
   // option allowing to set parameters
-  std::vector options{ConfigParamSpec{"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}}};
+  std::vector options{
+    ConfigParamSpec{"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}},
+    ConfigParamSpec{"ctf-dict", VariantType::String, "none", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}},
+    ConfigParamSpec{"select-ir-frames", VariantType::Bool, false, {"Subscribe and filter according to external IR Frames"}}};
 
   std::swap(workflowOptions, options);
 }
@@ -35,6 +38,6 @@ WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
   WorkflowSpec wf;
   // Update the (declared) parameters if changed from the command line
   o2::conf::ConfigurableParam::updateFromString(cfgc.options().get("configKeyValues"));
-  wf.emplace_back(o2::fv0::getEntropyEncoderSpec());
+  wf.emplace_back(o2::fv0::getEntropyEncoderSpec(cfgc.options().get("select-ir-frames"), cfgc.options().get("ctf-dict")));
   return wf;
 }
diff --git a/Detectors/FIT/FV0/workflow/src/fv0-flp-workflow.cxx b/Detectors/FIT/FV0/workflow/src/fv0-flp-workflow.cxx
index 4e1d816472f32..d288a3b9e49c8 100644
--- a/Detectors/FIT/FV0/workflow/src/fv0-flp-workflow.cxx
+++ b/Detectors/FIT/FV0/workflow/src/fv0-flp-workflow.cxx
@@ -58,6 +58,16 @@ void customize(std::vector& workflowOptions)
                     o2::framework::VariantType::Bool,
                     false,
                     {"do not subscribe to FLP/DISTSUBTIMEFRAME/0 message (no lost TF recovery)"}});
+  workflowOptions.push_back(
+    ConfigParamSpec{"input-sub-sampled",
+                    o2::framework::VariantType::Bool,
+                    false,
+                    {"SUB_RAWDATA DPL channel will be used as input, in case of dispatcher usage"}});
+  workflowOptions.push_back(
+    ConfigParamSpec{"disable-dpl-ccdb-fetcher",
+                    o2::framework::VariantType::Bool,
+                    false,
+                    {"Disable DPL CCDB fetcher, channel map will be uploaded during initialization by taking last entry in CCDB"}});
 }
 
 // ------------------------------------------------------------------
@@ -71,24 +81,26 @@ WorkflowSpec defineDataProcessing(ConfigContext const& configcontext)
   auto isExtendedMode = configcontext.options().get("tcm-extended-mode");
   auto disableRootOut = configcontext.options().get("disable-root-output");
   auto askSTFDist = !configcontext.options().get("ignore-dist-stf");
+  const auto isSubSampled = configcontext.options().get("input-sub-sampled");
+  const auto disableDplCcdbFetcher = configcontext.options().get("disable-dpl-ccdb-fetcher");
   o2::conf::ConfigurableParam::updateFromString(configcontext.options().get("configKeyValues"));
   LOG(info) << "WorkflowSpec FLPWorkflow";
-  //Type aliases
-  //using RawReaderFV0trgInput = o2::fit::RawReaderFIT;
+  // Type aliases
+  // using RawReaderFV0trgInput = o2::fit::RawReaderFIT;
   using RawReaderFV0 = o2::fit::RawReaderFIT;
-  //using RawReaderFV0trgInputExt = o2::fit::RawReaderFIT;
+  // using RawReaderFV0trgInputExt = o2::fit::RawReaderFIT;
   using RawReaderFV0ext = o2::fit::RawReaderFIT;
   using MCLabelCont = o2::dataformats::MCTruthContainer;
   o2::header::DataOrigin dataOrigin = o2::header::gDataOriginFV0;
   //
   WorkflowSpec specs;
   if (isExtendedMode) {
-    specs.emplace_back(o2::fit::getFITDataReaderDPLSpec(RawReaderFV0ext{dataOrigin, dumpReader}, askSTFDist));
+    specs.emplace_back(o2::fit::getFITDataReaderDPLSpec(RawReaderFV0ext{dataOrigin, dumpReader}, askSTFDist, isSubSampled, disableDplCcdbFetcher));
     if (!disableRootOut) {
       specs.emplace_back(o2::fit::FITDigitWriterSpecHelper::getFITDigitWriterSpec(false, false, dataOrigin));
     }
   } else {
-    specs.emplace_back(o2::fit::getFITDataReaderDPLSpec(RawReaderFV0{dataOrigin, dumpReader}, askSTFDist));
+    specs.emplace_back(o2::fit::getFITDataReaderDPLSpec(RawReaderFV0{dataOrigin, dumpReader}, askSTFDist, isSubSampled, disableDplCcdbFetcher));
     if (!disableRootOut) {
       specs.emplace_back(o2::fit::FITDigitWriterSpecHelper::getFITDigitWriterSpec(false, false, dataOrigin));
     }
diff --git a/Detectors/FIT/FV0/workflow/src/fv0-reco-workflow.cxx b/Detectors/FIT/FV0/workflow/src/fv0-reco-workflow.cxx
index 16d1383c7e8c4..309560e2d6b36 100644
--- a/Detectors/FIT/FV0/workflow/src/fv0-reco-workflow.cxx
+++ b/Detectors/FIT/FV0/workflow/src/fv0-reco-workflow.cxx
@@ -39,6 +39,7 @@ void customize(std::vector& workflowOptions)
     {"disable-mc", o2::framework::VariantType::Bool, false, {"disable MC propagation even if available"}},
     {"disable-root-input", o2::framework::VariantType::Bool, false, {"disable root-files input readers"}},
     {"disable-root-output", o2::framework::VariantType::Bool, false, {"disable root-files output writers"}},
+    {"disable-dead-channel-map", o2::framework::VariantType::Bool, false, {"disable dead channel map"}},
     {"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}}};
   o2::raw::HBFUtilsInitializer::addConfigOption(options);
   std::swap(workflowOptions, options);
@@ -59,9 +60,10 @@ WorkflowSpec defineDataProcessing(ConfigContext const& configcontext)
   auto useMC = !configcontext.options().get("disable-mc");
   auto disableRootInp = configcontext.options().get("disable-root-input");
   auto disableRootOut = configcontext.options().get("disable-root-output");
+  bool useDeadChannelMap = !configcontext.options().get("disable-dead-channel-map");
 
   LOG(info) << "WorkflowSpec getRecoWorkflow useMC " << useMC;
-  auto wf = o2::fv0::getRecoWorkflow(useMC, disableRootInp, disableRootOut);
+  auto wf = o2::fv0::getRecoWorkflow(useMC, disableRootInp, disableRootOut, useDeadChannelMap);
 
   // configure dpl timer to inject correct firstTForbit: start from the 1st orbit of TF containing 1st sampled orbit
   o2::raw::HBFUtilsInitializer hbfIni(configcontext, wf);
diff --git a/Detectors/FIT/FV0/workflow/src/recpoints-reader-workflow.cxx b/Detectors/FIT/FV0/workflow/src/recpoints-reader-workflow.cxx
new file mode 100644
index 0000000000000..ecbe89b8bbed3
--- /dev/null
+++ b/Detectors/FIT/FV0/workflow/src/recpoints-reader-workflow.cxx
@@ -0,0 +1,58 @@
+// Copyright 2019-2024 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file  recpoints-reader-workflow.cxx
+/// \brief FV0 RecPoints reader workflow
+///
+/// \author Andreas Molander andreas.molander@cern.ch
+
+#include "CommonUtils/ConfigurableParam.h"
+#include "DetectorsRaw/HBFUtilsInitializer.h"
+#include "Framework/CallbacksPolicy.h"
+#include "Framework/ConfigParamSpec.h"
+#include "Framework/Variant.h"
+
+#include "FV0Workflow/RecPointReaderSpec.h"
+
+#include 
+
+using namespace o2::framework;
+
+void customize(std::vector& policies)
+{
+  o2::raw::HBFUtilsInitializer::addNewTimeSliceCallback(policies);
+}
+
+// we need to add workflow options before including Framework/runDataProcessing
+void customize(std::vector& workflowOptions)
+{
+  std::vector options{
+    {"disable-mc", VariantType::Bool, false, {"disable MC propagation even if available"}},
+    {"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}}};
+  o2::raw::HBFUtilsInitializer::addConfigOption(options);
+  std::swap(workflowOptions, options);
+}
+
+#include "Framework/runDataProcessing.h"
+
+WorkflowSpec defineDataProcessing(const ConfigContext& ctx)
+{
+  o2::conf::ConfigurableParam::updateFromString(ctx.options().get("configKeyValues"));
+  bool disableMC = ctx.options().get("disable-mc");
+
+  WorkflowSpec specs;
+  DataProcessorSpec producer = o2::fv0::getRecPointReaderSpec(!disableMC);
+  specs.push_back(producer);
+
+  // configure dpl timer to inject correct firstTForbit: start from the 1st orbit of TF containing 1st sampled orbit
+  o2::raw::HBFUtilsInitializer hbfIni(ctx, specs);
+  return specs;
+}
diff --git a/Detectors/FIT/FV0/workflow/src/recpoints-writer-workflow.cxx b/Detectors/FIT/FV0/workflow/src/recpoints-writer-workflow.cxx
new file mode 100644
index 0000000000000..0fd3bd3bef2e8
--- /dev/null
+++ b/Detectors/FIT/FV0/workflow/src/recpoints-writer-workflow.cxx
@@ -0,0 +1,47 @@
+// Copyright 2019-2024 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file  recpoints-writer-workflow.cxx
+/// \brief FV0 RecPoints writer workflow
+///
+/// \author Andreas Molander andreas.molander@cern.ch
+
+#include "FV0Workflow/RecPointWriterSpec.h"
+
+#include "CommonUtils/ConfigurableParam.h"
+#include "Framework/ConfigParamSpec.h"
+#include "Framework/Variant.h"
+
+#include 
+
+using namespace o2::framework;
+
+// we need to add workflow options before including Framework/runDataProcessing
+void customize(std::vector& workflowOptions)
+{
+  std::vector options{
+    {"disable-mc", VariantType::Bool, false, {"disable MC propagation even if available"}},
+    {"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}}};
+  workflowOptions.insert(workflowOptions.end(), options.begin(), options.end());
+}
+
+#include "Framework/runDataProcessing.h"
+
+WorkflowSpec defineDataProcessing(const ConfigContext& ctx)
+{
+  o2::conf::ConfigurableParam::updateFromString(ctx.options().get("configKeyValues"));
+  bool disableMC = ctx.options().get("disable-mc");
+
+  WorkflowSpec specs;
+  DataProcessorSpec producer = o2::fv0::getRecPointWriterSpec(!disableMC);
+  specs.push_back(producer);
+  return specs;
+}
diff --git a/Detectors/FIT/README.md b/Detectors/FIT/README.md
index b079dce2cc972..8da3f32d7f4e2 100644
--- a/Detectors/FIT/README.md
+++ b/Detectors/FIT/README.md
@@ -9,6 +9,9 @@ This is a top page for the FIT detector documentation.
 
diff --git a/Detectors/FIT/common/CMakeLists.txt b/Detectors/FIT/common/CMakeLists.txt
index a1687d88a9a45..1066814a6cbaa 100644
--- a/Detectors/FIT/common/CMakeLists.txt
+++ b/Detectors/FIT/common/CMakeLists.txt
@@ -10,3 +10,7 @@
 # or submit itself to any jurisdiction.
 
 add_subdirectory(calibration)
+add_subdirectory(dcsmonitoring)
+if(BUILD_TESTING)
+    add_subdirectory(dcsmonitoring/macros)
+endif()
diff --git a/Detectors/FIT/common/calibration/include/FITCalibration/FITCalibrationDevice.h b/Detectors/FIT/common/calibration/include/FITCalibration/FITCalibrationDevice.h
index c732e2768396d..f6a4d2f6453de 100644
--- a/Detectors/FIT/common/calibration/include/FITCalibration/FITCalibrationDevice.h
+++ b/Detectors/FIT/common/calibration/include/FITCalibration/FITCalibrationDevice.h
@@ -30,25 +30,29 @@ namespace o2::fit
 template 
 class FITCalibrationDevice : public o2::framework::Task
 {
-  static constexpr const char* DEFAULT_INPUT_DATA_LABEL = "calib";
-  static constexpr const char* sDEFAULT_CCDB_URL = "http://localhost:8080";
-  using CalibratorType = FITCalibrator;
+  //  static constexpr const char* sDEFAULT_CCDB_URL = "http://localhost:8080";
+  static constexpr const char* sInputDataLabel = "calibData";
+  static constexpr const char* sOutputDataLabelCDBPayload = "cdbPayloadFIT";
+  static constexpr const char* sOutputDataLabelCDBWrapper = "cdbWrapperFIT";
+  static constexpr o2::header::DataDescription sOutputDataDescription = "FIT_CALIB";
+  using CalibratorType = FITCalibrator;
 
  public:
-  explicit FITCalibrationDevice(std::string inputDataLabel = DEFAULT_INPUT_DATA_LABEL, std::shared_ptr req = {})
-    : mInputDataLabel(std::move(inputDataLabel)), mCCDBRequest(req) {}
+  explicit FITCalibrationDevice(std::shared_ptr req = {}, const o2::header::DataDescription& dataDescription = sOutputDataDescription, const std::string& inputDataLabel = sInputDataLabel, const std::string& outputDataLabelCDBPayload = sOutputDataLabelCDBPayload, const std::string& outputDataLabelCDBWrapper = sOutputDataLabelCDBWrapper)
+    : mInputDataLabel(std::move(inputDataLabel)), mCCDBRequest(req), mOutputDataDescription(dataDescription), mOutputDataLabelCDBPayload(outputDataLabelCDBPayload), mOutputDataLabelCDBWrapper(outputDataLabelCDBWrapper) {}
+
   void init(o2::framework::InitContext& context) final
   {
     o2::base::GRPGeomHelper::instance().setRequest(mCCDBRequest);
     auto slotL = context.options().get("tf-per-slot");
     auto delay = context.options().get("max-delay");
-
+    const std::string extraInfo = context.options().get("extra-info-per-slot");
     mCalibrator = std::make_unique();
 
     mCalibrator->setSlotLength(slotL);
     mCalibrator->setMaxSlotsDelay(delay);
-
-    o2::ccdb::BasicCCDBManager::instance().setURL(sDEFAULT_CCDB_URL);
+    mCalibrator->setExtraInfo(extraInfo);
+    //    o2::ccdb::BasicCCDBManager::instance().setURL(sDEFAULT_CCDB_URL);
   }
 
   void run(o2::framework::ProcessingContext& context) final
@@ -73,6 +77,16 @@ class FITCalibrationDevice : public o2::framework::Task
   {
     o2::base::GRPGeomHelper::instance().finaliseCCDB(matcher, obj);
   }
+  void static prepareVecOutputSpec(std::vector& outputs, o2::header::DataDescription dataDescription)
+  {
+    outputs.emplace_back(o2::framework::ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBPayload, dataDescription}, o2::framework::Lifetime::Sporadic);
+    outputs.emplace_back(o2::framework::ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBWrapper, dataDescription}, o2::framework::Lifetime::Sporadic);
+  }
+
+  void static prepareVecInputSpec(std::vector& inputs, o2::header::DataOrigin dataOrigin, o2::header::DataDescription dataDescription)
+  {
+    inputs.emplace_back(sInputDataLabel, dataOrigin, dataDescription, o2::framework::Lifetime::Sporadic);
+  }
 
  private:
   void _sendCalibrationObjectIfSlotFinalized(o2::framework::DataAllocator& outputs)
@@ -89,8 +103,8 @@ class FITCalibrationDevice : public o2::framework::Task
 
     uint32_t iSendChannel = 0;
     for (const auto& [ccdbInfo, calibObject] : objectsToSend) {
-      outputs.snapshot(o2::framework::Output{clbUtils::gDataOriginCDBPayload, "FIT_CALIB", iSendChannel}, *calibObject);
-      outputs.snapshot(o2::framework::Output{clbUtils::gDataOriginCDBWrapper, "FIT_CALIB", iSendChannel}, ccdbInfo);
+      outputs.snapshot(o2::framework::Output{clbUtils::gDataOriginCDBPayload, mOutputDataDescription, iSendChannel}, *calibObject);
+      outputs.snapshot(o2::framework::Output{clbUtils::gDataOriginCDBWrapper, mOutputDataDescription, iSendChannel}, ccdbInfo);
       LOG(info) << "_sendOutputs " << ccdbInfo.getStartValidityTimestamp();
       ++iSendChannel;
     }
@@ -98,6 +112,9 @@ class FITCalibrationDevice : public o2::framework::Task
   }
 
   const std::string mInputDataLabel;
+  const std::string mOutputDataLabelCDBPayload;
+  const std::string mOutputDataLabelCDBWrapper;
+  const o2::header::DataDescription mOutputDataDescription;
   std::unique_ptr mCalibrator;
   std::shared_ptr mCCDBRequest;
 };
diff --git a/Detectors/FIT/common/calibration/include/FITCalibration/FITCalibrator.h b/Detectors/FIT/common/calibration/include/FITCalibration/FITCalibrator.h
index d817d26a1ae1a..dc70d6a521a77 100644
--- a/Detectors/FIT/common/calibration/include/FITCalibration/FITCalibrator.h
+++ b/Detectors/FIT/common/calibration/include/FITCalibration/FITCalibrator.h
@@ -27,8 +27,8 @@
 namespace o2::fit
 {
 
-template 
-class FITCalibrator final : public o2::calibration::TimeSlotCalibration
+template 
+class FITCalibrator final : public o2::calibration::TimeSlotCalibration
 {
 
   // probably will be set via run parameter
@@ -63,13 +63,12 @@ class FITCalibrator final : public o2::calibration::TimeSlotCalibration md;
     auto* container = slot.getContainer();
-    static const double TFlength = 1E-3 * o2::raw::HBFUtils::Instance().getNOrbitsPerTF() * o2::constants::lhc::LHCOrbitMUS; // in ms
-    auto starting = slot.getStartTimeMS();
-    auto stopping = slot.getEndTimeMS();
-    LOGP(info, "!!!! {}({})<=TF<={}({}), starting: {} stopping {}", slot.getTFStart(), slot.getStartTimeMS(), slot.getTFEnd(), slot.getEndTimeMS(), starting, stopping);
-    auto calibrationObject = container->generateCalibrationObject();
+    const auto startValidity = slot.getStartTimeMS() - o2::ccdb::CcdbObjectInfo::SECOND * 10;
+    const auto endValidity = slot.getEndTimeMS() + o2::ccdb::CcdbObjectInfo::MONTH;
+    LOGP(info, "!!!! {}<=TF<={}, startValidity: {} endValidity: {}", slot.getTFStart(), slot.getTFEnd(), startValidity, endValidity);
+    auto calibrationObject = container->generateCalibrationObject(startValidity, endValidity, mExtraInfo);
     std::vector preparedCalibObjects;
-    preparedCalibObjects.emplace_back(doSerializationAndPrepareObjectInfo(calibrationObject, starting, stopping));
+    preparedCalibObjects.emplace_back(doSerializationAndPrepareObjectInfo(calibrationObject, startValidity, endValidity));
     mStoredCalibrationObjects.insert(mStoredCalibrationObjects.end(),
                                      std::make_move_iterator(preparedCalibObjects.begin()),
                                      std::make_move_iterator(preparedCalibObjects.end()));
@@ -80,19 +79,18 @@ class FITCalibrator final : public o2::calibration::TimeSlotCalibration::getSlots();
+    auto& cont = o2::calibration::TimeSlotCalibration::getSlots();
     auto& slot = front ? cont.emplace_front(tstart, tend) : cont.emplace_back(tstart, tend);
     slot.setContainer(std::make_unique(mMinEntries));
     return slot;
   }
 
-  CalibObjWithInfoType doSerializationAndPrepareObjectInfo(const CalibrationObjectType& calibrationObject, TFType starting, TFType stopping)
+  CalibObjWithInfoType doSerializationAndPrepareObjectInfo(const CalibrationObjectType& calibrationObject, long starting, long stopping)
   {
     std::map metaData;
     CalibObjWithInfoType result;
     auto clName = o2::utils::MemFileHelper::getClassName(calibrationObject);
     auto flName = o2::ccdb::CcdbApi::generateFileName(clName);
-    stopping = stopping + 86400000; // +1 day
     LOG(info) << " clName " << clName << " flName " << flName;
     result.first = o2::ccdb::CcdbObjectInfo(CalibrationObjectType::getObjectPath(), clName, flName, metaData, starting, stopping);
     result.second = o2::ccdb::CcdbApi::createObjectImage(&calibrationObject, &result.first);
@@ -100,10 +98,15 @@ class FITCalibrator final : public o2::calibration::TimeSlotCalibration mStoredCalibrationObjects{};
   const unsigned int mMinEntries;
+  std::string mExtraInfo;
 };
 
 } // namespace o2::fit
diff --git a/Detectors/FIT/common/dcsmonitoring/CMakeLists.txt b/Detectors/FIT/common/dcsmonitoring/CMakeLists.txt
new file mode 100644
index 0000000000000..7bc0820e77a00
--- /dev/null
+++ b/Detectors/FIT/common/dcsmonitoring/CMakeLists.txt
@@ -0,0 +1,23 @@
+# Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+# See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+# All rights not expressly granted are reserved.
+#
+# This software is distributed under the terms of the GNU General Public
+# License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+#
+# In applying this license CERN does not waive the privileges and immunities
+# granted to it by virtue of its status as an Intergovernmental Organization
+# or submit itself to any jurisdiction.
+
+o2_add_library(FITDCSMonitoring
+               SOURCES src/FITDCSConfigReader.cxx
+                       src/FITDCSDataProcessor.cxx
+                       src/FITDCSDataReader.cxx
+               PUBLIC_LINK_LIBRARIES O2::DataFormatsFIT
+                                     O2::DetectorsCalibration
+                                     O2::DetectorsDCS)
+
+# TODO AM: remove include/FITDCSMonitoring/FITDCSDataReader.h when dictionaries are created in DataFormatsFIT instead
+o2_target_root_dictionary(FITDCSMonitoring
+                          HEADERS include/FITDCSMonitoring/FITDCSConfigReader.h
+                                  include/FITDCSMonitoring/FITDCSDataReader.h)
diff --git a/Detectors/FIT/common/dcsmonitoring/include/FITDCSMonitoring/FITDCSConfigProcessorSpec.h b/Detectors/FIT/common/dcsmonitoring/include/FITDCSMonitoring/FITDCSConfigProcessorSpec.h
new file mode 100644
index 0000000000000..18c0b593b0a02
--- /dev/null
+++ b/Detectors/FIT/common/dcsmonitoring/include/FITDCSMonitoring/FITDCSConfigProcessorSpec.h
@@ -0,0 +1,138 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file FITDCSConfigProcessorSpec.cxx
+/// \brief FIT processor spec for DCS configurations
+///
+/// \author Andreas Molander , University of Jyvaskyla, Finland
+
+#ifndef O2_FIT_DCSCONFIGPROCESSORSPEC_H
+#define O2_FIT_DCSCONFIGPROCESSORSPEC_H
+
+#include "CCDB/CcdbApi.h"
+#include "DetectorsCalibration/Utils.h"
+#include "FITDCSMonitoring/FITDCSConfigReader.h"
+#include "Framework/ConfigParamRegistry.h"
+#include "Framework/Task.h"
+#include "Framework/WorkflowSpec.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+using namespace o2::framework;
+
+namespace o2
+{
+namespace fit
+{
+
+class FITDCSConfigProcessor : public o2::framework::Task
+{
+ public:
+  FITDCSConfigProcessor(const std::string& detectorName, const o2::header::DataDescription& dataDescriptionDChM)
+    : mDetectorName(detectorName),
+      mDataDescriptionDChM(dataDescriptionDChM) {} // TODO AM: how to pass dd
+
+  void init(o2::framework::InitContext& ic) final
+  {
+    initDCSConfigReader();
+    mDCSConfigReader->setFileNameDChM(ic.options().get("filename-dchm"));
+    mDCSConfigReader->setValidDaysDChM(ic.options().get("valid-days-dchm"));
+    mDCSConfigReader->setCcdbPathDChM(mDetectorName + "/Calib/DeadChannelMap");
+    mVerbose = ic.options().get("use-verbose-mode");
+    mDCSConfigReader->setVerboseMode(mVerbose);
+    mValidateUpload = !ic.options().get("no-validate");
+    mDCSConfigReader->setValidateUploadMode(mValidateUpload);
+
+    LOG(info) << "Verbose mode: " << mVerbose;
+    LOG(info) << "Validate upload: " << mValidateUpload;
+    LOG(info) << "Expected dead channel map file name: " << mDCSConfigReader->getFileNameDChM();
+    LOG(info) << "Dead channel maps will be valid for " << mDCSConfigReader->getValidDaysDChM() << " days";
+  }
+
+  void run(o2::framework::ProcessingContext& pc) final
+  {
+    // Get the time of the data
+    auto timeNow = std::chrono::high_resolution_clock::now();
+    long dataTime = (long)(pc.services().get().creation);
+    if (dataTime == 0xffffffffffffffff) {                                                                   // means it is not set
+      dataTime = std::chrono::duration_cast(timeNow.time_since_epoch()).count(); // in ms
+    }
+
+    // Get the input file
+    gsl::span configBuf = pc.inputs().get>("inputConfig");
+    std::string configFileName = pc.inputs().get("inputConfigFileName");
+    LOG(info) << "Got input file " << configFileName << " of size " << configBuf.size();
+
+    if (!configFileName.compare(mDCSConfigReader->getFileNameDChM())) {
+      // Got dead channel map
+      processDChM(dataTime, configBuf);
+      sendDChMOutput(pc.outputs());
+      mDCSConfigReader->resetStartValidityDChM();
+      mDCSConfigReader->resetDChM();
+    } else {
+      LOG(error) << "Unknown input file: " << configFileName;
+    }
+  }
+
+  void endOfStream(o2::framework::EndOfStreamContext& ec) final
+  {
+  }
+
+ protected:
+  /// Initializes the DCS config reader.
+  /// Can be overriden in case another reader (subclass of o2::fit::FITDCSConfigReader) is needed.
+  virtual void initDCSConfigReader()
+  {
+    mDCSConfigReader = std::make_unique(FITDCSConfigReader());
+  }
+
+  std::unique_ptr mDCSConfigReader; ///< Reader for the DCS configurations
+
+ private:
+  /// Processing the dead channel map
+  void processDChM(const long& dataTime, gsl::span configBuf)
+  {
+    if (!mDCSConfigReader->isStartValidityDChMSet()) {
+      if (mVerbose) {
+        LOG(info) << "Start validity for DCS data set to = " << dataTime;
+      }
+      mDCSConfigReader->setStartValidityDChM(dataTime);
+    }
+    mDCSConfigReader->processDChM(configBuf);
+    mDCSConfigReader->updateDChMCcdbObjectInfo();
+  }
+
+  /// Sending the dead channeel map output to CCDB
+  void sendDChMOutput(o2::framework::DataAllocator& output)
+  {
+    const auto& payload = mDCSConfigReader->getDChM();
+    auto& info = mDCSConfigReader->getObjectInfoDChM();
+    auto image = o2::ccdb::CcdbApi::createObjectImage(&payload, &info);
+    LOG(info) << "Sending object " << info.getPath() << "/" << info.getFileName() << " of size " << image->size()
+              << " bytes, valid for " << info.getStartValidityTimestamp() << " : " << info.getEndValidityTimestamp();
+    output.snapshot(Output{o2::calibration::Utils::gDataOriginCDBPayload, mDataDescriptionDChM, 0}, *image.get());
+    output.snapshot(Output{o2::calibration::Utils::gDataOriginCDBWrapper, mDataDescriptionDChM, 0}, info);
+  }
+
+  std::string mDetectorName;                        ///< Detector name
+  o2::header::DataDescription mDataDescriptionDChM; ///< DataDescription for the dead channel map
+  bool mVerbose = false;                            ///< Verbose mode
+  bool mValidateUpload = true;                      ///< Validate upload
+};
+
+} // namespace fit
+} // namespace o2
+
+#endif // O2_FIT_DCSCONFIGPROCESSORSPEC_H
diff --git a/Detectors/FIT/common/dcsmonitoring/include/FITDCSMonitoring/FITDCSConfigReader.h b/Detectors/FIT/common/dcsmonitoring/include/FITDCSMonitoring/FITDCSConfigReader.h
new file mode 100644
index 0000000000000..c43607e091493
--- /dev/null
+++ b/Detectors/FIT/common/dcsmonitoring/include/FITDCSMonitoring/FITDCSConfigReader.h
@@ -0,0 +1,82 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file FITDCSConfigReader.h
+/// \brief DCS configuration reader for FIT
+///
+/// \author Andreas Molander , University of Jyvaskyla, Finland
+
+#ifndef O2_FIT_DCSCONFIGREADER_H
+#define O2_FIT_DCSCONFIGREADER_H
+
+#include "CCDB/CcdbObjectInfo.h"
+#include "DataFormatsFIT/DeadChannelMap.h"
+
+#include 
+#include 
+
+namespace o2
+{
+namespace fit
+{
+
+class FITDCSConfigReader
+{
+ public:
+  FITDCSConfigReader() = default;
+  ~FITDCSConfigReader() = default;
+
+  virtual void processDChM(gsl::span configBuf);
+  void updateDChMCcdbObjectInfo();
+
+  const o2::fit::DeadChannelMap& getDChM() const;
+  void resetDChM();
+  const std::string& getCcdbPathDChm() const;
+  void setCcdbPathDChM(const std::string& ccdbPath);
+  const long getStartValidityDChM() const;
+  const long getEndValidityDChM() const;
+  void setStartValidityDChM(const long startValidity);
+  const bool isStartValidityDChMSet() const;
+  void resetStartValidityDChM();
+  const o2::ccdb::CcdbObjectInfo& getObjectInfoDChM() const;
+  o2::ccdb::CcdbObjectInfo& getObjectInfoDChM();
+
+  const std::string& getFileNameDChM() const;
+  void setFileNameDChM(const std::string& fileName);
+
+  const uint getValidDaysDChM() const;
+  void setValidDaysDChM(const uint validDays);
+
+  const bool getVerboseMode() const;
+  void setVerboseMode(const bool verboseMode);
+
+  const bool getValidateUploadMode() const;
+  void setValidateUploadMode(const bool validateUpload);
+
+ protected:
+  o2::fit::DeadChannelMap mDChM; ///< The dead channel map CCDB object
+  bool mVerbose = false;         ///< Verbose mode
+
+ private:
+  std::string mFileNameDChM;                                              ///< The expected file name of the dead channel map
+  uint mValidDaysDChM = 180u;                                             ///< The dead channel map validity in days
+  std::string mCcdbPathDChM;                                              ///< The dead channel map CCDB path
+  long mStartValidityDChM = o2::ccdb::CcdbObjectInfo::INFINITE_TIMESTAMP; ///< Start validity of the dead channel map CCDB object
+  o2::ccdb::CcdbObjectInfo mCcdbObjectInfoDChM;                           ///< CCDB object info for the dead channel map
+  bool mValidateUpload = true;                                            ///< Validate upload mode
+
+  ClassDefNV(FITDCSConfigReader, 1);
+};
+
+} // namespace fit
+} // namespace o2
+
+#endif // O2_FIT_DCSCONFIGREADER_H
\ No newline at end of file
diff --git a/Detectors/FIT/common/dcsmonitoring/include/FITDCSMonitoring/FITDCSDataProcessor.h b/Detectors/FIT/common/dcsmonitoring/include/FITDCSMonitoring/FITDCSDataProcessor.h
new file mode 100644
index 0000000000000..5fa5ab2342773
--- /dev/null
+++ b/Detectors/FIT/common/dcsmonitoring/include/FITDCSMonitoring/FITDCSDataProcessor.h
@@ -0,0 +1,77 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// @file  FITDCSDataProcessor.h
+/// @brief Task for processing FIT DCS data
+///
+/// \author Andreas Molander , University of Jyvaskyla, Finland
+
+#ifndef O2_FIT_DCSDATAPROCESSOR_H
+#define O2_FIT_DCSDATAPROCESSOR_H
+
+#include "DetectorsDCS/DataPointIdentifier.h"
+#include "FITDCSMonitoring/FITDCSDataReader.h"
+#include "Framework/DataAllocator.h"
+#include "Framework/Task.h"
+#include "Headers/DataHeader.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+namespace o2
+{
+namespace fit
+{
+
+/// Virtual task for processing FIT DCS data
+///
+/// Virtual task for processing FIT DCS data. Each subdetector implements
+/// a sub-class with detector specific functionality, e.g. the hard coded
+/// DP IDs.
+class FITDCSDataProcessor : public o2::framework::Task
+{
+ public:
+  FITDCSDataProcessor(const std::string& detectorName, const o2::header::DataDescription& dataDescription)
+    : mDetectorName(detectorName),
+      mDataDescription(dataDescription) {}
+  void init(o2::framework::InitContext& ic) final;
+  void run(o2::framework::ProcessingContext& pc) final;
+  void endOfStream(o2::framework::EndOfStreamContext& ec) final;
+  const std::string& getDetectorName() const;
+  bool getVerboseMode() const;
+  void setVerboseMode(bool verboseMode = true);
+
+ protected:
+  /// Gets the DP IDs. They are hard coded in the function implementations
+  /// as an alternative to fetch them from CCDB.
+  virtual std::vector getHardCodedDPIDs() = 0;
+
+  std::string mDetectorName; ///< Detector name
+  bool mVerbose = false;     ///< Verbose mode
+
+ private:
+  /// Send the DP output
+  void sendDPsOutput(o2::framework::DataAllocator& output);
+
+  std::unique_ptr mDataReader;
+  std::chrono::high_resolution_clock::time_point mTimer;
+  int64_t mDPsUpdateInterval;
+  o2::header::DataDescription mDataDescription; ///< DataDescription for the DCS DPs
+
+}; // end class
+
+} // namespace fit
+} // namespace o2
+
+#endif // O2_FIT_DCSDATAPROCESSOR_H
diff --git a/Detectors/FIT/common/dcsmonitoring/include/FITDCSMonitoring/FITDCSDataReader.h b/Detectors/FIT/common/dcsmonitoring/include/FITDCSMonitoring/FITDCSDataReader.h
new file mode 100644
index 0000000000000..3a793466fc8a5
--- /dev/null
+++ b/Detectors/FIT/common/dcsmonitoring/include/FITDCSMonitoring/FITDCSDataReader.h
@@ -0,0 +1,94 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file FITDCSDataReader.h
+/// \brief DCS data point reader for FIT
+///
+/// \author Andreas Molander , University of Jyvaskyla, Finland
+
+#ifndef O2_FIT_DCSDATAREADER_H
+#define O2_FIT_DCSDATAREADER_H
+
+#include "CCDB/CcdbObjectInfo.h"
+#include "DataFormatsFIT/DCSDPValues.h"
+#include "DetectorsDCS/DataPointCompositeObject.h"
+#include "DetectorsDCS/DataPointIdentifier.h"
+#include "DetectorsDCS/DataPointValue.h"
+#include "Rtypes.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+namespace o2
+{
+namespace fit
+{
+
+class FITDCSDataReader
+{
+ public:
+  using DPID = o2::dcs::DataPointIdentifier;
+  using DPVAL = o2::dcs::DataPointValue;
+  using DPCOM = o2::dcs::DataPointCompositeObject;
+  using CcdbObjectInfo = o2::ccdb::CcdbObjectInfo;
+
+  FITDCSDataReader() = default;
+  ~FITDCSDataReader() = default;
+
+  void init(const std::vector& pids);
+  int process(const gsl::span dps);
+  int processDP(const DPCOM& dpcom);
+  uint64_t processFlags(uint64_t flag, const char* pid);
+  void updateCcdbObjectInfo();
+
+  const std::unordered_map& getDpData() const;
+  void resetDpData();
+  const std::string& getCcdbPath() const;
+  void setCcdbPath(const std::string& ccdbPath);
+  long getStartValidity() const;
+  void setStartValidity(long startValidity);
+  bool isStartValiditySet() const;
+  void resetStartValidity();
+  long getEndValidity() const;
+  const CcdbObjectInfo& getccdbDPsInfo() const;
+  CcdbObjectInfo& getccdbDPsInfo();
+
+  bool getVerboseMode() const;
+  void setVerboseMode(bool verboseMode = true);
+
+ private:
+  std::unordered_map mDpData; // the object that will go to the CCDB
+  std::unordered_map mPids;                   // contains all PIDs for the processor, the bool
+                                                          // will be true if the DP was processed at least once
+  std::unordered_map mDpsMap;                // this is the map that will hold the DPs
+
+  std::string mCcdbPath;
+  long mStartValidity = o2::ccdb::CcdbObjectInfo::INFINITE_TIMESTAMP; // TF index for processing, used to store CCDB object for DPs
+  CcdbObjectInfo mCcdbDpInfo;
+
+  union DPValueConverter {
+    uint64_t raw_data;
+    double double_value;
+    uint uint_value;
+  } dpValueConverter;
+
+  bool mVerbose = false;
+
+  ClassDefNV(FITDCSDataReader, 0);
+}; // end class
+
+} // namespace fit
+} // namespace o2
+
+#endif // O2_FIT_DCSDATAREADER_H
diff --git a/Detectors/FIT/common/dcsmonitoring/macros/CMakeLists.txt b/Detectors/FIT/common/dcsmonitoring/macros/CMakeLists.txt
new file mode 100644
index 0000000000000..173e91011df2c
--- /dev/null
+++ b/Detectors/FIT/common/dcsmonitoring/macros/CMakeLists.txt
@@ -0,0 +1,16 @@
+# Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+# See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+# All rights not expressly granted are reserved.
+#
+# This software is distributed under the terms of the GNU General Public
+# License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+#
+# In applying this license CERN does not waive the privileges and immunities
+# granted to it by virtue of its status as an Intergovernmental Organization
+# or submit itself to any jurisdiction.
+
+o2_add_test_root_macro(makeDefaultDeadChannelMap.C
+                       PUBLIC_LINK_LIBRARIES O2::CCDB
+                                             O2::DataFormatsFIT
+                                             O2::Framework
+                       LABELS fit)
diff --git a/Detectors/FIT/common/dcsmonitoring/macros/makeDefaultDeadChannelMap.C b/Detectors/FIT/common/dcsmonitoring/macros/makeDefaultDeadChannelMap.C
new file mode 100644
index 0000000000000..d5311df91c354
--- /dev/null
+++ b/Detectors/FIT/common/dcsmonitoring/macros/makeDefaultDeadChannelMap.C
@@ -0,0 +1,77 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file makeDefaultDeadChannelMap.C
+/// \brief Macro for uploading default dead channel maps to CCDB
+///
+/// \author Andreas Molander , University of Jyvaskyla, Finland
+
+#if !defined(__CLING__) || defined(__ROOTCLING__)
+
+#include "CCDB/CcdbApi.h"
+#include "DataFormatsFIT/DeadChannelMap.h"
+#include "TFile.h"
+
+#include 
+#include 
+#include 
+
+#endif
+
+#include "Framework/Logger.h"
+
+#include 
+
+void makeDefaultDeadChannelMap(std::string detectorName,
+                               const std::string ccdbUrl = "http://localhost:8080",
+                               const std::string fileName = "")
+{
+  boost::to_upper(detectorName);
+
+  int nChannels;
+
+  if (detectorName == "FT0") {
+    nChannels = 212;
+  } else if (detectorName == "FV0") {
+    nChannels = 49;
+  } else if (detectorName == "FDD") {
+    nChannels = 19;
+  } else {
+    LOGP(fatal, "Invalid detector name provided: '{}'. Please use [FT0/FV0/FDD].", detectorName);
+    return;
+  }
+
+  LOGP(info, "Creating default dead channel map for {}.", detectorName);
+
+  o2::fit::DeadChannelMap deadChannelMap;
+
+  for (int iChannel = 0; iChannel < nChannels; iChannel++) {
+    deadChannelMap.setChannelAlive(iChannel, true);
+  }
+
+  if (!ccdbUrl.empty()) {
+    const std::string ccdbPath = detectorName + "/Calib/DeadChannelMap";
+    std::map metadata;
+    metadata["default"] = "true";
+    metadata["comment"] = "Default dead channel map, all channels are alive.";
+    LOGP(info, "Storing default dead channel map on {}/{}.", ccdbUrl, ccdbPath);
+    o2::ccdb::CcdbApi api;
+    api.init(ccdbUrl);
+    api.storeAsTFileAny(&deadChannelMap, ccdbPath, metadata, 1, 99999999999999);
+  }
+
+  if (!fileName.empty()) {
+    LOGP(info, "Storing default dead channel map locally in {}.", fileName);
+    std::unique_ptr file(TFile::Open(fileName.c_str(), "RECREATE"));
+    file->WriteObject(&deadChannelMap, "deadChannelMap");
+  }
+  return;
+}
diff --git a/Detectors/FIT/common/dcsmonitoring/src/FITDCSConfigReader.cxx b/Detectors/FIT/common/dcsmonitoring/src/FITDCSConfigReader.cxx
new file mode 100644
index 0000000000000..3540d3a2f33c0
--- /dev/null
+++ b/Detectors/FIT/common/dcsmonitoring/src/FITDCSConfigReader.cxx
@@ -0,0 +1,75 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file FITDCSConfigReader.cxx
+/// \brief FIT reader for DCS configurations
+///
+/// \author Andreas Molander , University of Jyvaskyla, Finland
+
+#include "FITDCSMonitoring/FITDCSConfigReader.h"
+
+#include "DetectorsCalibration/Utils.h"
+
+#include 
+#include 
+
+using namespace o2::fit;
+
+void FITDCSConfigReader::processDChM(gsl::span configBuf)
+{
+  LOG(info) << "Processing dead channel map";
+
+  // AM: need to specify the size,
+  // otherwise the configBuf.data() pointer might point to an array that is too long,
+  // the array sometimes includes "extra memory from previous runs"
+  std::istringstream iss(std::string(configBuf.data(), configBuf.size()));
+  uint8_t iLine = 0; // line 0 corresponds to cahnnel id 0 and so on
+
+  for (std::string line; std::getline(iss, line);) {
+    mDChM.setChannelAlive(iLine, std::stoi(line) == 1);
+    iLine++;
+  }
+
+  if (getVerboseMode()) {
+    LOGP(info, "Processed {} channels", iLine);
+  }
+}
+
+void FITDCSConfigReader::updateDChMCcdbObjectInfo()
+{
+  std::map metadata;
+  o2::calibration::Utils::prepareCCDBobjectInfo(mDChM, mCcdbObjectInfoDChM, mCcdbPathDChM, metadata, getStartValidityDChM(), getEndValidityDChM());
+  mCcdbObjectInfoDChM.setValidateUpload(getValidateUploadMode());
+}
+
+const o2::fit::DeadChannelMap& FITDCSConfigReader::getDChM() const { return mDChM; }
+void FITDCSConfigReader::resetDChM() { mDChM.clear(); }
+const std::string& FITDCSConfigReader::getCcdbPathDChm() const { return mCcdbPathDChM; }
+void FITDCSConfigReader::setCcdbPathDChM(const std::string& ccdbPath) { mCcdbPathDChM = ccdbPath; }
+const long FITDCSConfigReader::getStartValidityDChM() const { return mStartValidityDChM; }
+const long FITDCSConfigReader::getEndValidityDChM() const { return mStartValidityDChM + getValidDaysDChM() * o2::ccdb::CcdbObjectInfo::DAY; }
+void FITDCSConfigReader::setStartValidityDChM(const long startValidity) { mStartValidityDChM = startValidity; }
+const bool FITDCSConfigReader::isStartValidityDChMSet() const { return mStartValidityDChM != o2::ccdb::CcdbObjectInfo::INFINITE_TIMESTAMP; }
+void FITDCSConfigReader::resetStartValidityDChM() { mStartValidityDChM = o2::ccdb::CcdbObjectInfo::INFINITE_TIMESTAMP; }
+const o2::ccdb::CcdbObjectInfo& FITDCSConfigReader::getObjectInfoDChM() const { return mCcdbObjectInfoDChM; }
+o2::ccdb::CcdbObjectInfo& FITDCSConfigReader::getObjectInfoDChM() { return mCcdbObjectInfoDChM; }
+
+const std::string& FITDCSConfigReader::getFileNameDChM() const { return mFileNameDChM; }
+void FITDCSConfigReader::setFileNameDChM(const std::string& fileName) { mFileNameDChM = fileName; }
+
+const uint FITDCSConfigReader::getValidDaysDChM() const { return mValidDaysDChM; }
+void FITDCSConfigReader::setValidDaysDChM(const uint validDays) { mValidDaysDChM = validDays; }
+
+const bool FITDCSConfigReader::getVerboseMode() const { return mVerbose; }
+void FITDCSConfigReader::setVerboseMode(const bool verboseMode) { mVerbose = verboseMode; }
+
+const bool FITDCSConfigReader::getValidateUploadMode() const { return mValidateUpload; }
+void FITDCSConfigReader::setValidateUploadMode(const bool validateUpload) { mValidateUpload = validateUpload; };
\ No newline at end of file
diff --git a/Detectors/FIT/common/dcsmonitoring/src/FITDCSDataProcessor.cxx b/Detectors/FIT/common/dcsmonitoring/src/FITDCSDataProcessor.cxx
new file mode 100644
index 0000000000000..9c32172fb8b49
--- /dev/null
+++ b/Detectors/FIT/common/dcsmonitoring/src/FITDCSDataProcessor.cxx
@@ -0,0 +1,132 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// @file  FITDCSDataProcessor.cxx
+/// @brief Task for processing FIT DCS data
+///
+/// \author Andreas Molander , University of Jyvaskyla, Finland
+
+#include "FITDCSMonitoring/FITDCSDataProcessor.h"
+
+#include "CCDB/BasicCCDBManager.h"
+#include "CCDB/CcdbApi.h"
+#include "DetectorsCalibration/Utils.h"
+#include "DetectorsDCS/DataPointCompositeObject.h"
+#include "DetectorsDCS/DataPointIdentifier.h"
+#include "FITDCSMonitoring/FITDCSDataReader.h"
+#include "Framework/ConfigParamRegistry.h"
+#include "Framework/DataAllocator.h"
+#include "Framework/Output.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+using namespace o2::fit;
+
+using DPCOM = o2::dcs::DataPointCompositeObject;
+using DPID = o2::dcs::DataPointIdentifier;
+using HighResClock = std::chrono::high_resolution_clock;
+using Duration = std::chrono::duration>;
+
+void FITDCSDataProcessor::init(o2::framework::InitContext& ic)
+{
+  setVerboseMode(ic.options().get("use-verbose-mode"));
+  LOG(info) << "Verbose mode: " << getVerboseMode();
+
+  mDPsUpdateInterval = ic.options().get("DPs-update-interval");
+  if (mDPsUpdateInterval == 0) {
+    LOG(error) << mDetectorName << " DPs update interval set to zero seconds --> changed to 10 min.";
+    mDPsUpdateInterval = 600;
+  }
+
+  std::vector vect;
+
+  const bool useCcdbToConfigure = ic.options().get("use-ccdb-to-configure");
+  if (useCcdbToConfigure) {
+    LOG(info) << "Configuring via CCDB";
+    const std::string ccdbPath = ic.options().get("ccdb-path");
+    auto& mgr = o2::ccdb::BasicCCDBManager::instance();
+    mgr.setURL(ccdbPath);
+    long timestamp = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count();
+    std::unordered_map* dpid2DataDesc = mgr.getForTimeStamp>(mDetectorName + "/Config/DCSDPconfig", timestamp);
+    for (auto& i : *dpid2DataDesc) {
+      vect.push_back(i.first);
+    }
+  } else {
+    LOG(info) << "Configuring via hardcoded strings";
+    vect = getHardCodedDPIDs();
+  }
+
+  if (getVerboseMode()) {
+    LOGP(info, "Listing Data Points for {}:", mDetectorName);
+    for (auto& i : vect) {
+      LOG(info) << i;
+    }
+  }
+
+  mDataReader = std::make_unique();
+  mDataReader->setVerboseMode(getVerboseMode());
+  mDataReader->setCcdbPath(mDetectorName + "/Calib/DCSDPs");
+  mDataReader->init(vect);
+  mTimer = HighResClock::now();
+}
+
+void FITDCSDataProcessor::run(o2::framework::ProcessingContext& pc)
+{
+  auto timeNow = HighResClock::now();
+  long dataTime = (long)(pc.services().get().creation);
+
+  if (dataTime == 0xffffffffffffffff) {                                                                   // it means it is not set
+    dataTime = std::chrono::duration_cast(timeNow.time_since_epoch()).count(); // in ms
+  }
+
+  if (!mDataReader->isStartValiditySet()) {
+    if (getVerboseMode()) {
+      LOG(info) << "Start valitidy for DPs changed to = " << dataTime;
+    }
+    mDataReader->setStartValidity(dataTime);
+  }
+  auto dps = pc.inputs().get>("input");
+  mDataReader->process(dps);
+  Duration elapsedTime = timeNow - mTimer; // in seconds
+  if (elapsedTime.count() >= mDPsUpdateInterval) {
+    sendDPsOutput(pc.outputs());
+    mTimer = timeNow;
+  }
+}
+
+void FITDCSDataProcessor::endOfStream(o2::framework::EndOfStreamContext& ec)
+{
+  sendDPsOutput(ec.outputs());
+}
+
+const std::string& FITDCSDataProcessor::getDetectorName() const { return mDetectorName; }
+bool FITDCSDataProcessor::getVerboseMode() const { return mVerbose; }
+void FITDCSDataProcessor::setVerboseMode(bool verboseMode) { mVerbose = verboseMode; }
+
+void FITDCSDataProcessor::sendDPsOutput(o2::framework::DataAllocator& output)
+{
+  // extract CCDB infos and calibration object for DPs
+  mDataReader->updateCcdbObjectInfo();
+  const auto& payload = mDataReader->getDpData();
+  auto& info = mDataReader->getccdbDPsInfo();
+  auto image = o2::ccdb::CcdbApi::createObjectImage(&payload, &info);
+  LOG(info) << "Sending object " << info.getPath() << "/" << info.getFileName() << " of size " << image->size()
+            << " bytes, valid for " << info.getStartValidityTimestamp() << " : " << info.getEndValidityTimestamp();
+  output.snapshot(o2::framework::Output{o2::calibration::Utils::gDataOriginCDBPayload, mDataDescription, 0}, *image.get());
+  output.snapshot(o2::framework::Output{o2::calibration::Utils::gDataOriginCDBWrapper, mDataDescription, 0}, info);
+  mDataReader->resetDpData();
+  mDataReader->resetStartValidity();
+}
diff --git a/Detectors/FIT/common/dcsmonitoring/src/FITDCSDataReader.cxx b/Detectors/FIT/common/dcsmonitoring/src/FITDCSDataReader.cxx
new file mode 100644
index 0000000000000..e1cc17a775a6a
--- /dev/null
+++ b/Detectors/FIT/common/dcsmonitoring/src/FITDCSDataReader.cxx
@@ -0,0 +1,209 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file FITDCSDataReader.cxx
+/// \brief DCS data point reader for FIT
+///
+/// \author Andreas Molander , University of Jyvaskyla, Finland
+
+#include "FITDCSMonitoring/FITDCSDataReader.h"
+
+#include "DetectorsCalibration/Utils.h"
+#include "DetectorsDCS/DataPointCompositeObject.h"
+#include "DetectorsDCS/DataPointIdentifier.h"
+#include "DetectorsDCS/DataPointValue.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+using namespace o2::fit;
+using namespace o2::dcs;
+
+using DPID = o2::dcs::DataPointIdentifier;
+using DPCOM = o2::dcs::DataPointCompositeObject;
+
+void FITDCSDataReader::init(const std::vector& pids)
+{
+  // Fill the array of sub-detector specific DPIDs that will be processed
+  for (const auto& it : pids) {
+    mPids[it] = false;
+    mDpData[it].makeEmpty();
+  }
+}
+
+int FITDCSDataReader::process(const gsl::span dps)
+{
+  // first we check which DPs are missing - if some are, it means that
+  // the delta map was sent
+
+  if (getVerboseMode()) {
+    LOG(info) << "\n\nProcessing new DCS DP map\n-------------------------";
+  }
+
+  if (false) {
+    std::unordered_map mapin;
+    for (auto& it : dps) {
+      mapin[it.id] = it.data;
+    }
+    for (auto& it : mPids) {
+      const auto& el = mapin.find(it.first);
+      if (el == mapin.end()) {
+        LOG(debug) << "DP " << it.first << " not found in DPs from DCS";
+      } else {
+        LOG(debug) << "DP " << it.first << " found in DPs from DCS";
+      }
+    }
+  }
+
+  // now we process all DPs, one by one
+  for (const auto& it : dps) {
+    // we process only the DPs defined in the configuration
+    const auto& el = mPids.find(it.id);
+    if (el == mPids.end()) {
+      LOG(info) << "DP " << it.id << " not found in FITDCSProcessor, we will not process it";
+      continue;
+    }
+    processDP(it);
+    mPids[it.id] = true;
+  }
+
+  return 0;
+}
+
+int FITDCSDataReader::processDP(const DPCOM& dpcom)
+{
+  // Processing a single DP
+  const auto& dpid = dpcom.id;
+  const auto& type = dpid.get_type();
+  const auto& val = dpcom.data;
+
+  if (getVerboseMode()) {
+    if (type == DPVAL_DOUBLE) {
+      LOG(info) << "Processing DP = " << dpcom << " (epoch " << val.get_epoch_time() << "), with value = " << o2::dcs::getValue(dpcom);
+    } else if (type == DPVAL_UINT) {
+      LOG(info) << "Processing DP = " << dpcom << " (epoch " << val.get_epoch_time() << "), with value = " << o2::dcs::getValue(dpcom);
+    }
+  }
+
+  auto flags = val.get_flags();
+  if (processFlags(flags, dpid.get_alias()) == 0) {
+    // Store all DP values
+    if (mDpData[dpid].values.empty() || val.get_epoch_time() > mDpData[dpid].values.back().first) {
+      dpValueConverter.raw_data = val.payload_pt1;
+      if (type == DPVAL_DOUBLE) {
+        mDpData[dpid].add(val.get_epoch_time(), llround(dpValueConverter.double_value * 1000)); // store as nA
+      } else if (type == DPVAL_UINT) {
+        mDpData[dpid].add(val.get_epoch_time(), dpValueConverter.uint_value);
+      }
+    }
+  }
+
+  return 0;
+}
+
+uint64_t FITDCSDataReader::processFlags(const uint64_t flags, const char* pid)
+{
+  // function to process the flag. the return code zero means that all is fine.
+  // anything else means that there was an issue
+
+  // for now, I don't know how to use the flags, so I do nothing
+
+  if (flags & DataPointValue::KEEP_ALIVE_FLAG) {
+    LOG(debug) << "KEEP_ALIVE_FLAG active for DP " << pid;
+  }
+  if (flags & DataPointValue::END_FLAG) {
+    LOG(debug) << "FBI_FLAG active for DP " << pid;
+  }
+  if (flags & DataPointValue::NEW_FLAG) {
+    LOG(debug) << "NEW_FLAG active for DP " << pid;
+  }
+  if (flags & DataPointValue::DIRTY_FLAG) {
+    LOG(debug) << "DIRTY_FLAG active for DP " << pid;
+  }
+  if (flags & DataPointValue::TURN_FLAG) {
+    LOG(debug) << "TURN_FLAG active for DP " << pid;
+  }
+  if (flags & DataPointValue::WRITE_FLAG) {
+    LOG(debug) << "WRITE_FLAG active for DP " << pid;
+  }
+  if (flags & DataPointValue::READ_FLAG) {
+    LOG(debug) << "READ_FLAG active for DP " << pid;
+  }
+  if (flags & DataPointValue::OVERWRITE_FLAG) {
+    LOG(debug) << "OVERWRITE_FLAG active for DP " << pid;
+  }
+  if (flags & DataPointValue::VICTIM_FLAG) {
+    LOG(debug) << "VICTIM_FLAG active for DP " << pid;
+  }
+  if (flags & DataPointValue::DIM_ERROR_FLAG) {
+    LOG(debug) << "DIM_ERROR_FLAG active for DP " << pid;
+  }
+  if (flags & DataPointValue::BAD_DPID_FLAG) {
+    LOG(debug) << "BAD_DPID_FLAG active for DP " << pid;
+  }
+  if (flags & DataPointValue::BAD_FLAGS_FLAG) {
+    LOG(debug) << "BAD_FLAGS_FLAG active for DP " << pid;
+  }
+  if (flags & DataPointValue::BAD_TIMESTAMP_FLAG) {
+    LOG(debug) << "BAD_TIMESTAMP_FLAG active for DP " << pid;
+  }
+  if (flags & DataPointValue::BAD_PAYLOAD_FLAG) {
+    LOG(debug) << "BAD_PAYLOAD_FLAG active for DP " << pid;
+  }
+  if (flags & DataPointValue::BAD_FBI_FLAG) {
+    LOG(debug) << "BAD_FBI_FLAG active for DP " << pid;
+  }
+
+  return 0;
+}
+
+void FITDCSDataReader::updateCcdbObjectInfo()
+{
+  // Prepare the object to be sent to CCDB
+  if (getVerboseMode()) {
+    for (auto& dp : mDpData) {
+      // if (dp.second.values.empty()) {
+      //   continue;
+      // }
+      LOG(info) << "PID = " << dp.first.get_alias();
+      dp.second.print();
+    }
+  }
+
+  std::map metadata;
+  o2::calibration::Utils::prepareCCDBobjectInfo(getDpData(), getccdbDPsInfo(), getCcdbPath(), metadata, getStartValidity(), getEndValidity());
+
+  return;
+}
+
+const std::unordered_map& FITDCSDataReader::getDpData() const { return mDpData; }
+
+void FITDCSDataReader::resetDpData()
+{
+  mDpsMap.clear();
+  mDpData.clear();
+}
+
+const std::string& FITDCSDataReader::getCcdbPath() const { return mCcdbPath; }
+void FITDCSDataReader::setCcdbPath(const std::string& ccdbPath) { mCcdbPath = ccdbPath; }
+long FITDCSDataReader::getStartValidity() const { return mStartValidity; }
+void FITDCSDataReader::setStartValidity(const long startValidity) { mStartValidity = startValidity; }
+bool FITDCSDataReader::isStartValiditySet() const { return mStartValidity != o2::ccdb::CcdbObjectInfo::INFINITE_TIMESTAMP; }
+void FITDCSDataReader::resetStartValidity() { mStartValidity = o2::ccdb::CcdbObjectInfo::INFINITE_TIMESTAMP; }
+long FITDCSDataReader::getEndValidity() const { return mStartValidity + 3 * o2::ccdb::CcdbObjectInfo::DAY; }
+const o2::ccdb::CcdbObjectInfo& FITDCSDataReader::getccdbDPsInfo() const { return mCcdbDpInfo; }
+o2::ccdb::CcdbObjectInfo& FITDCSDataReader::getccdbDPsInfo() { return mCcdbDpInfo; }
+
+bool FITDCSDataReader::getVerboseMode() const { return mVerbose; }
+void FITDCSDataReader::setVerboseMode(bool verboseMode) { mVerbose = verboseMode; }
diff --git a/Detectors/FIT/common/dcsmonitoring/src/FITDCSMonitoringLinkDef.h b/Detectors/FIT/common/dcsmonitoring/src/FITDCSMonitoringLinkDef.h
new file mode 100644
index 0000000000000..521673ad062f3
--- /dev/null
+++ b/Detectors/FIT/common/dcsmonitoring/src/FITDCSMonitoringLinkDef.h
@@ -0,0 +1,24 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifdef __CLING__
+
+#pragma link off all globals;
+#pragma link off all classes;
+#pragma link off all functions;
+
+#pragma link C++ std::unordered_map < uint8_t, bool> + ;
+#pragma link C++ struct o2::fit::DeadChannelMap + ;
+
+// TODO AM: Move this to DataFormatsFIT when unused class warning is solved.
+#pragma link C++ class std::unordered_map < o2::dcs::DataPointIdentifier, o2::fit::DCSDPValues> + ;
+
+#endif
diff --git a/Detectors/FIT/macros/CMakeLists.txt b/Detectors/FIT/macros/CMakeLists.txt
index 02b876df0b189..a6bf1799a5dde 100644
--- a/Detectors/FIT/macros/CMakeLists.txt
+++ b/Detectors/FIT/macros/CMakeLists.txt
@@ -28,3 +28,26 @@ o2_add_test_root_macro(readFV0hits.C
 o2_add_test_root_macro(readFV0digits.C
                        PUBLIC_LINK_LIBRARIES O2::FV0Simulation
                        LABELS fit)
+
+o2_add_test_root_macro(readFITDeadChannelMap.C
+                       PUBLIC_LINK_LIBRARIES O2::CCDB
+                                             O2::DataFormatsFIT
+                                             O2::Framework
+                       LABELS fit)
+
+o2_add_test_root_macro(readFITDCSdata.C
+                       PUBLIC_LINK_LIBRARIES O2::DetectorsDCS
+                                             O2::CCDB
+                       LABELS fit)
+
+o2_add_test_root_macro(compareRecPoints.C
+                       PUBLIC_LINK_LIBRARIES O2::DataFormatsFT0
+                                             O2::DataFormatsFIT
+                       LABELS fit)
+
+o2_add_test_root_macro(readAlignParam.C
+                       PUBLIC_LINK_LIBRARIES O2::CCDB
+                       LABELS fit)
+
+o2_data_file(COPY readFITDCSdata.C DESTINATION Detectors/FIT/macros/)
+o2_data_file(COPY readFITDeadChannelMap.C DESTINATION Detectors/FIT/macros/)
\ No newline at end of file
diff --git a/Detectors/FIT/macros/compareRecPoints.C b/Detectors/FIT/macros/compareRecPoints.C
new file mode 100644
index 0000000000000..0ce077bc616ba
--- /dev/null
+++ b/Detectors/FIT/macros/compareRecPoints.C
@@ -0,0 +1,110 @@
+// Copyright 2019-2024 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file compareRecPoints.C
+/// \brief ROOT macro to compare two trees with RecPoints
+///
+/// \author Artur Furs artur.furs@cern.ch, Andreas Molander andreas.molander@cern.ch
+
+#if !defined(__CLING__) || defined(__ROOTCLING__)
+#include "DataFormatsFT0/RecPoints.h"
+#include "DataFormatsFV0/RecPoints.h"
+#include "DataFormatsFDD/RecPoint.h"
+
+#include "TFile.h"
+#include "TTree.h"
+
+#include 
+#include 
+#include 
+#include 
+#endif
+
+void compareRecPoints(std::string filename1, std::string filename2)
+{
+  std::unique_ptr file1(TFile::Open(filename1.c_str(), "READ"));
+  TTree* tree1 = (TTree*)file1->Get("o2sim");
+
+  std::unique_ptr file2(TFile::Open(filename2.c_str(), "READ"));
+  TTree* tree2 = (TTree*)file2->Get("o2sim");
+
+  if (tree1->GetEntries() != tree2->GetEntries()) {
+    std::cout << "Non equal number of entries in trees!" << std::endl;
+    return;
+  }
+
+  typedef typename o2::ft0::RecPoints RecPoint;
+  typedef typename o2::ft0::ChannelDataFloat ChannelDataFloat;
+
+  std::vector vecRecPoints1;
+  std::vector* ptrVecRecPoints1 = &vecRecPoints1;
+
+  std::vector vecChannelDataFloat1;
+  std::vector* ptrVecChannelDataFloat1 = &vecChannelDataFloat1;
+
+  tree1->SetBranchAddress("FT0Cluster", &ptrVecRecPoints1);
+  tree1->SetBranchAddress("FT0RecChData", &ptrVecChannelDataFloat1);
+
+  std::vector vecRecPoints2;
+  std::vector* ptrVecRecPoints2 = &vecRecPoints2;
+
+  std::vector vecChannelDataFloat2;
+  std::vector* ptrVecChannelDataFloat2 = &vecChannelDataFloat2;
+
+  tree2->SetBranchAddress("FT0Cluster", &ptrVecRecPoints2);
+  tree2->SetBranchAddress("FT0RecChData", &ptrVecChannelDataFloat2);
+
+  for (int iEntry = 0; iEntry < tree1->GetEntries(); iEntry++) {
+    tree1->GetEntry(iEntry);
+    tree2->GetEntry(iEntry);
+
+    if (vecRecPoints1 != vecRecPoints2) {
+      std::cout << "Non equal RecPoints vector!" << std::endl;
+
+      if (vecRecPoints1.size() == vecRecPoints2.size()) {
+        for (int iEvent = 0; iEvent < vecRecPoints1.size(); iEvent++) {
+          const auto& recPoint1 = vecRecPoints1[iEvent];
+          const auto& recPoint2 = vecRecPoints2[iEvent];
+
+          if (!(recPoint1 == recPoint2)) {
+            std::cout << "First RecPoint" << std::endl;
+            recPoint1.print();
+            std::cout << "Second RecPoint" << std::endl;
+            recPoint2.print();
+          }
+        }
+      } else {
+        std::cout << "Non equal number of RecPoints!" << std::endl;
+      }
+    }
+    if (vecChannelDataFloat1 != vecChannelDataFloat2) {
+      std::cout << "Non equal ChannelDataFloat vector!" << std::endl;
+
+      if (vecChannelDataFloat1.size() == vecChannelDataFloat2.size()) {
+        for (int iEvent = 0; iEvent < vecChannelDataFloat1.size(); iEvent++) {
+          const auto& channelDataFloat1 = vecChannelDataFloat1[iEvent];
+          const auto& channelDataFloat2 = vecChannelDataFloat2[iEvent];
+
+          if (!(channelDataFloat1 == channelDataFloat2)) {
+            std::cout << "First ChannelDataFloat" << std::endl;
+            channelDataFloat1.print();
+            std::cout << "Second ChannelDataFloat" << std::endl;
+            channelDataFloat2.print();
+          }
+        }
+      } else {
+        std::cout << "Non equal number of ChannelDataFloat!" << std::endl;
+      }
+    }
+  }
+
+  return;
+}
\ No newline at end of file
diff --git a/Detectors/FIT/macros/readAlignParam.C b/Detectors/FIT/macros/readAlignParam.C
new file mode 100644
index 0000000000000..c438e7a0c86a5
--- /dev/null
+++ b/Detectors/FIT/macros/readAlignParam.C
@@ -0,0 +1,51 @@
+// Copyright 2019-2025 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file  readAlignParam.C
+/// \brief ROOT macro for reading geometry alignment parameters
+///
+/// \author Andreas Molander 
+
+#if !defined(__CLING__) || defined(__ROOTCLING__)
+
+#include "CCDB/BasicCCDBManager.h"
+#include "DetectorsCommonDataFormats/AlignParam.h"
+#include "DetectorsCommonDataFormats/DetID.h"
+#include "DetectorsCommonDataFormats/DetectorNameConf.h"
+
+#include 
+#include 
+
+#endif
+
+int readAlignParam(const std::string& detectorName = "FT0",
+                   long timestamp = -1,
+                   const std::string& ccdbUrl = "https://alice-ccdb.cern.ch")
+{
+  o2::ccdb::BasicCCDBManager& ccdbManager = o2::ccdb::BasicCCDBManager::instance();
+  ccdbManager.setURL(ccdbUrl);
+  ccdbManager.setTimestamp(timestamp);
+
+  const o2::detectors::DetID detID(detectorName.c_str());
+  const std::string alignmentPath = o2::base::DetectorNameConf::getAlignmentPath(detID);
+  const auto alignments = ccdbManager.get>(alignmentPath);
+
+  if (!alignments) {
+    std::cerr << "No alignment parameters found at " << alignmentPath << std::endl;
+    return 1;
+  }
+
+  for (auto alignment : *alignments) {
+    alignment.print();
+  }
+
+  return 0;
+}
\ No newline at end of file
diff --git a/Detectors/FIT/macros/readFITDCSdata.C b/Detectors/FIT/macros/readFITDCSdata.C
new file mode 100644
index 0000000000000..43f59ca8d0cfa
--- /dev/null
+++ b/Detectors/FIT/macros/readFITDCSdata.C
@@ -0,0 +1,420 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file readFITDCSdata.C
+/// \brief ROOT macro for reading the FIT DCS data from CCDB
+///
+/// \author Andreas Molander , University of Jyvaskyla, Finland
+
+#if !defined(__CLING__) || defined(__ROOTCLING__)
+
+#include "CCDB/CcdbApi.h"
+#include "CCDB/CCDBTimeStampUtils.h"
+#include "DataFormatsFIT/DCSDPValues.h"
+#include "DetectorsDCS/DataPointIdentifier.h"
+#include "DetectorsDCS/DeliveryType.h"
+#include "TAxis.h"
+#include "TCanvas.h"
+#include "TFile.h"
+#include "TGraph.h"
+#include "TLegend.h"
+#include "TMultiGraph.h"
+#include "TStyle.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#endif
+
+#include "Framework/Logger.h"
+
+#include 
+
+// Helper functions
+
+const std::string epochToReadable(const long timestamp);
+std::vector getAliases(const std::string& input, const o2::ccdb::CcdbApi& ccdbApi, const std::string& detectorName, const long timestamp);
+void plotFITDCSmultigraph(const TMultiGraph& multiGraph);
+
+/// ROOT macro for reading FIT DCS data from CCDB.
+///
+/// The macro can:
+/// - Plot the trends (default ON)
+/// - Store a TMultiGraph of the trends to file (default OFF)
+/// - Store the trend values to file (default OFF). The data format is std::map, where the key is the data point alias.
+/// - Print the trend values to the output (default OFF).
+///
+/// Please note that query timestamps behaves a bit "approximately" sometimes. If values in the ends of the queried time period are missing,
+/// try to include some extra time in the end where values are missing.
+///
+/// \param detectorName     FIT subdetector, i.e. FT0/FV0/FDD, for which to query the data points.
+/// \param dataPointAliases A string specifying what data points to query.
+///                         It can be a semicolon separated list of data point aliases, e.g. "FT0/HV/FT0_A/MCP_A1/actual/iMon;FT0/HV/FT0_A/MCP_A2/actual/iMon"
+///                         or a relative path to a file where the aliases are listed (one alias on each line), e.g. "aliases.txt" or "path/to/aliases.txt".
+///                         If left empty, all data points defined in [ccdbUrl]/[detectorName]/Config/DCSDPconfig are queried.
+/// \param timeStart        UNIX timestamp (in ms) for start of data point query. If omitted, one hour before the end time is used.
+/// \param timeEnd          UNIX timestamp (in ms) for end of data point query. If omitted, current time is used.
+/// \param ccdbUrl          CCDB url.
+/// \param plot             Whether to plot the data point values.
+/// \param rootOutput       If specified, a plot and raw values are stored in [rootOutput].
+/// \param print            Whether to print the data point values.
+/// \param textOutput       If specified the data point values are stored in '[textOutput]'. TODO: implement
+/// \param verbose          Verbose mode for debugging.
+void readFITDCSdata(std::string detectorName = "FT0",
+                    const std::string& dataPointAliases = "",
+                    long timeStart = -1,
+                    long timeEnd = -1,
+                    const std::string& ccdbUrl = "http://alice-ccdb.cern.ch",
+                    const bool plot = true,
+                    const std::string& rootOutput = "",
+                    const bool print = false,
+                    const std::string& textOutput = "",
+                    const bool verbose = false)
+{
+  // Parse and check detector name
+  boost::to_upper(detectorName);
+  if (detectorName != "FT0" && detectorName != "FV0" && detectorName != "FDD") {
+    LOGP(fatal, "Invalid detector name provided: '{}'. Please use [FT0/FV0/FDD].", detectorName);
+    return;
+  }
+  LOGP(info, "Fetching DCS data points for {}.", detectorName);
+
+  // Init CCDB stuff
+  o2::ccdb::CcdbApi ccdbApi;
+  ccdbApi.init(ccdbUrl);
+  const std::string ccdbPath = detectorName + "/Calib/DCSDPs";
+  const std::map metadata;
+
+  // Set query time interval
+  long timeNow = o2::ccdb::getCurrentTimestamp();
+  if (timeEnd < 0 || timeEnd > timeNow) {
+    timeEnd = timeNow;
+  }
+  if (timeStart < 0 || timeStart > timeNow) {
+    timeStart = std::max(timeEnd - 1 * o2::ccdb::CcdbObjectInfo::HOUR, 0L);
+  }
+  if (timeStart > timeEnd) {
+    long timeTmp = timeEnd;
+    timeEnd = timeStart;
+    timeStart = timeTmp;
+  }
+  LOG(info) << "Querying data points for time interval:";
+  LOGP(info, "START {} ({})", timeStart, epochToReadable(timeStart));
+  LOGP(info, "END   {} ({})", timeEnd, epochToReadable(timeEnd));
+
+  // Define what data points to query
+  std::vector requestedDPaliases = getAliases(dataPointAliases, ccdbApi, detectorName, timeStart);
+
+  LOG(info) << "Querying datapoint(s):";
+  for (auto& alias : requestedDPaliases) {
+    LOG(info) << alias;
+  }
+
+  // Set up data point value storage
+  std::map dataSeries;
+  for (std::string& alias : requestedDPaliases) {
+    dataSeries[alias] = o2::fit::DCSDPValues();
+  }
+
+  // Query data points
+
+  long queryTimeStamp = timeStart;
+  // Headers are used to compare fetched CCDB objects
+  std::map headers;
+  std::map headersPrev;
+  std::map headersLatest = ccdbApi.retrieveHeaders(ccdbPath, metadata, timeEnd);
+  std::unordered_map* ccdbMap = nullptr;     // Pointer to the CCDB object
+  std::unordered_map* ccdbMapPrev = nullptr; // Pointer to the previously queried CCDB object
+  o2::dcs::DataPointIdentifier dpIdTmp;                                                          // DataPointIdentifier object used as CCDB object map key
+  o2::fit::DCSDPValues* ccdbDPValuesPointer = nullptr;                                           // Pointer to the CCDB object map values
+
+  while (queryTimeStamp <= timeEnd) {
+    ccdbMap = ccdbApi.retrieveFromTFileAny>(ccdbPath, metadata, queryTimeStamp, &headers);
+
+    if (!ccdbMap) {
+      // TODO: Improve functionality when data not found
+      if (queryTimeStamp == timeStart && headersLatest.empty()) {
+        LOGP(fatal, "No CCDB object found for start time {} ({}) or end time {} ({}).\nThe macro can't handle this, aborting.", queryTimeStamp, epochToReadable(queryTimeStamp), timeEnd, epochToReadable(timeEnd));
+        return;
+      } else if (headersLatest.empty()) {
+        LOGP(error, "No CCDB object found for timestamp {} ({}) or end time {} ({}).", queryTimeStamp, epochToReadable(queryTimeStamp), timeEnd, epochToReadable(timeEnd));
+        LOGP(error, "The O2 workflow might have been stopped, and there are periods without valid objects in CCDB. The macro can't handle this, stopping here.");
+        break;
+      } else {
+        LOGP(error, "No valid CCDB object for timestamp {} ({}).", queryTimeStamp, epochToReadable(queryTimeStamp));
+        queryTimeStamp += 10 * o2::ccdb::CcdbObjectInfo::MINUTE;
+        continue;
+      }
+    } else if (headers["Location"] == headersPrev["Location"]) {
+      // CCDB was maybe not updated as it should after the last CCDB object (i.e. there should always be a new object after 10 mins)
+
+      if ((headers["Location"] == headersLatest["Location"])) {
+        LOGP(warning, "No newer CCDB objects for query period found. The O2 workflow might have been stopped.");
+        break;
+      } else {
+        LOGP(warning, "No new CCDB object for time {} ({}). The O2 workflow might have been stopped temporarily.", queryTimeStamp, epochToReadable(queryTimeStamp));
+        // There are newer objects for the query period, move on to the next query
+        // Currently the CCDB is updated every 10 mins
+        queryTimeStamp += 10 * o2::ccdb::CcdbObjectInfo::MINUTE;
+        continue;
+      }
+    } else {
+      if (verbose) {
+        LOGP(info, "CCDB object for timestamp {} ({}) found.", queryTimeStamp, epochToReadable(queryTimeStamp));
+      }
+      headersPrev = headers;
+    }
+
+    // The CCDB object should always contain values for all datapoints. This is just to check that.
+    if (verbose && ((detectorName == "FT0" && ccdbMap->size() != 501) || (detectorName == "FV0" && ccdbMap->size() != 147) || (detectorName == "FDD" && ccdbMap->size() != 76))) {
+      LOGP(error,
+           "Wrong number of DCS datapoints fetched for {}, got {}. There is a bug, please send output of this script, with input parameters, to andreas.molander@cern.ch.",
+           detectorName, ccdbMap->size());
+    }
+
+    for (std::string& alias : requestedDPaliases) {
+      // Need to fetch data point based on its type. To avoid specifying/knowing the type of each data point, try all used types instead.
+      // TODO: This is not very nice, the dataformat in CCDB should be changed.
+      o2::dcs::DataPointIdentifier::FILL(dpIdTmp, alias, o2::dcs::DeliveryType::DPVAL_DOUBLE);
+      if (ccdbMap->find(dpIdTmp) != ccdbMap->end()) {
+        ccdbDPValuesPointer = &ccdbMap->at(dpIdTmp);
+      } else {
+        o2::dcs::DataPointIdentifier::FILL(dpIdTmp, alias, o2::dcs::DeliveryType::DPVAL_UINT);
+        if (ccdbMap->find(dpIdTmp) != ccdbMap->end()) {
+          ccdbDPValuesPointer = &ccdbMap->at(dpIdTmp);
+        } else {
+          ccdbDPValuesPointer = nullptr;
+        }
+      }
+
+      if (ccdbDPValuesPointer) {
+        // If there are no values read for this DP yet, or there are newer values in the current CCDB object
+        if (dataSeries[alias].empty() || (dataSeries[alias].values.back().first < ccdbDPValuesPointer->values.back().first)) {
+          // if (verbose) {
+          //   LOGP(info, "Newer values found.");
+          // }
+
+          // Iterate through the values in the current CCDB object
+          for (auto& it : ccdbDPValuesPointer->values) {
+            if ((it.first >= timeStart && it.first <= timeEnd) && (dataSeries[alias].empty() || it.first > dataSeries[alias].values.back().first)) {
+              // if (dataSeries[alias].empty() || it.first > dataSeries[alias].values.back().first) {
+              dataSeries[alias].add(it.first, it.second);
+            }
+          }
+        }
+      } else {
+        LOGP(error, "Requested DP '{}' not found in CCDB object for timestamp {}.", alias, queryTimeStamp);
+      }
+    }
+
+    // Currently the CCDB is updated every 10 mins
+    queryTimeStamp += 10 * o2::ccdb::CcdbObjectInfo::MINUTE;
+  }
+
+  if (print) {
+    LOG(info) << "Printing data point values:";
+    for (auto& it : dataSeries) {
+      LOGP(info, "{}", it.first);
+      int nValues = it.second.values.size();
+      LOGP(info, "{} value(s):", nValues);
+      if (nValues) {
+        if (verbose) {
+          for (auto& value : it.second.values) {
+            LOGP(info, "TIME = {} ({}), VALUE = {}", value.first, epochToReadable(value.first), value.second);
+          }
+        } else {
+          LOGP(info, "First value:");
+          LOGP(info, "TIME = {} ({}), VALUE = {}", it.second.values.front().first, epochToReadable(it.second.values.front().first), it.second.values.front().second);
+          LOGP(info, "Last value:");
+          LOGP(info, "TIME = {} ({}), VALUE = {}", it.second.values.back().first, epochToReadable(it.second.values.back().first), it.second.values.back().second);
+        }
+      }
+      LOG(info);
+    }
+  }
+
+  if (plot || !rootOutput.empty()) {
+    std::unique_ptr multiGraph(new TMultiGraph);
+    multiGraph->SetName("mgDCSDPTrends");
+    std::vector graphs;
+    int pointCounter = 0;
+
+    for (auto& dp : dataSeries) {
+      if (!dp.second.empty()) {
+        graphs.push_back(new TGraph);
+        graphs.back()->SetName(dp.first.c_str());
+        graphs.back()->GetXaxis()->SetTimeDisplay(1);
+        graphs.back()->GetXaxis()->SetTimeFormat("#splitline{%d.%m.%y}{%H:%M:%S}");
+        graphs.back()->SetMarkerStyle(20);
+        pointCounter = 0;
+
+        for (size_t i = 0; i < dp.second.values.size(); i++) {
+          if (i > 0) {
+            graphs.back()->SetPoint(pointCounter++, dp.second.values.at(i).first / 1000., dp.second.values.at(i - 1).second);
+          }
+          graphs.back()->SetPoint(pointCounter++, dp.second.values.at(i).first / 1000., dp.second.values.at(i).second);
+        }
+        // for (auto& dpValue : dp.second.values) {
+        //   graphs.back()->SetPoint(pointCounter++, dpValue.first / 1000., dpValue.second);
+        // }
+        multiGraph->Add(graphs.back());
+      } else {
+        LOGP(warning, "No CCDB data found for alias '{}'. Data point excluded from graph.", dp.first);
+      }
+    }
+
+    multiGraph->GetXaxis()->SetTimeDisplay(1);
+    multiGraph->GetXaxis()->SetTimeFormat("#splitline{%d.%m.%y}{%H:%M:%S}");
+
+    if (plot) {
+      plotFITDCSmultigraph(*multiGraph);
+    }
+
+    if (!rootOutput.empty()) {
+      LOGP(info, "Storing trends in '{}'", rootOutput);
+      std::unique_ptr rootFile(TFile::Open(rootOutput.c_str(), "RECREATE"));
+      multiGraph->Write();
+      rootFile->WriteObject(&dataSeries, "DCSDPValues");
+    }
+
+    if (!textOutput.empty()) {
+      LOG(info) << "Storing data point values to text file is not implemented yet.";
+    }
+  }
+}
+
+void plotFITDCSdataFromFile(const std::string& fileName)
+{
+  std::unique_ptr file(TFile::Open(fileName.c_str()));
+  if (!file || file->IsZombie()) {
+    LOGP(fatal, "Error opening file '{}'", fileName);
+    return;
+  }
+
+  std::unique_ptr multiGraph(file->Get("mgDCSDPTrends"));
+  if (!multiGraph) {
+    LOGP(fatal, "Cannot find TMultiGraph 'mgDCSDPTrends' in {}", fileName);
+    return;
+  }
+
+  plotFITDCSmultigraph(*multiGraph);
+}
+
+/// ROOT macro for printing the contents of one CCDB object. Used for debugging.
+void printCCDBObject(const std::string detectorName = "FT0",
+                     long timestamp = -1,
+                     const std::string ccdbUrl = "http://alice-ccdb.cern.ch")
+{
+  // // Parse and check detector name
+  // boost::to_upper(detectorName);
+  // if (detectorName != "FT0" && detectorName != "FV0" && detectorName != "FDD") {
+  //   LOGP(fatal, "Invalid detector name provided: '{}'. Please use [FT0/FV0/FDD].", detectorName);
+  //   return;
+  // }
+  LOGP(info, "Querying CCDB object for {}.", detectorName);
+
+  // Init CCDB stuff
+  o2::ccdb::CcdbApi ccdbApi;
+  ccdbApi.init(ccdbUrl);
+  const std::string ccdbPath = detectorName + "/Calib/DCSDPs";
+  const std::map metadata;
+
+  if (timestamp < 0) {
+    timestamp = o2::ccdb::getCurrentTimestamp();
+  }
+
+  std::unordered_map* map = ccdbApi.retrieveFromTFileAny>(ccdbPath, metadata, timestamp);
+
+  if (!map) {
+    LOGP(fatal, "CCDB object not found in {}/{} for timestamp {}", ccdbUrl, ccdbPath, timestamp);
+    return;
+  }
+
+  int nEmptyDPs = 0;
+
+  for (auto& it : *map) {
+    if (it.second.empty()) {
+      nEmptyDPs++;
+    }
+    std::stringstream tmp;
+    tmp << it.first;
+    LOGP(info, "DPID = {}", tmp.str());
+    it.second.print();
+  }
+
+  LOGP(info, "Size of map = {}", map->size());
+  LOGP(info, "Empty DPs   = {}", nEmptyDPs);
+}
+
+// Helper functions
+
+void plotFITDCSmultigraph(const TMultiGraph& multiGraph)
+{
+  gStyle->SetPalette(kRainBow);
+  gStyle->SetTimeOffset(0);
+  gStyle->SetLabelOffset(0.03);
+
+  TCanvas* canvas = new TCanvas("canvas", "FIT DCS DP trends");
+  multiGraph.DrawClone("apl pmc plc");
+  canvas->BuildLegend();
+}
+
+const std::string epochToReadable(const long timestamp)
+{
+  std::string readableTime;
+  time_t timeSeconds = timestamp / 1000;
+  readableTime = std::string(asctime(localtime(&timeSeconds)));
+  readableTime.pop_back();
+  return readableTime;
+}
+
+std::vector getAliases(const std::string& input, const o2::ccdb::CcdbApi& ccdbApi, const std::string& detectorName, const long timestamp)
+{
+  std::vector aliases;
+  std::string alias;
+  std::ifstream file(input);
+
+  if (file.is_open()) {
+    LOGP(info, "Parsing input file '{}'", input);
+    while (std::getline(file, alias)) {
+      if (!alias.empty()) {
+        aliases.push_back(alias);
+      }
+    }
+  } else if (!input.empty()) {
+    LOGP(info, "Parsing input string '{}'", input);
+    std::stringstream ss(input);
+    while (std::getline(ss, alias, ';')) {
+      aliases.push_back(alias);
+    }
+  } else {
+    // Input string empty, fetching all datapoints
+    LOGP(info, "Data point input empty, fetching data point definitions from CCDB.");
+    std::string ccdbPath = detectorName + "/Config/DCSDPconfig";
+    std::map metadata;
+    std::unordered_map* dpConfig = ccdbApi.retrieveFromTFileAny>(ccdbPath, metadata, timestamp);
+    if (dpConfig) {
+      for (auto& it : *dpConfig) {
+        aliases.push_back(it.first.get_alias());
+      }
+    } else {
+      LOGP(fatal, "No data point input provided, and can't fetch data point definitions from {}/{} for timestamp {}", ccdbApi.getURL(), ccdbPath, timestamp);
+    }
+  }
+  return aliases;
+}
diff --git a/Detectors/FIT/macros/readFITDeadChannelMap.C b/Detectors/FIT/macros/readFITDeadChannelMap.C
new file mode 100644
index 0000000000000..3ba0d6f338e50
--- /dev/null
+++ b/Detectors/FIT/macros/readFITDeadChannelMap.C
@@ -0,0 +1,96 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file readFITDeadChannelMap.C
+/// \brief Macro to read the FIT dead channel maps from CCDB
+///
+/// \author Andreas Molander , University of Jyvaskyla, Finland
+
+#if !defined(__CLING__) || defined(__ROOTCLING__)
+
+#include "CCDB/CcdbApi.h"
+#include "DataFormatsFIT/DeadChannelMap.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#endif
+
+#include "Framework/Logger.h"
+
+#include 
+
+void readFITDeadChannelMap(std::string detectorName = "FT0",
+                           long timestamp = -1,
+                           const std::string& ccdbUrl = "http://alice-ccdb.cern.ch",
+                           const bool verbose = false)
+{
+  // Parse and check detector name
+  boost::to_upper(detectorName);
+  if (detectorName != "FT0" && detectorName != "FV0" && detectorName != "FDD") {
+    LOGP(fatal, "Invalid detector name provided: '{}'. Please use [FT0/FV0/FDD].", detectorName);
+    return;
+  }
+  LOGP(info, "Fetching dead channel map for {}.", detectorName);
+
+  // Init CCDB stuff
+  o2::ccdb::CcdbApi ccdbApi;
+  ccdbApi.init(ccdbUrl);
+  const std::string ccdbPath = detectorName + "/Calib/DeadChannelMap";
+  std::map metadata;
+
+  if (timestamp < 0) {
+    timestamp = o2::ccdb::getCurrentTimestamp();
+  }
+
+  o2::fit::DeadChannelMap* map = ccdbApi.retrieveFromTFileAny(ccdbPath, metadata, timestamp);
+  if (!map) {
+    LOGP(fatal, "Dead channel map not found in {}/{} for timestamp {}.", ccdbUrl, ccdbPath, timestamp);
+    return;
+  }
+
+  std::vector aliveChannels;
+  std::vector deadChannels;
+
+  uint8_t chId;
+  for (auto& i : map->map) {
+    chId = i.first;
+    if (verbose) {
+      LOGP(info, "Channel {}: {}", chId, i.second);
+    }
+    if (map->isChannelAlive(chId)) {
+      aliveChannels.push_back(chId);
+    } else {
+      deadChannels.push_back(chId);
+    }
+  }
+
+  std::sort(aliveChannels.begin(), aliveChannels.end());
+  std::sort(deadChannels.begin(), deadChannels.end());
+
+  LOG(info) << "Alive channels: ";
+  for (auto& ch : aliveChannels) {
+    LOGP(info, "{}", ch);
+  }
+  LOG(info) << "Dead channels: ";
+  for (auto& ch : deadChannels) {
+    LOGP(info, "{}", ch);
+  }
+
+  LOGP(info, "Number of channels      = {}", map->map.size());
+  LOGP(info, "Number of alive channels = {}", aliveChannels.size());
+  LOGP(info, "Number of dead channels  = {}", deadChannels.size());
+
+  return;
+}
diff --git a/Detectors/FIT/macros/readFT0digits2.C b/Detectors/FIT/macros/readFT0digits2.C
index 42ecb14573e1d..86d2a4dea19ff 100644
--- a/Detectors/FIT/macros/readFT0digits2.C
+++ b/Detectors/FIT/macros/readFT0digits2.C
@@ -25,7 +25,7 @@
 #include "SimulationDataFormat/MCTruthContainer.h"
 #include "SimulationDataFormat/MCCompLabel.h"
 #include "DataFormatsFT0/MCLabel.h"
-#include "FairLogger.h"
+#include 
 
 void AdjustStatBox(TH1* h, float x1ndc, float x2ndc, float y1ndc, float y2ndc)
 {
diff --git a/Detectors/FIT/macros/readFT0hits.C b/Detectors/FIT/macros/readFT0hits.C
index 14d25fa4a99a8..fafcaac570311 100644
--- a/Detectors/FIT/macros/readFT0hits.C
+++ b/Detectors/FIT/macros/readFT0hits.C
@@ -1,13 +1,29 @@
+// Copyright 2019-2025 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
 #if !defined(__CLING__) || defined(__ROOTCLING__)
+
+#include "DataFormatsFIT/Triggers.h"
 #include "DataFormatsFT0/Digit.h"
 #include "DataFormatsFT0/HitType.h"
 #include "SimulationDataFormat/MCEventHeader.h"
 #include 
+#include 
 #include 
 #include 
 #include "DetectorsCommonDataFormats/DetID.h"
 #include "DetectorsCommonDataFormats/DetectorNameConf.h"
 
+#endif
+
 void readFT0hits()
 {
 
@@ -24,6 +40,8 @@ void readFT0hits()
   TH2F* hPel = new TH2F("hPelDig", "N p.e. ", 220, 0, 220, 500, 0, 10000);
   TH2F* hXYA = new TH2F("hXYA", "X vs Y A side", 400, -20, 20, 400, -20, 20);
   TH2F* hXYC = new TH2F("hXYC", "X vs Y C side", 400, -20, 20, 400, -20, 20);
+  TH1F* hZA = new TH1F("hZA", "Z A side", 200, 330, 340);
+  TH1F* hZC = new TH1F("hZC", "Z C side", 200, -90, -80);
 
   gDirectory = cwd;
 
@@ -59,10 +77,13 @@ void readFT0hits()
       hTimeHitA->Fill(detID, hit_time[detID] - 11.04);
       hTimeHitC->Fill(detID, hit_time[detID] - 2.91);
       countE[detID]++;
-      if (detID < 96)
+      if (detID < 96) {
         hXYA->Fill(hit.GetX(), hit.GetY());
-      if (detID > 95)
+        hZA->Fill(hit.GetZ());
+      } else {
         hXYC->Fill(hit.GetX(), hit.GetY());
+        hZC->Fill(hit.GetZ());
+      }
     }
     for (int ii = 0; ii < 220; ii++) {
       if (countE[ii] > 100) {
@@ -82,6 +103,6 @@ void readFT0hits()
   hMultHit->Write();
   hXYA->Write();
   hXYC->Write();
-
+  hZA->Write();
+  hZC->Write();
 } // end of macro
-#endif
diff --git a/Detectors/FIT/macros/readFV0digits.C b/Detectors/FIT/macros/readFV0digits.C
index e978c36bccb5c..95ce1a0aae42b 100644
--- a/Detectors/FIT/macros/readFV0digits.C
+++ b/Detectors/FIT/macros/readFV0digits.C
@@ -26,7 +26,7 @@
 #include "SimulationDataFormat/MCCompLabel.h"
 #include "FV0Base/Constants.h"
 #include "DataFormatsFV0/MCLabel.h"
-#include "FairLogger.h"
+#include 
 
 void AdjustStatBox(TH1* h, float x1ndc, float x2ndc, float y1ndc, float y2ndc)
 {
diff --git a/Detectors/FIT/macros/readFV0hits.C b/Detectors/FIT/macros/readFV0hits.C
index 1235eea6bc352..933138fb1434b 100644
--- a/Detectors/FIT/macros/readFV0hits.C
+++ b/Detectors/FIT/macros/readFV0hits.C
@@ -1,3 +1,14 @@
+// Copyright 2019-2025 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
 #if !defined(__CLING__) || defined(__ROOTCLING__)
 
 #include 
@@ -18,10 +29,12 @@
 #include 
 #include 
 #include "DataFormatsFV0/Hit.h"
-#include "FairLogger.h"
+#include 
 #include "DetectorsCommonDataFormats/DetectorNameConf.h"
 #include "DetectorsCommonDataFormats/DetID.h"
 
+#endif
+
 void AdjustStatBox(TH1* h, float x1ndc, float x2ndc, float y1ndc, float y2ndc)
 {
   gPad->Update();
@@ -54,6 +67,9 @@ void InitHistoNames(std::vector& vhName, std::vector& vPdg)
   vhName.push_back("hElossDet");
   vhName.push_back("hEtotVsR");
   vhName.push_back("hEtotVsEloss");
+  vhName.push_back("hXY");
+  vhName.push_back("hXYzoom");
+  vhName.push_back("hZ");
 
   for (UInt_t ipdg = 0; ipdg < vPdg.size(); ipdg++) {
     std::stringstream ss;
@@ -63,7 +79,7 @@ void InitHistoNames(std::vector& vhName, std::vector& vPdg)
   }
 }
 
-void readFV0Hits(std::string simPrefix = "o2sim", UInt_t rebin = 1)
+void readFV0hits(std::string simPrefix = "o2sim", UInt_t rebin = 1)
 {
   using namespace o2::detectors;
   std::string simFName(o2::base::DetectorNameConf::getHitsFileName(DetID::FV0, simPrefix));
@@ -85,6 +101,9 @@ void readFV0Hits(std::string simPrefix = "o2sim", UInt_t rebin = 1)
   TH2F* hElossDet = new TH2F(vHistoNames.at(8).c_str(), "", nEl, 0, el1, nCells, 0, nCells);
   TH2F* hEtotVsR = new TH2F(vHistoNames.at(9).c_str(), "", 30000, 0, 300, 80, 0, 80);
   TH2F* hEtotVsEloss = new TH2F(vHistoNames.at(10).c_str(), "", 30000, 0, 300, nEl, 0, el1);
+  TH2F* hXY = new TH2F(vHistoNames.at(11).c_str(), "", 200, -100, 100, 200, -100, 100);
+  TH2F* hXYzoom = new TH2F(vHistoNames.at(12).c_str(), "", 200, -20, 20, 200, -20, 20);
+  TH1F* hZ = new TH1F(vHistoNames.at(13).c_str(), "", 200, 315, 325);
 
   // Setup histo properties
   hElossDet->SetXTitle("Energy loss [MeV]");
@@ -96,6 +115,14 @@ void readFV0Hits(std::string simPrefix = "o2sim", UInt_t rebin = 1)
   hEtotVsEloss->SetXTitle("Total energy at entrance [MeV]");
   hEtotVsEloss->SetYTitle("Energy loss [MeV]");
   hEtotVsEloss->SetZTitle("Counts");
+  hXY->SetXTitle("X [cm]");
+  hXY->SetYTitle("Y [cm]");
+  hXY->SetZTitle("Counts");
+  hXYzoom->SetXTitle("X [cm]");
+  hXYzoom->SetYTitle("Y [cm]");
+  hXYzoom->SetZTitle("Counts");
+  hZ->SetXTitle("Hit Z-coordinate [cm]");
+  hZ->SetYTitle("Counts");
   for (UInt_t ih = 0; ih < vhElossVsDistance.size(); ih++) {
     TH2F* h = vhElossVsDistance.at(ih);
     std::stringstream ss;
@@ -124,6 +151,9 @@ void readFV0Hits(std::string simPrefix = "o2sim", UInt_t rebin = 1)
   vh.push_back(hEtotVsEloss);
   vh.insert(vh.end(), vhElossVsDistance.begin(), vhElossVsDistance.end());
   vh.insert(vh.end(), vhElossVsEtot.begin(), vhElossVsEtot.end());
+  vh.push_back(hXY);
+  vh.push_back(hXYzoom);
+  vh.push_back(hZ);
   for (UInt_t ih = 0; ih < vh.size(); ih++) {
     vh[ih]->SetDirectory(0);
     vh[ih]->GetXaxis()->SetTitleSize(fontsize);
@@ -177,6 +207,9 @@ void readFV0Hits(std::string simPrefix = "o2sim", UInt_t rebin = 1)
         vhElossVsDistance.at(vhElossVsDistance.size() - 1)->Fill(hit->GetEnergyLoss() * 1e3, distance);
         vhElossVsEtot.at(vhElossVsEtot.size() - 1)->Fill(hit->GetEnergyLoss() * 1e3, hit->GetTotalEnergyAtEntrance() * 1e3);
       }
+      hXY->Fill(hit->GetX(), hit->GetY());
+      hXYzoom->Fill(hit->GetX(), hit->GetY());
+      hZ->Fill(hit->GetZ());
     }
   }
 
@@ -323,5 +356,3 @@ int compareFV0Hits(std::string simFName1 = "fv0hit-rawhistos.root", std::string
   }
   return 0;
 }
-
-#endif
diff --git a/Detectors/FIT/raw/CMakeLists.txt b/Detectors/FIT/raw/CMakeLists.txt
index 9090f54029017..d8ab9964dbee4 100644
--- a/Detectors/FIT/raw/CMakeLists.txt
+++ b/Detectors/FIT/raw/CMakeLists.txt
@@ -10,5 +10,19 @@
 # or submit itself to any jurisdiction.
 
 o2_add_library(FITRaw
-               SOURCES src/DataBlockBase.cxx src/DataBlockFIT.cxx src/DigitBlockBase.cxx src/DigitBlockFIT.cxx src/RawReaderBase.cxx src/RawReaderBaseFIT.cxx src/RawWriterFIT.cxx
-               PUBLIC_LINK_LIBRARIES O2::CommonDataFormat O2::Headers Microsoft.GSL::GSL O2::DetectorsRaw O2::CommonUtils)
+               SOURCES src/DataBlockBase.cxx
+                       src/DataBlockFIT.cxx
+                       src/DigitBlockBase.cxx
+                       src/DigitBlockFIT.cxx
+                       src/RawReaderBase.cxx
+                       src/RawReaderBaseFIT.cxx
+                       src/RawWriterFIT.cxx
+               PUBLIC_LINK_LIBRARIES O2::CommonDataFormat
+                                     O2::Headers
+                                     Microsoft.GSL::GSL
+                                     O2::DetectorsRaw
+                                     O2::CommonUtils
+                                     O2::DataFormatsFIT
+                                     O2::DetectorsCommonDataFormats
+                                     O2::DataFormatsParameters
+                                     Boost::program_options)
diff --git a/Detectors/FIT/raw/include/FITRaw/DataBlockBase.h b/Detectors/FIT/raw/include/FITRaw/DataBlockBase.h
index c034f7c81a7da..1c95f354c6292 100644
--- a/Detectors/FIT/raw/include/FITRaw/DataBlockBase.h
+++ b/Detectors/FIT/raw/include/FITRaw/DataBlockBase.h
@@ -9,30 +9,30 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 //
-//file DataBlockBase.h base class for RAW data format data blocks
+// file DataBlockBase.h base class for RAW data format data blocks
 //
 // Artur.Furs
 // afurs@cern.ch
-//DataBlockWrapper - wrapper for raw data structures
-//There should be three static fields in raw data structs, which defines its "signature":
+// DataBlockWrapper - wrapper for raw data structures
+// There should be three static fields in raw data structs, which defines its "signature":
 //  payloadSize - actual payload size per one raw data struct element (can be larger than GBTword size!)
 //  payloadPerGBTword - maximum payload per one GBT word
 //  MaxNelements - maximum number of elements per data block(for header it should be equal to 1)
 //  MinNelements - minimum number of elements per data block(for header it should be equal to 1)
 
-//Also it requares several methods:
-//  print() - for printing raw data structs
-//  getIntRec() - for InteractionRecord extraction, should be in Header struct
-
-//DataBlockBase - base class for making composition of raw data structures, uses CRTP(static polyporphism)
-//usage:
-//  class DataBlockOfYourModule: public DataBlockBase< DataBlockOfYourModule, RawHeaderStruct, RawDataStruct ...>
-//  define "deserialization" method with deserialization logic for current DataBlock
-//  define "sanityCheck" method for checking if the DataBlock is correct
-//Warning! Classes should be simple, without refs and pointers!
-//TODO:
-//  need to use references on the DataBlock fileds, with fast access
-//  traites for classes and structs
+// Also it requares several methods:
+//   print() - for printing raw data structs
+//   getIntRec() - for InteractionRecord extraction, should be in Header struct
+
+// DataBlockBase - base class for making composition of raw data structures, uses CRTP(static polyporphism)
+// usage:
+//   class DataBlockOfYourModule: public DataBlockBase< DataBlockOfYourModule, RawHeaderStruct, RawDataStruct ...>
+//   define "deserialization" method with deserialization logic for current DataBlock
+//   define "sanityCheck" method for checking if the DataBlock is correct
+// Warning! Classes should be simple, without refs and pointers!
+// TODO:
+//   need to use references on the DataBlock fileds, with fast access
+//   traites for classes and structs
 //
 
 #ifndef ALICEO2_FIT_DATABLOCKBASE_H_
@@ -41,6 +41,7 @@
 #include 
 #include 
 #include "CommonDataFormat/InteractionRecord.h"
+#include "DataFormatsFIT/RawDataMetric.h"
 #include "Headers/RAWDataHeader.h"
 #include 
 #include 
@@ -57,12 +58,17 @@ namespace o2
 namespace fit
 {
 
-using namespace std;
-
-static constexpr size_t SIZE_WORD = 16;                                                       // should be changed to gloabal variable
+static constexpr size_t SIZE_WORD = 16;
+static constexpr size_t SIZE_WORD_GBT = 10;                                                   // should be changed to gloabal variable
 static constexpr size_t SIZE_MAX_PAGE = 8192;                                                 // should be changed to gloabal variable
 static constexpr size_t SIZE_MAX_PAYLOAD = SIZE_MAX_PAGE - sizeof(o2::header::RAWDataHeader); // should be changed to gloabal variable
 
+template 
+struct DataBlockConfig {
+  constexpr static bool sIsPadded = isPadded;
+  typedef DataBlockConfig InvertedPadding_t;
+};
+
 template 
 struct DataBlockHelper {
   template 
@@ -119,57 +125,78 @@ struct DataBlockHelper {
   }
 };
 
-template ::check()>>
+template ::check()>>
 struct DataBlockWrapper {
   DataBlockWrapper() = default;
   DataBlockWrapper(const DataBlockWrapper&) = default;
   typedef T Data_t;
+  typedef ConfigType Config_t;
+  constexpr static bool sIsPadded = Config_t::sIsPadded;
+  static constexpr std::size_t getWordSize()
+  {
+    if constexpr (sIsPadded) {
+      return SIZE_WORD;
+    } else {
+      return SIZE_WORD_GBT;
+    }
+  }
+
+  constexpr static std::size_t sSizeWord = getWordSize();
   void serialize(std::vector& vecBytes, size_t nWords, size_t& destPos) const
   {
-    const uint8_t* srcAddress = (uint8_t*)mData;
-    if (nWords == 0 || nWords > MaxNwords || vecBytes.size() - destPos < nWords * SIZE_WORD) {
+    const auto nBytesToWrite = nWords * sSizeWord;
+    if ((vecBytes.size() - destPos) < nBytesToWrite || nWords < MinNwords || nWords > MaxNwords) {
       LOG(info) << "Warning! Incorrect serialisation procedure!";
       return;
+    } else if (nWords == 0) {
+      // nothing to do
+      return;
     }
+    const uint8_t* srcAddress = (uint8_t*)mData;
     gsl::span serializedBytes(vecBytes);
-    size_t countBytes = 0;
-    int nSteps = std::get(sReadingLookupTable[nWords]);
-    for (int iStep = 0; iStep < nSteps; iStep++) {
-      memcpy(serializedBytes.data() + std::get(sByteLookupTable[iStep]) + destPos, srcAddress + std::get(sByteLookupTable[iStep]), std::get(sByteLookupTable[iStep]));
-      countBytes += std::get(sByteLookupTable[iStep]);
+    if constexpr (sIsPadded) { // in case of padding one need to use byte map
+      int nSteps = std::get(sReadingLookupTable[nWords]);
+      for (int iStep = 0; iStep < nSteps; iStep++) {
+        memcpy(serializedBytes.data() + std::get(sByteLookupTable[iStep]) + destPos, srcAddress + std::get(sByteLookupTable[iStep]), std::get(sByteLookupTable[iStep]));
+      }
+    } else { // no need in byte map
+      memcpy(serializedBytes.data() + destPos, srcAddress, nBytesToWrite);
     }
-    destPos += nWords * SIZE_WORD;
+    destPos += nBytesToWrite;
   }
 
   void deserialize(const gsl::span inputBytes, size_t nWords, size_t& srcPos)
   {
     mNelements = 0;
-    mNwords = 0;
-    if (nWords < MinNwords || nWords > MaxNwords || inputBytes.size() - srcPos < nWords * SIZE_WORD) {
-      //in case of bad fields responsible for deserialization logic, byte position will be pushed to the end of binary sequence
+    const auto nBytesToRead = nWords * sSizeWord;
+    if ((inputBytes.size() - srcPos) < nBytesToRead || nWords < MinNwords || nWords > MaxNwords) {
+      // in case of bad fields responsible for deserialization logic, byte position will be pushed to the end of binary sequence
+      LOG(error) << "Incomplete payload! |N GBT words " << nWords << " |bytes to read " << nBytesToRead << " |src pos " << srcPos << " |payload size " << inputBytes.size() << " |";
       srcPos = inputBytes.size();
+      RawDataMetric::setStatusBit(mStatusBits, RawDataMetric::EStatusBits::kIncompletePayload);
       mIsIncorrect = true;
       return;
     }
     uint8_t* destAddress = (uint8_t*)mData;
-    size_t countBytes = 0;
-    int nSteps = std::get(sReadingLookupTable[nWords]);
-    mNwords = nWords;
     mNelements = std::get(sReadingLookupTable[nWords]);
-    for (int iStep = 0; iStep < nSteps; iStep++) {
-      memcpy(destAddress + std::get(sByteLookupTable[iStep]), inputBytes.data() + std::get(sByteLookupTable[iStep]) + srcPos, std::get(sByteLookupTable[iStep]));
-      countBytes += std::get(sByteLookupTable[iStep]);
+    if constexpr (sIsPadded) { // in case of padding one need to use byte map
+      int nSteps = std::get(sReadingLookupTable[nWords]);
+      for (int iStep = 0; iStep < nSteps; iStep++) {
+        memcpy(destAddress + std::get(sByteLookupTable[iStep]), inputBytes.data() + std::get(sByteLookupTable[iStep]) + srcPos, std::get(sByteLookupTable[iStep]));
+      }
+    } else { // no need in byte map
+      memcpy(destAddress, inputBytes.data() + srcPos, nBytesToRead);
     }
-    srcPos += mNwords * SIZE_WORD;
+    srcPos += nBytesToRead;
   }
 
-  static constexpr int MaxNwords = Data_t::PayloadSize * Data_t::MaxNelements / Data_t::PayloadPerGBTword + (Data_t::PayloadSize * Data_t::MaxNelements % Data_t::PayloadPerGBTword > 0); //calculating max GBT words per block
+  static constexpr int MaxNwords = Data_t::PayloadSize * Data_t::MaxNelements / Data_t::PayloadPerGBTword + (Data_t::PayloadSize * Data_t::MaxNelements % Data_t::PayloadPerGBTword > 0); // calculating max GBT words per block
   static constexpr int MaxNbytes = SIZE_WORD * MaxNwords;
 
-  static constexpr int MinNwords = Data_t::PayloadSize * Data_t::MinNelements / Data_t::PayloadPerGBTword + (Data_t::PayloadSize * Data_t::MinNelements % Data_t::PayloadPerGBTword > 0); //calculating min GBT words per block
+  static constexpr int MinNwords = Data_t::PayloadSize * Data_t::MinNelements / Data_t::PayloadPerGBTword + (Data_t::PayloadSize * Data_t::MinNelements % Data_t::PayloadPerGBTword > 0); // calculating min GBT words per block
   static constexpr int MinNbytes = SIZE_WORD * MinNwords;
 
-  //get number of byte reading steps
+  // get number of byte reading steps
   static constexpr size_t getNsteps()
   {
     int count = 0;
@@ -197,13 +224,13 @@ struct DataBlockWrapper {
     }
     return count;
   }
-  //enumerator for tuple access:
+  // enumerator for tuple access:
   //[Index] is index of step to read bytes
-  //kNBYTES - number of bytes to read from source and to write into destination
-  //kSRCBYTEPOS - Byte position in the source(binary raw data sequence)
-  //kDESTBYTEPOS - Byte position to write at destionation(memory allocation of T-element array)
-  //kELEMENTINDEX - element index at current step
-  //kWORDINDEX - word index at current step
+  // kNBYTES - number of bytes to read from source and to write into destination
+  // kSRCBYTEPOS - Byte position in the source(binary raw data sequence)
+  // kDESTBYTEPOS - Byte position to write at destionation(memory allocation of T-element array)
+  // kELEMENTINDEX - element index at current step
+  // kWORDINDEX - word index at current step
   //
   enum AccessByteLUT { kNBYTES,
                        kSRCBYTEPOS,
@@ -229,7 +256,7 @@ struct DataBlockWrapper {
     uint64_t indexLastElem = Data_t::MaxNelements - 1;
 
     while (payloadFull > 0) {
-      if (payloadPerElem < payloadInWord) { //new element
+      if (payloadPerElem < payloadInWord) { // new element
         std::get(seqBytes[count]) = payloadPerElem;
         std::get(seqBytes[count]) = srcBytePos;
         std::get(seqBytes[count]) = destBytePosPerElem;
@@ -275,11 +302,11 @@ struct DataBlockWrapper {
   }
   static constexpr std::array, getNsteps()> sByteLookupTable = GetByteLookupTable();
 
-  //enumerator for tuple access:
+  // enumerator for tuple access:
   //[Index] is word index position, i.e. "Index" number of words will be deserialized
-  //kNELEMENTS - number of T elements will be fully deserialized in "Index+1" words
-  //kNSTEPS - number of steps for reading "Index" words
-  //kISPARTED - if one T-element is parted at current word,i.e. current word contains partially deserialized T element at the end of the word
+  // kNELEMENTS - number of T elements will be fully deserialized in "Index+1" words
+  // kNSTEPS - number of steps for reading "Index" words
+  // kISPARTED - if one T-element is parted at current word,i.e. current word contains partially deserialized T element at the end of the word
   enum AccessReadingLUT { kNELEMENTS,
                           kNSTEPS,
                           kISPARTED };
@@ -292,7 +319,7 @@ struct DataBlockWrapper {
     std::get(readingScheme[0]) = false;
     int countWord = 1;
     for (int iStep = 0; iStep < getNsteps(); iStep++) {
-      if (countWord - 1 < std::get((GetByteLookupTable())[iStep])) { //new word
+      if (countWord - 1 < std::get((GetByteLookupTable())[iStep])) { // new word
         std::get(readingScheme[countWord]) = iStep;
         std::get(readingScheme[countWord]) = std::get((GetByteLookupTable())[iStep]);
         if (payloadPerElem > 0) {
@@ -307,7 +334,7 @@ struct DataBlockWrapper {
       }
       payloadPerElem -= std::get((GetByteLookupTable())[iStep]);
     }
-    //Last step checking
+    // Last step checking
     std::get(readingScheme[countWord]) = getNsteps();
     if (payloadPerElem > 0) {
       std::get(readingScheme[countWord]) = true;
@@ -320,7 +347,7 @@ struct DataBlockWrapper {
   }
   static constexpr std::array, MaxNwords + 1> sReadingLookupTable = GetReadingLookupTable();
   //
-  //Printing LookupTables
+  // Printing LookupTables
   static void printLUT()
   {
     LOG(info) << "-------------------------------------------";
@@ -349,56 +376,65 @@ struct DataBlockWrapper {
     }
   }
   Data_t mData[Data_t::MaxNelements];
-  unsigned int mNelements; //number of deserialized elements;
-  unsigned int mNwords;    //number of deserialized GBT words; //can be excluded
+  unsigned int mNelements;                        // number of deserialized elements;
+  typename RawDataMetric::Status_t mStatusBits{}; // Contains status bits
   bool mIsIncorrect;
 };
 
-//CRTP(static polymorphism) + Composition over multiple inheritance(Header + multiple data structures)
-template