TokenParseUtils.h

#pragma once
 
#include <Foundation/CodeUtils/Tokenizer.h>
#include <Foundation/Containers/HybridArray.h>
 
namespace plTokenParseUtils
{
  using TokenStream = plHybridArray<const plToken*, 32>;
 
  PL_FOUNDATION_DLL void SkipWhitespace(const TokenStream& tokens, plUInt32& ref_uiCurToken); // [tested]
 
  PL_FOUNDATION_DLL void SkipWhitespaceAndNewline(const TokenStream& tokens, plUInt32& ref_uiCurToken); // [tested]
 
  PL_FOUNDATION_DLL bool IsEndOfLine(const TokenStream& tokens, plUInt32 uiCurToken, bool bIgnoreWhitespace); // [tested]
 
  PL_FOUNDATION_DLL void CopyRelevantTokens(const TokenStream& source, plUInt32 uiFirstSourceToken, TokenStream& ref_destination, bool bPreserveNewLines); // [tested]
 
  PL_FOUNDATION_DLL bool Accept(const TokenStream& tokens, plUInt32& ref_uiCurToken, plStringView sToken, plUInt32* pAccepted = nullptr); // [tested]
 
  PL_FOUNDATION_DLL bool Accept(const TokenStream& tokens, plUInt32& ref_uiCurToken, plTokenType::Enum type, plUInt32* pAccepted = nullptr); // [tested]
 
  PL_FOUNDATION_DLL bool Accept(const TokenStream& tokens, plUInt32& ref_uiCurToken, plStringView sToken1, plStringView sToken2, plUInt32* pAccepted = nullptr); // [tested]
 
  PL_FOUNDATION_DLL bool AcceptUnless(const TokenStream& tokens, plUInt32& ref_uiCurToken, plStringView sToken1, plStringView sToken2, plUInt32* pAccepted = nullptr); // [tested]
 
  struct TokenMatch
  {
    TokenMatch(plStringView sToken)
      : m_sToken(sToken)
    {
    }
    TokenMatch(plTokenType::Enum type)
      : m_Type(type)
    {
    }
 
    TokenMatch(plTokenType::Enum type, plStringView sToken)
      : m_Type(type)
      , m_sToken(sToken)
    {
    }
 
    plTokenType::Enum m_Type = plTokenType::Unknown;
    plStringView m_sToken;
  };
 
  PL_FOUNDATION_DLL bool Accept(const TokenStream& tokens, plUInt32& ref_uiCurToken, plArrayPtr<const TokenMatch> matches, plDynamicArray<plUInt32>* pAccepted = nullptr); // [tested]
 
  PL_FOUNDATION_DLL void CombineTokensToString(const TokenStream& tokens, plUInt32 uiCurToken, plStringBuilder& ref_sResult, bool bKeepComments = true, bool bRemoveRedundantWhitespace = false, bool bInsertLine = false);
 
  PL_FOUNDATION_DLL void CombineRelevantTokensToString(const TokenStream& tokens, plUInt32 uiCurToken, plStringBuilder& ref_sResult);
 
  PL_FOUNDATION_DLL void CreateCleanTokenStream(const TokenStream& tokens, plUInt32 uiCurToken, TokenStream& ref_destination);
} // namespace plTokenParseUtils