Preprocessor.h

#pragma once
 
#include <Foundation/Basics.h>
#include <Foundation/CodeUtils/TokenParseUtils.h>
#include <Foundation/CodeUtils/Tokenizer.h>
#include <Foundation/Containers/Map.h>
#include <Foundation/Containers/Set.h>
#include <Foundation/IO/Stream.h>
#include <Foundation/Logging/Log.h>
#include <Foundation/Memory/CommonAllocators.h>
#include <Foundation/Time/Timestamp.h>
 
class PL_FOUNDATION_DLL plTokenizedFileCache
{
public:
  struct FileData
  {
    plTokenizer m_Tokens;
    plTimestamp m_Timestamp;
  };
 
  plMap<plString, FileData>::ConstIterator Lookup(const plString& sFileName) const;
 
  void Remove(const plString& sFileName);
 
  void Clear();
 
  const plTokenizer* Tokenize(const plString& sFileName, plArrayPtr<const plUInt8> fileContent, const plTimestamp& fileTimeStamp, plLogInterface* pLog);
 
private:
  void SkipWhitespace(plDeque<plToken>& Tokens, plUInt32& uiCurToken);
 
  mutable plMutex m_Mutex;
  plMap<plString, FileData> m_Cache;
};
 
class PL_FOUNDATION_DLL plPreprocessor
{
public:
  enum IncludeType
  {
    MainFile,        
    RelativeInclude, 
    GlobalInclude    
  };
 
  using FileOpenCB = plDelegate<plResult(plStringView, plDynamicArray<plUInt8>&, plTimestamp&)>;
 
  using FileLocatorCB = plDelegate<plResult(plStringView, plStringView, IncludeType, plStringBuilder&)>;
 
  using PassThroughUnknownCmdCB = plDelegate<bool(plStringView)>;
 
  using MacroParameters = plDeque<plTokenParseUtils::TokenStream>;
 
  struct ProcessingEvent
  {
    enum EventType
    {
      BeginExpansion,  
      EndExpansion,    
      Error,           
      Warning,         
      CheckDefined,    
      CheckIfdef,      
      CheckIfndef,     
      EvaluateUnknown, 
      Define,          
      Redefine,        
    };
 
    EventType m_Type = EventType::Error;
 
    const plToken* m_pToken = nullptr;
    plStringView m_sInfo;
  };
 
  plEvent<const ProcessingEvent&> m_ProcessingEvents;
 
  plPreprocessor();
 
  void SetLogInterface(plLogInterface* pLog);
 
  void SetCustomFileCache(plTokenizedFileCache* pFileCache = nullptr);
 
  void SetPassThroughPragma(bool bPassThrough) { m_bPassThroughPragma = bPassThrough; }
 
  void SetPassThroughLine(bool bPassThrough) { m_bPassThroughLine = bPassThrough; }
 
  void SetPassThroughUnknownCmdsCB(PassThroughUnknownCmdCB callback) { m_PassThroughUnknownCmdCB = callback; }
 
  void SetFileOpenFunction(FileOpenCB openAbsFileCB);
 
  void SetFileLocatorFunction(FileLocatorCB locateAbsFileCB);
 
  plResult AddCustomDefine(plStringView sDefinition);
 
  plResult Process(plStringView sMainFile, plTokenParseUtils::TokenStream& ref_tokenOutput);
 
  plResult Process(plStringView sMainFile, plStringBuilder& ref_sOutput, bool bKeepComments = true, bool bRemoveRedundantWhitespace = false, bool bInsertLine = false);
 
 
private:
  struct FileData
  {
    FileData()
    {
      m_iCurrentLine = 1;
      m_iExpandDepth = 0;
    }
 
    plHashedString m_sVirtualFileName;
    plHashedString m_sFileName;
    plInt32 m_iCurrentLine;
    plInt32 m_iExpandDepth;
  };
 
  enum IfDefActivity
  {
    IsActive,
    IsInactive,
    WasActive,
  };
 
  struct CustomDefine
  {
    plHybridArray<plUInt8, 64> m_Content;
    plTokenizer m_Tokenized;
  };
 
  // This class-local allocator is used to get rid of some of the memory allocation
  // tracking that would otherwise occur for allocations made by the preprocessor.
  // If changing its position in the class, make sure it always comes before all
  // other members that depend on it to ensure deallocations in those members
  // happen before the allocator get destroyed.
  plAllocatorWithPolicy<plAllocPolicyHeap, plAllocatorTrackingMode::Nothing> m_ClassAllocator;
 
  bool m_bPassThroughPragma;
  bool m_bPassThroughLine;
  PassThroughUnknownCmdCB m_PassThroughUnknownCmdCB;
 
  // this file cache is used as long as the user does not provide his own
  plTokenizedFileCache m_InternalFileCache;
 
  // pointer to the file cache that is in use
  plTokenizedFileCache* m_pUsedFileCache;
 
  plDeque<FileData> m_CurrentFileStack;
 
  plLogInterface* m_pLog;
 
  plDeque<CustomDefine> m_CustomDefines;
 
  struct IfDefState
  {
    IfDefState(IfDefActivity activeState = IfDefActivity::IsActive)
      : m_ActiveState(activeState)
 
    {
    }
 
    IfDefActivity m_ActiveState;
    bool m_bIsInElseClause = false;
  };
 
  plDeque<IfDefState> m_IfdefActiveStack;
 
  plResult ProcessFile(plStringView sFile, plTokenParseUtils::TokenStream& TokenOutput);
  plResult ProcessCmd(const plTokenParseUtils::TokenStream& Tokens, plTokenParseUtils::TokenStream& TokenOutput);
 
public:
  static plResult DefaultFileLocator(plStringView sCurAbsoluteFile, plStringView sIncludeFile, plPreprocessor::IncludeType incType, plStringBuilder& out_sAbsoluteFilePath);
  static plResult DefaultFileOpen(plStringView sAbsoluteFile, plDynamicArray<plUInt8>& ref_fileContent, plTimestamp& out_fileModification);
 
private: // *** File Handling ***
  plResult OpenFile(plStringView sFile, const plTokenizer** pTokenizer);
 
  FileOpenCB m_FileOpenCallback;
  FileLocatorCB m_FileLocatorCallback;
  plSet<plTempHashedString> m_PragmaOnce;
 
private: // *** Macro Definition ***
  bool RemoveDefine(plStringView sName);
  plResult HandleDefine(const plTokenParseUtils::TokenStream& Tokens, plUInt32& uiCurToken);
 
  struct MacroDefinition
  {
    MacroDefinition();
 
    const plToken* m_MacroIdentifier;
    bool m_bIsFunction;
    bool m_bCurrentlyExpanding;
    bool m_bHasVarArgs;
    plInt32 m_iNumParameters;
    plTokenParseUtils::TokenStream m_Replacement;
  };
 
  plResult StoreDefine(const plToken* pMacroNameToken, const plTokenParseUtils::TokenStream* pReplacementTokens, plUInt32 uiFirstReplacementToken, plInt32 iNumParameters, bool bUsesVarArgs);
  plResult ExtractParameterName(const plTokenParseUtils::TokenStream& Tokens, plUInt32& uiCurToken, plString& sIdentifierName);
 
  plMap<plString256, MacroDefinition> m_Macros;
 
  static constexpr plInt32 s_iMacroParameter0 = plTokenType::ENUM_COUNT + 2;
  static plString s_ParamNames[32];
  plToken m_ParameterTokens[32];
 
private: // *** #if condition parsing ***
  plResult EvaluateCondition(const plTokenParseUtils::TokenStream& Tokens, plUInt32& uiCurToken, plInt64& iResult);
  plResult ParseCondition(const plTokenParseUtils::TokenStream& Tokens, plUInt32& uiCurToken, plInt64& iResult);
  plResult ParseFactor(const plTokenParseUtils::TokenStream& Tokens, plUInt32& uiCurToken, plInt64& iResult);
  plResult ParseExpressionMul(const plTokenParseUtils::TokenStream& Tokens, plUInt32& uiCurToken, plInt64& iResult);
  plResult ParseExpressionOr(const plTokenParseUtils::TokenStream& Tokens, plUInt32& uiCurToken, plInt64& iResult);
  plResult ParseExpressionAnd(const plTokenParseUtils::TokenStream& Tokens, plUInt32& uiCurToken, plInt64& iResult);
  plResult ParseExpressionPlus(const plTokenParseUtils::TokenStream& Tokens, plUInt32& uiCurToken, plInt64& iResult);
  plResult ParseExpressionShift(const plTokenParseUtils::TokenStream& Tokens, plUInt32& uiCurToken, plInt64& iResult);
  plResult ParseExpressionBitOr(const plTokenParseUtils::TokenStream& Tokens, plUInt32& uiCurToken, plInt64& iResult);
  plResult ParseExpressionBitAnd(const plTokenParseUtils::TokenStream& Tokens, plUInt32& uiCurToken, plInt64& iResult);
  plResult ParseExpressionBitXor(const plTokenParseUtils::TokenStream& Tokens, plUInt32& uiCurToken, plInt64& iResult);
 
 
private: // *** Parsing ***
  plResult CopyTokensAndEvaluateDefined(const plTokenParseUtils::TokenStream& Source, plUInt32 uiFirstSourceToken, plTokenParseUtils::TokenStream& Destination);
  void CopyTokensReplaceParams(const plTokenParseUtils::TokenStream& Source, plUInt32 uiFirstSourceToken, plTokenParseUtils::TokenStream& Destination, const plHybridArray<plString, 16>& parameters);
 
  plResult Expect(const plTokenParseUtils::TokenStream& Tokens, plUInt32& uiCurToken, plStringView sToken, plUInt32* pAccepted = nullptr);
  plResult Expect(const plTokenParseUtils::TokenStream& Tokens, plUInt32& uiCurToken, plTokenType::Enum Type, plUInt32* pAccepted = nullptr);
  plResult Expect(const plTokenParseUtils::TokenStream& Tokens, plUInt32& uiCurToken, plStringView sToken1, plStringView sToken2, plUInt32* pAccepted = nullptr);
  plResult ExpectEndOfLine(const plTokenParseUtils::TokenStream& Tokens, plUInt32 uiCurToken);
 
private: // *** Macro Expansion ***
  plResult Expand(const plTokenParseUtils::TokenStream& Tokens, plTokenParseUtils::TokenStream& Output);
  plResult ExpandOnce(const plTokenParseUtils::TokenStream& Tokens, plTokenParseUtils::TokenStream& Output);
  plResult ExpandObjectMacro(MacroDefinition& Macro, plTokenParseUtils::TokenStream& Output, const plToken* pMacroToken);
  plResult ExpandFunctionMacro(MacroDefinition& Macro, const MacroParameters& Parameters, plTokenParseUtils::TokenStream& Output, const plToken* pMacroToken);
  plResult ExpandMacroParam(const plToken& MacroToken, plUInt32 uiParam, plTokenParseUtils::TokenStream& Output, const MacroDefinition& Macro);
  void PassThroughFunctionMacro(MacroDefinition& Macro, const MacroParameters& Parameters, plTokenParseUtils::TokenStream& Output);
  plToken* AddCustomToken(const plToken* pPrevious, const plStringView& sNewText);
  void OutputNotExpandableMacro(MacroDefinition& Macro, plTokenParseUtils::TokenStream& Output);
  plResult ExtractAllMacroParameters(const plTokenParseUtils::TokenStream& Tokens, plUInt32& uiCurToken, plDeque<plTokenParseUtils::TokenStream>& AllParameters);
  plResult ExtractParameterValue(const plTokenParseUtils::TokenStream& Tokens, plUInt32& uiCurToken, plTokenParseUtils::TokenStream& ParamTokens);
 
  plResult InsertParameters(const plTokenParseUtils::TokenStream& Tokens, plTokenParseUtils::TokenStream& Output, const MacroDefinition& Macro);
 
  plResult InsertStringifiedParameters(const plTokenParseUtils::TokenStream& Tokens, plTokenParseUtils::TokenStream& Output, const MacroDefinition& Macro);
  plResult ConcatenateParameters(const plTokenParseUtils::TokenStream& Tokens, plTokenParseUtils::TokenStream& Output, const MacroDefinition& Macro);
  void MergeTokens(const plToken* pFirst, const plToken* pSecond, plTokenParseUtils::TokenStream& Output, const MacroDefinition& Macro);
 
  struct CustomToken
  {
    plToken m_Token;
    plString m_sIdentifierString;
  };
 
  enum TokenFlags : plUInt32
  {
    NoFurtherExpansion = PL_BIT(0),
  };
 
  plToken m_TokenFile;
  plToken m_TokenLine;
  const plToken* m_pTokenOpenParenthesis;
  const plToken* m_pTokenClosedParenthesis;
  const plToken* m_pTokenComma;
 
  plDeque<const MacroParameters*> m_MacroParamStack;
  plDeque<const MacroParameters*> m_MacroParamStackExpanded;
  plDeque<CustomToken> m_CustomTokens;
 
private: // *** Other ***
  static void StringifyTokens(const plTokenParseUtils::TokenStream& Tokens, plStringBuilder& sResult, bool bSurroundWithQuotes);
  plToken* CreateStringifiedParameter(plUInt32 uiParam, const plToken* pParamToken, const MacroDefinition& Macro);
 
  plResult HandleErrorDirective(const plTokenParseUtils::TokenStream& Tokens, plUInt32 uiCurToken, plUInt32 uiDirectiveToken);
  plResult HandleWarningDirective(const plTokenParseUtils::TokenStream& Tokens, plUInt32 uiCurToken, plUInt32 uiDirectiveToken);
  plResult HandleUndef(const plTokenParseUtils::TokenStream& Tokens, plUInt32 uiCurToken, plUInt32 uiDirectiveToken);
 
  plResult HandleEndif(const plTokenParseUtils::TokenStream& Tokens, plUInt32 uiCurToken, plUInt32 uiDirectiveToken);
  plResult HandleElif(const plTokenParseUtils::TokenStream& Tokens, plUInt32 uiCurToken, plUInt32 uiDirectiveToken);
  plResult HandleIf(const plTokenParseUtils::TokenStream& Tokens, plUInt32 uiCurToken, plUInt32 uiDirectiveToken);
  plResult HandleElse(const plTokenParseUtils::TokenStream& Tokens, plUInt32 uiCurToken, plUInt32 uiDirectiveToken);
  plResult HandleIfdef(const plTokenParseUtils::TokenStream& Tokens, plUInt32 uiCurToken, plUInt32 uiDirectiveToken, bool bIsIfdef);
  plResult HandleInclude(const plTokenParseUtils::TokenStream& Tokens, plUInt32 uiCurToken, plUInt32 uiDirectiveToken, plTokenParseUtils::TokenStream& TokenOutput);
  plResult HandleLine(const plTokenParseUtils::TokenStream& Tokens, plUInt32 uiCurToken, plUInt32 uiDirectiveToken, plTokenParseUtils::TokenStream& TokenOutput);
};
 
#define PP_LOG0(Type, FormatStr, ErrorToken)                                                                                                        \
  {                                                                                                                                                 \
    ProcessingEvent pe;                                                                                                                             \
    pe.m_Type = ProcessingEvent::Type;                                                                                                              \
    pe.m_pToken = ErrorToken;                                                                                                                       \
    pe.m_sInfo = FormatStr;                                                                                                                         \
    if (pe.m_pToken->m_uiLine == 0 && pe.m_pToken->m_uiColumn == 0)                                                                                 \
    {                                                                                                                                               \
      const_cast<plToken*>(pe.m_pToken)->m_uiLine = m_CurrentFileStack.PeekBack().m_iCurrentLine;                                                   \
      const_cast<plToken*>(pe.m_pToken)->m_File = m_CurrentFileStack.PeekBack().m_sVirtualFileName;                                                 \
    }                                                                                                                                               \
    m_ProcessingEvents.Broadcast(pe);                                                                                                               \
    plLog::Type(m_pLog, "File '{0}', Line {1} ({2}): " FormatStr, pe.m_pToken->m_File.GetString(), pe.m_pToken->m_uiLine, pe.m_pToken->m_uiColumn); \
  }
 
#define PP_LOG(Type, FormatStr, ErrorToken, ...)                                                                                                       \
  {                                                                                                                                                    \
    ProcessingEvent _pe;                                                                                                                               \
    _pe.m_Type = ProcessingEvent::Type;                                                                                                                \
    _pe.m_pToken = ErrorToken;                                                                                                                         \
    if (_pe.m_pToken->m_uiLine == 0 && _pe.m_pToken->m_uiColumn == 0)                                                                                  \
    {                                                                                                                                                  \
      const_cast<plToken*>(_pe.m_pToken)->m_uiLine = m_CurrentFileStack.PeekBack().m_iCurrentLine;                                                     \
      const_cast<plToken*>(_pe.m_pToken)->m_File = m_CurrentFileStack.PeekBack().m_sVirtualFileName;                                                   \
    }                                                                                                                                                  \
    plStringBuilder sInfo;                                                                                                                             \
    sInfo.SetFormat(FormatStr, ##__VA_ARGS__);                                                                                                         \
    _pe.m_sInfo = sInfo;                                                                                                                               \
    m_ProcessingEvents.Broadcast(_pe);                                                                                                                 \
    plLog::Type(m_pLog, "File '{0}', Line {1} ({2}): {3}", _pe.m_pToken->m_File.GetString(), _pe.m_pToken->m_uiLine, _pe.m_pToken->m_uiColumn, sInfo); \
  }