Tokenizer.h

#pragma once
 
#include <Foundation/Basics.h>
#include <Foundation/Containers/Deque.h>
#include <Foundation/IO/Stream.h>
#include <Foundation/Logging/Log.h>
#include <Foundation/Strings/HashedString.h>
 
struct PL_FOUNDATION_DLL plTokenType
{
  enum Enum
  {
    Unknown,           
    Whitespace,        
    Identifier,        
    NonIdentifier,     
    Newline,           
    LineComment,       
    BlockComment,      
    String1,           
    String2,           
    Integer,           
    Float,             
    RawString1,        
    RawString1Prefix,  
    RawString1Postfix, 
    EndOfFile,         
    ENUM_COUNT,
  };
 
  static const char* EnumNames[ENUM_COUNT];
};
 
struct PL_FOUNDATION_DLL plToken
{
  plToken()
  {
    m_iType = plTokenType::Unknown;
    m_uiLine = 0;
    m_uiColumn = 0;
    m_uiCustomFlags = 0;
  }
 
  plInt32 m_iType;
 
  plUInt32 m_uiLine;
 
  plUInt32 m_uiColumn;
 
  plStringView m_DataView;
 
  plUInt32 m_uiCustomFlags;
 
  plHashedString m_File;
};
 
class PL_FOUNDATION_DLL plTokenizer
{
public:
  plTokenizer(plAllocator* pAllocator = nullptr);
 
  ~plTokenizer();
 
  void Tokenize(plArrayPtr<const plUInt8> data, plLogInterface* pLog, bool bCopyData = true);
 
  const plDeque<plToken>& GetTokens() const { return m_Tokens; }
 
  plDeque<plToken>& GetTokens() { return m_Tokens; }
 
  void GetAllTokens(plDynamicArray<const plToken*>& ref_tokens) const;
 
  void GetAllLines(plDynamicArray<const plToken*>& ref_tokens) const;
 
  plResult GetNextLine(plUInt32& ref_uiFirstToken, plHybridArray<const plToken*, 32>& ref_tokens) const;
 
  plResult GetNextLine(plUInt32& ref_uiFirstToken, plHybridArray<plToken*, 32>& ref_tokens);
 
  const plArrayPtr<const plUInt8> GetTokenizedData() const { return m_Data; }
 
  void SetTreatHashSignAsLineComment(bool bHashSignIsLineComment) { m_bHashSignIsLineComment = bHashSignIsLineComment; }
 
private:
  void NextChar();
  void AddToken();
 
  void HandleUnknown();
  void HandleString(char terminator);
  void HandleRawString();
  void HandleNumber();
  void HandleLineComment();
  void HandleBlockComment();
  void HandleWhitespace();
  void HandleIdentifier();
  void HandleNonIdentifier();
 
  plLogInterface* m_pLog = nullptr;
  plTokenType::Enum m_CurMode = plTokenType::Unknown;
  plStringView m_sIterator;
  plStringView m_sRawStringMarker;
  plUInt32 m_uiCurLine = 1;
  plUInt32 m_uiCurColumn = -1;
  plUInt32 m_uiCurChar = '\0';
  plUInt32 m_uiNextChar = '\0';
 
  plUInt32 m_uiLastLine = 1;
  plUInt32 m_uiLastColumn = 1;
 
  const char* m_szCurCharStart = nullptr;
  const char* m_szNextCharStart = nullptr;
  const char* m_szTokenStart = nullptr;
 
  plDeque<plToken> m_Tokens;
  plDynamicArray<plUInt8> m_Data;
 
  bool m_bHashSignIsLineComment = false;
};