Allow parsing arbitrary source code fragments.

This commit is contained in:
Bartosz Taudul 2024-06-08 12:55:15 +02:00
parent 16bc01551e
commit ef1c8e8512
No known key found for this signature in database
GPG Key ID: B7FE2008B7575DF3
2 changed files with 40 additions and 22 deletions

View File

@ -60,30 +60,45 @@ void SourceContents::Parse( const char* fileName, const Worker& worker, const Vi
}
}
if( m_file )
if( m_file ) Tokenize( m_data, sz );
}
}
void SourceContents::Parse( const char* source )
{
if( source == m_data ) return;
const size_t len = strlen( source );
m_file = nullptr;
m_fileStringIdx = 0;
m_data = source;
m_dataBuf = nullptr;
m_dataSize = len;
Tokenize( source, len );
}
void SourceContents::Tokenize( const char* txt, size_t sz )
{
Tokenizer tokenizer;
for(;;)
{
auto end = txt;
while( *end != '\n' && *end != '\r' && end - m_data < sz ) end++;
m_lines.emplace_back( Tokenizer::Line { txt, end, tokenizer.Tokenize( txt, end ) } );
if( end - m_data == sz ) break;
if( *end == '\n' )
{
Tokenizer tokenizer;
auto txt = m_data;
for(;;)
{
auto end = txt;
while( *end != '\n' && *end != '\r' && end - m_data < sz ) end++;
m_lines.emplace_back( Tokenizer::Line { txt, end, tokenizer.Tokenize( txt, end ) } );
if( end - m_data == sz ) break;
if( *end == '\n' )
{
end++;
if( end - m_data < sz && *end == '\r' ) end++;
}
else if( *end == '\r' )
{
end++;
if( end - m_data < sz && *end == '\n' ) end++;
}
if( end - m_data == sz ) break;
txt = end;
}
end++;
if( end - m_data < sz && *end == '\r' ) end++;
}
else if( *end == '\r' )
{
end++;
if( end - m_data < sz && *end == '\n' ) end++;
}
if( end - m_data == sz ) break;
txt = end;
}
}

View File

@ -20,6 +20,7 @@ public:
~SourceContents();
void Parse( const char* fileName, const Worker& worker, const View& view );
void Parse( const char* source );
const std::vector<Tokenizer::Line>& get() const { return m_lines; }
bool empty() const { return m_lines.empty(); }
@ -31,6 +32,8 @@ public:
size_t data_size() const { return m_dataSize; }
private:
void Tokenize( const char* txt, size_t sz );
const char* m_file;
uint32_t m_fileStringIdx;