TestTokenizer() : TestFixture("TestTokenizer") {}

Settings settings0;

Settings settings1;

Settings settings2;

Settings settings_windows;

void run() override {

LOAD_LIB_2(settings_windows.library, "windows.cfg");

// If there are unused templates, keep those

settings0.checkUnusedTemplates = true;

settings1.checkUnusedTemplates = true;

settings2.checkUnusedTemplates = true;

settings_windows.checkUnusedTemplates = true;

TEST_CASE(tokenize1);

TEST_CASE(tokenize2);

TEST_CASE(tokenize4);

TEST_CASE(tokenize5);

TEST_CASE(tokenize7);

TEST_CASE(tokenize8);

TEST_CASE(tokenize9);

TEST_CASE(tokenize11);

TEST_CASE(tokenize13); // bailout if the code contains "@" - that is not handled well.

TEST_CASE(tokenize14); // tokenize "0X10" => 16

TEST_CASE(tokenizeHexWithSuffix); // tokenize 0xFFFFFFul

TEST_CASE(tokenize15); // tokenize ".123"

TEST_CASE(tokenize17); // #2759

TEST_CASE(tokenize18); // tokenize "(X&&Y)" into "( X && Y )" instead of "( X & & Y )"

TEST_CASE(tokenize19); // #3006 (segmentation fault)

TEST_CASE(tokenize21); // tokenize 0x0E-7

TEST_CASE(tokenize22); // special marker $ from preprocessor

TEST_CASE(tokenize25); // #4239 (segmentation fault)

TEST_CASE(tokenize26); // #4245 (segmentation fault)

TEST_CASE(tokenize27); // #4525 (segmentation fault)

TEST_CASE(tokenize31); // #3503 (Wrong handling of member function taking function pointer as argument)

TEST_CASE(tokenize32); // #5884 (fsanitize=undefined: left shift of negative value -10000 in lib/templatesimplifier.cpp:852:46)

TEST_CASE(tokenize33); // #5780 Various crashes on valid template code

TEST_CASE(tokenize34); // #8031

TEST_CASE(tokenize35); // #8361

TEST_CASE(tokenize36); // #8436

TEST_CASE(tokenize37); // #8550

TEST_CASE(tokenize38); // #9569

TEST_CASE(tokenize39); // #9771

TEST_CASE(validate);

TEST_CASE(objectiveC); // Syntax error should be written for objective C/C++ code.

TEST_CASE(syntax_case_default);

TEST_CASE(removePragma);

TEST_CASE(foreach); // #3690

TEST_CASE(ifconstexpr);

TEST_CASE(combineOperators);

TEST_CASE(concatenateNegativeNumber);

TEST_CASE(longtok);

TEST_CASE(simplifyHeadersAndUnusedTemplates1);

TEST_CASE(simplifyHeadersAndUnusedTemplates2);

TEST_CASE(simplifyAt);

TEST_CASE(inlineasm);

TEST_CASE(simplifyAsm2); // #4725 (writing asm() around "^{}")

TEST_CASE(ifAddBraces1);

TEST_CASE(ifAddBraces2);

TEST_CASE(ifAddBraces3);

TEST_CASE(ifAddBraces4);

TEST_CASE(ifAddBraces5);

TEST_CASE(ifAddBraces7);

TEST_CASE(ifAddBraces9);

TEST_CASE(ifAddBraces11);

TEST_CASE(ifAddBraces12);

TEST_CASE(ifAddBraces13);

TEST_CASE(ifAddBraces15); // #2616 - unknown macro before if

TEST_CASE(ifAddBraces16);

TEST_CASE(ifAddBraces17); // '} else' should be in the same line

TEST_CASE(ifAddBraces18); // #3424 - if if { } else else

TEST_CASE(ifAddBraces19); // #3928 - if for if else

TEST_CASE(ifAddBraces20); // #5012 - syntax error 'else }'

TEST_CASE(ifAddBracesLabels); // #5332 - if (x) label: {} ..

TEST_CASE(switchAddBracesLabels);

TEST_CASE(whileAddBraces);

TEST_CASE(whileAddBracesLabels);

TEST_CASE(doWhileAddBraces);

TEST_CASE(doWhileAddBracesLabels);

TEST_CASE(forAddBraces1);

TEST_CASE(forAddBraces2); // #5088

TEST_CASE(forAddBracesLabels);

TEST_CASE(simplifyExternC);

TEST_CASE(simplifyKeyword); // #5842 - remove C99 static keyword between []

TEST_CASE(isOneNumber);

TEST_CASE(isTwoNumber);

TEST_CASE(simplifyFunctionParameters);

TEST_CASE(simplifyFunctionParameters1); // #3721

TEST_CASE(simplifyFunctionParameters2); // #4430

TEST_CASE(simplifyFunctionParameters3); // #4436

TEST_CASE(simplifyFunctionParameters4); // #9421

TEST_CASE(simplifyFunctionParametersMultiTemplate);

TEST_CASE(simplifyFunctionParametersErrors);

TEST_CASE(simplifyFunctionTryCatch);

TEST_CASE(removeParentheses1); // Ticket #61

TEST_CASE(removeParentheses3);

TEST_CASE(removeParentheses4); // Ticket #390

TEST_CASE(removeParentheses5); // Ticket #392

TEST_CASE(removeParentheses6);

TEST_CASE(removeParentheses7);

TEST_CASE(removeParentheses8); // Ticket #1865

TEST_CASE(removeParentheses9); // Ticket #1962

TEST_CASE(removeParentheses10); // Ticket #2320

TEST_CASE(removeParentheses11); // Ticket #2505

TEST_CASE(removeParentheses12); // Ticket #2760 ',(b)='

TEST_CASE(removeParentheses13);

TEST_CASE(removeParentheses14); // Ticket #3309

TEST_CASE(removeParentheses15); // Ticket #4142

TEST_CASE(removeParentheses16); // Ticket #4423 '*(x.y)='

TEST_CASE(removeParentheses17); // Don't remove parentheses in 'a ? b : (c>0 ? d : e);'

TEST_CASE(removeParentheses18); // 'float(*a)[2]' => 'float *a[2]'

TEST_CASE(removeParentheses19); // ((typeof(x) *)0)

TEST_CASE(removeParentheses20); // Ticket #5479: a<b<int>>(2);

TEST_CASE(removeParentheses21); // Don't "simplify" casts

TEST_CASE(removeParentheses22);

TEST_CASE(removeParentheses23); // Ticket #6103 - Infinite loop upon valid input

TEST_CASE(removeParentheses24); // Ticket #7040

TEST_CASE(removeParentheses25); // daca@home - a=(b,c)

TEST_CASE(removeParentheses26); // Ticket #8875 a[0](0)

TEST_CASE(tokenize_double);

TEST_CASE(tokenize_strings);

TEST_CASE(simplifyStructDecl);

TEST_CASE(vardecl1);

TEST_CASE(vardecl2);

TEST_CASE(vardecl3);

TEST_CASE(vardecl4);

TEST_CASE(vardecl5); // #7048

TEST_CASE(vardec_static);

TEST_CASE(vardecl6);

TEST_CASE(vardecl7);

TEST_CASE(vardecl8);

TEST_CASE(vardecl9);

TEST_CASE(vardecl10);

TEST_CASE(vardecl11);

TEST_CASE(vardecl12);

TEST_CASE(vardecl13);

TEST_CASE(vardecl14);

TEST_CASE(vardecl15);

TEST_CASE(vardecl16);

TEST_CASE(vardecl17);

TEST_CASE(vardecl18);

TEST_CASE(vardecl19);

TEST_CASE(vardecl20); // #3700 - register const int H = 0;

TEST_CASE(vardecl21); // #4042 - a::b const *p = 0;

TEST_CASE(vardecl22); // #4211 - segmentation fault

TEST_CASE(vardecl23); // #4276 - segmentation fault

TEST_CASE(vardecl24); // #4187 - variable declaration within lambda function

TEST_CASE(vardecl25); // #4799 - segmentation fault

TEST_CASE(vardecl26); // #5907 - incorrect handling of extern declarations

TEST_CASE(vardecl27); // #7850 - crash on valid C code

TEST_CASE(vardecl28);

TEST_CASE(vardecl_stl_1);

TEST_CASE(vardecl_stl_2);

TEST_CASE(vardecl_stl_3);

TEST_CASE(vardecl_template_1);

TEST_CASE(vardecl_template_2);

TEST_CASE(vardecl_union);

TEST_CASE(vardecl_par); // #2743 - set links if variable type contains parentheses

TEST_CASE(vardecl_par2); // #3912 - set correct links

TEST_CASE(vardecl_par3); // #6556 - Fred x1(a), x2(b);

TEST_CASE(vardecl_class_ref);

TEST_CASE(volatile_variables);

// unsigned i; => unsigned int i;

TEST_CASE(implicitIntConst);

TEST_CASE(implicitIntExtern);

TEST_CASE(implicitIntSigned1);

TEST_CASE(implicitIntUnsigned1);

TEST_CASE(implicitIntUnsigned2);

TEST_CASE(implicitIntUnsigned3); // template arguments

TEST_CASE(simplifyStdType); // #4947, #4950, #4951

TEST_CASE(createLinks);

TEST_CASE(createLinks2);

TEST_CASE(simplifyString);

TEST_CASE(simplifyConst);

TEST_CASE(switchCase);

TEST_CASE(simplifyPointerToStandardType);

TEST_CASE(simplifyFunctionPointers1);

TEST_CASE(simplifyFunctionPointers2);

TEST_CASE(simplifyFunctionPointers3);

TEST_CASE(simplifyFunctionPointers4);

TEST_CASE(simplifyFunctionPointers5);

TEST_CASE(simplifyFunctionPointers6);

TEST_CASE(simplifyFunctionPointers7);

TEST_CASE(simplifyFunctionPointers8); // #7410 - throw

TEST_CASE(simplifyFunctionPointers9); // #6113 - function call with function pointer

TEST_CASE(removedeclspec);

TEST_CASE(removeattribute);

TEST_CASE(functionAttributeBefore1);

TEST_CASE(functionAttributeBefore2);

TEST_CASE(functionAttributeBefore3);

TEST_CASE(functionAttributeBefore4);

TEST_CASE(functionAttributeAfter1);

TEST_CASE(functionAttributeAfter2);

TEST_CASE(functionAttributeListBefore);

TEST_CASE(functionAttributeListAfter);

TEST_CASE(splitTemplateRightAngleBrackets);

TEST_CASE(cpp03template1);

TEST_CASE(cpp0xtemplate1);

TEST_CASE(cpp0xtemplate2);

TEST_CASE(cpp0xtemplate3);

TEST_CASE(cpp0xtemplate4); // Ticket #6181: Mishandled C++11 syntax

TEST_CASE(cpp0xtemplate5); // Ticket #9154 change >> to > >

TEST_CASE(cpp14template); // Ticket #6708

TEST_CASE(arraySize);

TEST_CASE(labels);

TEST_CASE(simplifyInitVar);

TEST_CASE(simplifyInitVar2);

TEST_CASE(simplifyInitVar3);

TEST_CASE(bitfields1);

TEST_CASE(bitfields2);

TEST_CASE(bitfields3);

TEST_CASE(bitfields4); // ticket #1956

TEST_CASE(bitfields5); // ticket #1956

TEST_CASE(bitfields6); // ticket #2595

TEST_CASE(bitfields7); // ticket #1987

TEST_CASE(bitfields8);

TEST_CASE(bitfields9); // ticket #2706

TEST_CASE(bitfields10);

TEST_CASE(bitfields12); // ticket #3485 (segmentation fault)

TEST_CASE(bitfields13); // ticket #3502 (segmentation fault)

TEST_CASE(bitfields14); // ticket #4561 (segfault for 'class a { signals: };')

TEST_CASE(bitfields15); // ticket #7747 (enum Foo {A,B}:4;)

TEST_CASE(bitfields16); // Save bitfield bit count

TEST_CASE(simplifyNamespaceStd);

TEST_CASE(microsoftMemory);

TEST_CASE(microsoftString);

TEST_CASE(borland);

TEST_CASE(simplifyQtSignalsSlots1);

TEST_CASE(simplifyQtSignalsSlots2);

TEST_CASE(simplifySQL);

TEST_CASE(simplifyCAlternativeTokens);

// x = ({ 123; }); => { x = 123; }

TEST_CASE(simplifyRoundCurlyParentheses);

TEST_CASE(simplifyOperatorName1);

TEST_CASE(simplifyOperatorName2);

TEST_CASE(simplifyOperatorName3);

TEST_CASE(simplifyOperatorName4);

TEST_CASE(simplifyOperatorName5);

TEST_CASE(simplifyOperatorName6); // ticket #3194

TEST_CASE(simplifyOperatorName7); // ticket #4619

TEST_CASE(simplifyOperatorName8); // ticket #5706

TEST_CASE(simplifyOperatorName9); // ticket #5709 - comma operator not properly tokenized

TEST_CASE(simplifyOperatorName10); // #8746 - using a::operator=

TEST_CASE(simplifyOperatorName11); // #8889

TEST_CASE(simplifyOperatorName12); // #9110

TEST_CASE(simplifyOperatorName13); // user defined literal

TEST_CASE(simplifyOperatorName14); // std::complex operator "" if

TEST_CASE(simplifyOperatorName15); // ticket #9468 syntaxError

TEST_CASE(simplifyOperatorName16); // ticket #9472

TEST_CASE(simplifyOperatorName17);

TEST_CASE(simplifyOperatorName18); // global namespace

TEST_CASE(simplifyOperatorName19);

TEST_CASE(simplifyOperatorName20);

TEST_CASE(simplifyOperatorName21);

TEST_CASE(simplifyOperatorName22);

TEST_CASE(simplifyOperatorName23);

TEST_CASE(simplifyOperatorName24);

TEST_CASE(simplifyOperatorName25);

TEST_CASE(simplifyOperatorName26);

TEST_CASE(simplifyOperatorName27);

TEST_CASE(simplifyOperatorName28);

TEST_CASE(simplifyOperatorName29); // spaceship operator

TEST_CASE(simplifyOperatorName31); // #6342

TEST_CASE(simplifyOperatorName32); // #10256

TEST_CASE(simplifyOverloadedOperators1);

TEST_CASE(simplifyOverloadedOperators2); // (*this)(123)

TEST_CASE(simplifyOverloadedOperators3); // #9881 - hang

TEST_CASE(simplifyNullArray);

// Some simple cleanups of unhandled macros in the global scope

TEST_CASE(removeMacrosInGlobalScope);

TEST_CASE(removeMacroInVarDecl);

TEST_CASE(addSemicolonAfterUnknownMacro);

// a = b = 0;

TEST_CASE(multipleAssignment);

TEST_CASE(platformWin32A);

TEST_CASE(platformWin32W);

TEST_CASE(platformWin32AStringCat); // ticket #5015

TEST_CASE(platformWin32WStringCat); // ticket #5015

TEST_CASE(platformWinWithNamespace);

TEST_CASE(simplifyStaticConst);

TEST_CASE(simplifyCPPAttribute);

TEST_CASE(simplifyCaseRange);

TEST_CASE(simplifyEmptyNamespaces);

TEST_CASE(compileLimits); // #5592 crash: gcc: testsuit: gcc.c-torture/compile/limits-declparen.c

TEST_CASE(prepareTernaryOpForAST);

// AST data

TEST_CASE(astexpr);

TEST_CASE(astexpr2); // limit large expressions

TEST_CASE(astpar);

TEST_CASE(astnewdelete);

TEST_CASE(astbrackets);

TEST_CASE(astunaryop);

TEST_CASE(astfunction);

TEST_CASE(asttemplate);

TEST_CASE(astcast);

TEST_CASE(astlambda);

TEST_CASE(astcase);

TEST_CASE(astrefqualifier);

TEST_CASE(astvardecl);

TEST_CASE(astnewscoped);

TEST_CASE(startOfExecutableScope);

TEST_CASE(removeMacroInClassDef); // #6058

TEST_CASE(sizeofAddParentheses);

TEST_CASE(reportUnknownMacros);

// Make sure the Tokenizer::findGarbageCode() does not have false positives

// The TestGarbage ensures that there are true positives

TEST_CASE(findGarbageCode);

TEST_CASE(checkEnableIf);

TEST_CASE(checkTemplates);

TEST_CASE(checkNamespaces);

TEST_CASE(checkLambdas);

TEST_CASE(checkIfCppCast);

TEST_CASE(checkRefQualifiers);

TEST_CASE(checkConditionBlock);

TEST_CASE(checkUnknownCircularVar);

// #9052

TEST_CASE(noCrash1);

TEST_CASE(noCrash2);

TEST_CASE(noCrash3);

TEST_CASE(noCrash4);

// --check-config

TEST_CASE(checkConfiguration);

TEST_CASE(unknownType); // #8952

TEST_CASE(unknownMacroBeforeReturn);

TEST_CASE(cppcast);

TEST_CASE(checkHeader1);

TEST_CASE(removeExtraTemplateKeywords);

TEST_CASE(removeAlignas1);

TEST_CASE(removeAlignas2); // Do not remove alignof in the same way

TEST_CASE(simplifyCoroutines);

TEST_CASE(simplifySpaceshipOperator);

TEST_CASE(simplifyIfSwitchForInit1);

TEST_CASE(simplifyIfSwitchForInit2);

TEST_CASE(simplifyIfSwitchForInit3);

TEST_CASE(simplifyIfSwitchForInit4);

TEST_CASE(simplifyIfSwitchForInit5);

TEST_CASE(cpp20_default_bitfield_initializer);

}

#define tokenizeAndStringify(...) tokenizeAndStringify_(__FILE__, __LINE__, __VA_ARGS__)

std::string tokenizeAndStringify_(const char* file, int linenr, const char code[], bool expand = true, Settings::PlatformType platform = Settings::Native, const char* filename = "test.cpp", bool cpp11 = true) {

errout.str("");

settings1.debugwarnings = true;

settings1.platform(platform);

settings1.standards.cpp = cpp11 ? Standards::CPP11 : Standards::CPP03;

// tokenize..

Tokenizer tokenizer(&settings1, this);

std::istringstream istr(code);

ASSERT_LOC(tokenizer.tokenize(istr, filename), file, linenr);

// filter out ValueFlow messages..

const std::string debugwarnings = errout.str();

errout.str("");

std::istringstream istr2(debugwarnings);

std::string line;

while (std::getline(istr2,line)) {

if (line.find("valueflow.cpp") == std::string::npos)

errout << line << "\n";

}

if (tokenizer.tokens())

return tokenizer.tokens()->stringifyList(false, expand, false, true, false, nullptr, nullptr);

else

return "";

}

#define tokenizeAndStringifyWindows(...) tokenizeAndStringifyWindows_(__FILE__, __LINE__, __VA_ARGS__)

std::string tokenizeAndStringifyWindows_(const char* file, int linenr, const char code[], bool expand = true, Settings::PlatformType platform = Settings::Native, const char* filename = "test.cpp", bool cpp11 = true) {

errout.str("");

settings_windows.debugwarnings = true;

settings_windows.platform(platform);

settings_windows.standards.cpp = cpp11 ? Standards::CPP11 : Standards::CPP03;

// tokenize..

Tokenizer tokenizer(&settings_windows, this);

std::istringstream istr(code);

ASSERT_LOC(tokenizer.tokenize(istr, filename), file, linenr);

// filter out ValueFlow messages..

const std::string debugwarnings = errout.str();

errout.str("");

std::istringstream istr2(debugwarnings);

std::string line;

while (std::getline(istr2,line)) {

if (line.find("valueflow.cpp") == std::string::npos)

errout << line << "\n";

}

if (tokenizer.tokens())

return tokenizer.tokens()->stringifyList(false, expand, false, true, false, nullptr, nullptr);

else

return "";

}

std::string tokenizeAndStringify_(const char* file, int line, const char code[], const Settings &settings, const char filename[] = "test.cpp") {

errout.str("");

// tokenize..

Tokenizer tokenizer(&settings, this);

std::istringstream istr(code);

ASSERT_LOC(tokenizer.tokenize(istr, filename), file, line);

if (!tokenizer.tokens())

return "";

return tokenizer.tokens()->stringifyList(false, true, false, true, false, nullptr, nullptr);

}

#define tokenizeDebugListing(...) tokenizeDebugListing_(__FILE__, __LINE__, __VA_ARGS__)

std::string tokenizeDebugListing_(const char* file, int line, const char code[], const char filename[] = "test.cpp") {

errout.str("");

settings2.standards.c = Standards::C89;

settings2.standards.cpp = Standards::CPP03;

Tokenizer tokenizer(&settings2, this);

std::istringstream istr(code);

ASSERT_LOC(tokenizer.tokenize(istr, filename), file, line);

// result..

return tokenizer.tokens()->stringifyList(true,true,true,true,false);

}

void tokenize1() {

const char code[] = "void f ( )\n"

"{ if ( p . y ( ) > yof ) { } }";

ASSERT_EQUALS(code, tokenizeAndStringify(code));

}

void tokenize2() {

const char code[] = "{ sizeof a, sizeof b }";

ASSERT_EQUALS("{ sizeof ( a ) , sizeof ( b ) }", tokenizeAndStringify(code));

}

void tokenize4() {

const char code[] = "class foo\n"

"{\n"

"public:\n"

" const int i;\n"

"}";

ASSERT_EQUALS("class foo\n"

"{\n"

"public:\n"

"const int i ;\n"

"}", tokenizeAndStringify(code));

ASSERT_EQUALS("", errout.str());

}

void tokenize5() {

// Tokenize values

ASSERT_EQUALS("; + 1E3 ;", tokenizeAndStringify("; +1E3 ;"));

ASSERT_EQUALS("; 1E-2 ;", tokenizeAndStringify("; 1E-2 ;"));

}

void tokenize7() {

const char code[] = "void f() {\n"

" int x1 = 1;\n"

" int x2(x1);\n"

"}\n";

ASSERT_EQUALS("void f ( ) {\nint x1 ; x1 = 1 ;\nint x2 ; x2 = x1 ;\n}",

tokenizeAndStringify(code));

}

void tokenize8() {

const char code[] = "void f() {\n"

" int x1(g());\n"

" int x2(x1);\n"

"}\n";

ASSERT_EQUALS("1: void f ( ) {\n"

"2: int x1@1 ; x1@1 = g ( ) ;\n"

"3: int x2@2 ; x2@2 = x1@1 ;\n"

"4: }\n",

tokenizeDebugListing(code));

}

void tokenize9() {

const char code[] = "typedef void (*fp)();\n"

"typedef fp (*fpp)();\n"

"void f() {\n"

" fpp x = (fpp)f();\n"

"}";

tokenizeAndStringify(code);

ASSERT_EQUALS("", errout.str());

}

void tokenize11() {

ASSERT_EQUALS("X * sizeof ( Y ( ) ) ;", tokenizeAndStringify("X * sizeof(Y());"));

}

// bailout if there is "@" - it is not handled well

void tokenize13() {

const char code[] = "@implementation\n"

"-(Foo *)foo: (Bar *)bar\n"

"{ }\n"

"@end\n";

ASSERT_THROW(tokenizeAndStringify(code), InternalError);

}

// Ticket #2361: 0X10 => 16

void tokenize14() {

ASSERT_EQUALS("; 0x10 ;", tokenizeAndStringify(";0x10;"));

ASSERT_EQUALS("; 0X10 ;", tokenizeAndStringify(";0X10;"));

ASSERT_EQUALS("; 0444 ;", tokenizeAndStringify(";0444;"));

}

// Ticket #8050

void tokenizeHexWithSuffix() {

ASSERT_EQUALS("; 0xFFFFFF ;", tokenizeAndStringify(";0xFFFFFF;"));

ASSERT_EQUALS("; 0xFFFFFFu ;", tokenizeAndStringify(";0xFFFFFFu;"));

ASSERT_EQUALS("; 0xFFFFFFul ;", tokenizeAndStringify(";0xFFFFFFul;"));

// Number of digits decides about internal representation...

ASSERT_EQUALS("; 0xFFFFFFFF ;", tokenizeAndStringify(";0xFFFFFFFF;"));

ASSERT_EQUALS("; 0xFFFFFFFFu ;", tokenizeAndStringify(";0xFFFFFFFFu;"));

ASSERT_EQUALS("; 0xFFFFFFFFul ;", tokenizeAndStringify(";0xFFFFFFFFul;"));

}

// Ticket #2429: 0.125

void tokenize15() {

ASSERT_EQUALS("0.125 ;", tokenizeAndStringify(".125;"));

ASSERT_EQUALS("005.125 ;", tokenizeAndStringify("005.125;")); // Don't confuse with octal values

}

void tokenize17() { // #2759

ASSERT_EQUALS("class B : private :: A { } ;", tokenizeAndStringify("class B : private ::A { };"));

}

void tokenize18() { // tokenize "(X&&Y)" into "( X && Y )" instead of "( X & & Y )"

ASSERT_EQUALS("( X && Y ) ;", tokenizeAndStringify("(X&&Y);"));

}

void tokenize19() {

// #3006 - added hasComplicatedSyntaxErrorsInTemplates to avoid segmentation fault

ASSERT_THROW(tokenizeAndStringify("x < () <"), InternalError);

// #3496 - make sure hasComplicatedSyntaxErrorsInTemplates works

ASSERT_EQUALS("void a ( Fred * f ) { for ( ; n < f . x ( ) ; ) { } }",

tokenizeAndStringify("void a(Fred* f) MACRO { for (;n < f->x();) {} }"));

// #6216 - make sure hasComplicatedSyntaxErrorsInTemplates works

ASSERT_EQUALS("C :: C ( )\n"

": v { }\n"

"{\n"

"for ( int dim = 0 ; dim < v . size ( ) ; ++ dim ) {\n"

"v [ dim ] . f ( ) ;\n"

"}\n"

"} ;",

tokenizeAndStringify("C::C()\n"

":v{}\n"

"{\n"

" for (int dim = 0; dim < v.size(); ++dim) {\n"

" v[dim]->f();\n"

" }\n"

"};"));

}

void tokenize21() { // tokenize 0x0E-7

ASSERT_EQUALS("0x0E - 7 ;", tokenizeAndStringify("0x0E-7;"));

}

void tokenize22() { // tokenize special marker $ from preprocessor

ASSERT_EQUALS("a$b", tokenizeAndStringify("a$b"));

ASSERT_EQUALS("a $b\nc", tokenizeAndStringify("a $b\nc"));

ASSERT_EQUALS("a = $0 ;", tokenizeAndStringify("a = $0;"));

ASSERT_EQUALS("a$ ++ ;", tokenizeAndStringify("a$++;"));

ASSERT_EQUALS("$if ( ! p )", tokenizeAndStringify("$if(!p)"));

}

// #4239 - segfault for "f ( struct { int typedef T x ; } ) { }"

void tokenize25() {

ASSERT_THROW(tokenizeAndStringify("f ( struct { int typedef T x ; } ) { }"), InternalError);

}

// #4245 - segfault

void tokenize26() {

ASSERT_THROW(tokenizeAndStringify("class x { protected : template < int y = } ;"), InternalError); // Garbage code

}

void tokenize27() {

// #4525 - segfault

tokenizeAndStringify("struct except_spec_d_good : except_spec_a, except_spec_b {\n"

"~except_spec_d_good();\n"

"};\n"

"struct S { S(); };\n"

"S::S() __attribute((pure)) = default;"

);

// original code: glibc-2.18/posix/bug-regex20.c

tokenizeAndStringify("static unsigned int re_string_context_at (const re_string_t *input, int idx, int eflags) internal_function __attribute__ ((pure));");

}

// #3503 - don't "simplify" SetFunction member function to a variable

void tokenize31() {

ASSERT_EQUALS("struct TTestClass { TTestClass ( ) { }\n"

"void SetFunction ( Other ( * m_f ) ( ) ) { }\n"

"} ;",

tokenizeAndStringify("struct TTestClass { TTestClass() { }\n"

" void SetFunction(Other(*m_f)()) { }\n"

"};"));

ASSERT_EQUALS("struct TTestClass { TTestClass ( ) { }\n"

"void SetFunction ( Other ( * m_f ) ( ) ) ;\n"

"} ;",

tokenizeAndStringify("struct TTestClass { TTestClass() { }\n"

" void SetFunction(Other(*m_f)());\n"

"};"));

}

// #5884 - Avoid left shift of negative integer value.

void tokenize32() {

// Do not simplify negative integer left shifts.

const char * code = "void f ( ) { int max_x ; max_x = -10000 << 16 ; }";

ASSERT_EQUALS(code, tokenizeAndStringify(code));

}

// #5780 Various crashes on valid template code in Tokenizer::setVarId()

void tokenize33() {

const char * code = "template<typename T, typename A = Alloc<T>> struct vector {};\n"

"void z() {\n"

" vector<int> VI;\n"

"}\n";

tokenizeAndStringify(code);

}

void tokenize34() { // #8031

{

const char code[] = "struct Container {\n"

" Container();\n"

" int* mElements;\n"

"};\n"

"Container::Container() : mElements(nullptr) {}\n"

"Container intContainer;";

const char exp[] = "1: struct Container {\n"

"2: Container ( ) ;\n"

"3: int * mElements@1 ;\n"

"4: } ;\n"

"5: Container :: Container ( ) : mElements@1 ( nullptr ) { }\n"

"6: Container intContainer@2 ;\n";

ASSERT_EQUALS(exp, tokenizeDebugListing(code));

}

{

const char code[] = "template<class T> struct Container {\n"

" Container();\n"

" int* mElements;\n"

"};\n"

"template <class T> Container<T>::Container() : mElements(nullptr) {}\n"

"Container<int> intContainer;";

const char exp[] = "1: struct Container<int> ;\n"

"2:\n"

"|\n"

"5:\n"

"6: Container<int> intContainer@1 ;\n"

"1: struct Container<int> {\n"

"2: Container<int> ( ) ;\n"

"3: int * mElements@2 ;\n"

"4: } ;\n"

"5: Container<int> :: Container<int> ( ) : mElements@2 ( nullptr ) { }\n";

ASSERT_EQUALS(exp, tokenizeDebugListing(code));

}

}

void tokenize35() { // #8361

tokenizeAndStringify("typedef int CRCWord; "

"template<typename T> ::CRCWord const Compute(T const t) { return 0; }");

}

void tokenize36() { // #8436

const char code[] = "int foo ( int i ) { return i ? * new int { 5 } : int { i ? 0 : 1 } ; }";

ASSERT_EQUALS(code, tokenizeAndStringify(code));

}

void tokenize37() { // #8550

const char codeC[] = "class name { public: static void init ( ) {} } ; "

"typedef class name N; "

"void foo ( ) { return N :: init ( ) ; }";

const char expC[] = "class name { public: static void init ( ) { } } ; "

"void foo ( ) { return name :: init ( ) ; }";

ASSERT_EQUALS(expC, tokenizeAndStringify(codeC));

const char codeS[] = "class name { public: static void init ( ) {} } ; "

"typedef struct name N; "

"void foo ( ) { return N :: init ( ) ; }";

const char expS[] = "class name { public: static void init ( ) { } } ; "

"void foo ( ) { return name :: init ( ) ; }";

ASSERT_EQUALS(expS, tokenizeAndStringify(codeS));

}

void tokenize38() { // #9569

const char code[] = "using Binary = std::vector<char>; enum Type { Binary };";

const char exp[] = "enum Type { Binary } ;";

ASSERT_EQUALS(exp, tokenizeAndStringify(code));

}

void tokenize39() { // #9771

const char code[] = "template <typename T> class Foo;"

"template <typename T> bool operator!=(const Foo<T> &, const Foo<T> &);"

"template <typename T> class Foo { friend bool operator!= <> (const Foo<T> &, const Foo<T> &); };";

const char exp[] = "template < typename T > class Foo ; "

"template < typename T > bool operator!= ( const Foo < T > & , const Foo < T > & ) ; "

"template < typename T > class Foo { friend bool operator!= < > ( const Foo < T > & , const Foo < T > & ) ; } ;";

ASSERT_EQUALS(exp, tokenizeAndStringify(code));

}

void validate() {

// C++ code in C file

ASSERT_THROW(tokenizeAndStringify(";using namespace std;",false,Settings::Native,"test.c"), InternalError);

ASSERT_THROW(tokenizeAndStringify(";std::map<int,int> m;",false,Settings::Native,"test.c"), InternalError);

ASSERT_THROW(tokenizeAndStringify(";template<class T> class X { };",false,Settings::Native,"test.c"), InternalError);

ASSERT_THROW(tokenizeAndStringify("int X<Y>() {};",false,Settings::Native,"test.c"), InternalError);

ASSERT_THROW(tokenizeAndStringify("void foo(int i) { reinterpret_cast<char>(i) };",false,Settings::Native,"test.h"), InternalError);

}

void objectiveC() {

ASSERT_THROW(tokenizeAndStringify("void f() { [foo bar]; }"), InternalError);

}

void syntax_case_default() { // correct syntax

tokenizeAndStringify("void f() {switch (n) { case 0: z(); break;}}");

ASSERT_EQUALS("", errout.str());

tokenizeAndStringify("void f() {switch (n) { case 0:; break;}}");

ASSERT_EQUALS("", errout.str());

// TODO: Do not throw AST validation exception

TODO_ASSERT_THROW(tokenizeAndStringify("void f() {switch (n) { case 0?1:2 : z(); break;}}"), InternalError);

//ASSERT_EQUALS("", errout.str());

// TODO: Do not throw AST validation exception

TODO_ASSERT_THROW(tokenizeAndStringify("void f() {switch (n) { case 0?(1?3:4):2 : z(); break;}}"), InternalError);

ASSERT_EQUALS("", errout.str());

//allow GCC '({ %name%|%num%|%bool% ; })' statement expression extension

// TODO: Do not throw AST validation exception

TODO_ASSERT_THROW(tokenizeAndStringify("void f() {switch (n) { case 0?({0;}):1: z(); break;}}"), InternalError);

ASSERT_EQUALS("", errout.str());

//'b' can be or a macro or an undefined enum

tokenizeAndStringify("void f() {switch (n) { case b: z(); break;}}");

ASSERT_EQUALS("", errout.str());

//valid, when there's this declaration: 'constexpr int g() { return 2; }'

tokenizeAndStringify("void f() {switch (n) { case g(): z(); break;}}");

ASSERT_EQUALS("", errout.str());

//valid, when there's also this declaration: 'constexpr int g[1] = {0};'

tokenizeAndStringify("void f() {switch (n) { case g[0]: z(); break;}}");

ASSERT_EQUALS("", errout.str());

//valid, similar to above case

tokenizeAndStringify("void f() {switch (n) { case *g: z(); break;}}");

ASSERT_EQUALS("", errout.str());

//valid, when 'x' and 'y' are constexpr.

tokenizeAndStringify("void f() {switch (n) { case sqrt(x+y): z(); break;}}");

ASSERT_EQUALS("", errout.str());

}

void removePragma() {

const char code[] = "_Pragma(\"abc\") int x;";

Settings s;

s.standards.c = Standards::C89;

ASSERT_EQUALS("_Pragma ( \"abc\" ) int x ;", tokenizeAndStringify(code, s, "test.c"));

s.standards.c = Standards::CLatest;

ASSERT_EQUALS("int x ;", tokenizeAndStringify(code, s, "test.c"));

s.standards.cpp = Standards::CPP03;

ASSERT_EQUALS("_Pragma ( \"abc\" ) int x ;", tokenizeAndStringify(code, s, "test.cpp"));

s.standards.cpp = Standards::CPPLatest;

ASSERT_EQUALS("int x ;", tokenizeAndStringify(code, s, "test.cpp"));

}

void foreach () {

// #3690,#5154

const char code[] ="void f() { for each ( char c in MyString ) { Console::Write(c); } }";

ASSERT_EQUALS("void f ( ) { asm ( \"char c in MyString\" ) { Console :: Write ( c ) ; } }", tokenizeAndStringify(code));

}

void ifconstexpr() {

ASSERT_EQUALS("void f ( ) { if ( FOO ) { bar ( c ) ; } }", tokenizeAndStringify("void f() { if constexpr ( FOO ) { bar(c); } }"));

}

void combineOperators() {

ASSERT_EQUALS("; private: ;", tokenizeAndStringify(";private:;"));

ASSERT_EQUALS("; protected: ;", tokenizeAndStringify(";protected:;"));

ASSERT_EQUALS("; public: ;", tokenizeAndStringify(";public:;"));

ASSERT_EQUALS("; __published: ;", tokenizeAndStringify(";__published:;"));

ASSERT_EQUALS("a . public : ;", tokenizeAndStringify("a.public:;"));

ASSERT_EQUALS("void f ( x & = 2 ) ;", tokenizeAndStringify("void f(x &= 2);"));

ASSERT_EQUALS("const_cast < a * > ( & e )", tokenizeAndStringify("const_cast<a*>(&e)"));

}

void concatenateNegativeNumber() {

ASSERT_EQUALS("i = -12 ;", tokenizeAndStringify("i = -12;"));

ASSERT_EQUALS("1 - 2 ;", tokenizeAndStringify("1-2;"));

ASSERT_EQUALS("foo ( -1 ) - 2 ;", tokenizeAndStringify("foo(-1)-2;"));

ASSERT_EQUALS("int f ( ) { return -2 ; }", tokenizeAndStringify("int f(){return -2;}"));

ASSERT_EQUALS("int x [ 2 ] = { -2 , 1 }", tokenizeAndStringify("int x[2] = {-2,1}"));

ASSERT_EQUALS("f ( 123 )", tokenizeAndStringify("f(+123)"));

}

void longtok() {

const std::string filedata(10000, 'a');

ASSERT_EQUALS(filedata, tokenizeAndStringify(filedata.c_str()));

}

void simplifyHeadersAndUnusedTemplates1() {

Settings s;

s.checkUnusedTemplates = false;

ASSERT_EQUALS(";",

tokenizeAndStringify("; template <typename... a> uint8_t b(std::tuple<uint8_t> d) {\n"

" std::tuple<a...> c{std::move(d)};\n"

" return std::get<0>(c);\n"

"}", s));

ASSERT_EQUALS("int g ( int ) ;",

tokenizeAndStringify("int g(int);\n"

"template <class F, class... Ts> auto h(F f, Ts... xs) {\n"

" auto e = f(g(xs)...);\n"

" return e;\n"

"}", s));

}

void simplifyHeadersAndUnusedTemplates2() {

const char code[] = "; template< typename T, u_int uBAR = 0 >\n"

"class Foo {\n"

"public:\n"

" void FooBar() {\n"

" new ( (uBAR ? uBAR : sizeof(T))) T;\n"

" }\n"

"};";

Settings s;

s.checkUnusedTemplates = false;

ASSERT_EQUALS(";", tokenizeAndStringify(code, s));

s.checkUnusedTemplates = true;

ASSERT_EQUALS("; template < typename T , u_int uBAR = 0 >\n"

"class Foo {\n"

"public:\n"

"void FooBar ( ) {\n"

"new ( uBAR ? uBAR : sizeof ( T ) ) T ;\n"

"}\n"

"} ;", tokenizeAndStringify(code, s));

}

void simplifyAt() {

ASSERT_EQUALS("int x ;", tokenizeAndStringify("int x@123;"));

ASSERT_EQUALS("bool x ;", tokenizeAndStringify("bool x@123:1;"));

ASSERT_EQUALS("char PORTB ; bool PB3 ;", tokenizeAndStringify("char PORTB @ 0x10; bool PB3 @ PORTB:3;\n"));

ASSERT_EQUALS("int x ;", tokenizeAndStringify("int x @ (0x1000 + 18);"));

ASSERT_EQUALS("int x [ 10 ] ;", tokenizeAndStringify("int x[10]@0x100;"));

ASSERT_EQUALS("interrupt@ f ( ) { }", tokenizeAndStringify("@interrupt f() {}"));

}

void inlineasm() {

ASSERT_EQUALS("asm ( \"mov ax , bx\" ) ;", tokenizeAndStringify("asm { mov ax,bx };"));

ASSERT_EQUALS("asm ( \"mov ax , bx\" ) ;", tokenizeAndStringify("_asm { mov ax,bx };"));

ASSERT_EQUALS("asm ( \"mov ax , bx\" ) ;", tokenizeAndStringify("_asm mov ax,bx"));

ASSERT_EQUALS("asm ( \"mov ax , bx\" ) ;", tokenizeAndStringify("__asm { mov ax,bx };"));

ASSERT_EQUALS("asm ( \"\"mov ax,bx\"\" ) ;", tokenizeAndStringify("__asm__ __volatile__ ( \"mov ax,bx\" );"));

ASSERT_EQUALS("asm ( \"_emit 12h\" ) ;", tokenizeAndStringify("__asm _emit 12h ;"));

ASSERT_EQUALS("asm ( \"mov a , b\" ) ;", tokenizeAndStringify("__asm mov a, b ;"));

ASSERT_EQUALS("asm ( \"\"fnstcw %0\" : \"= m\" ( old_cw )\" ) ;", tokenizeAndStringify("asm volatile (\"fnstcw %0\" : \"= m\" (old_cw));"));

ASSERT_EQUALS("asm ( \"\"fnstcw %0\" : \"= m\" ( old_cw )\" ) ;", tokenizeAndStringify(" __asm__ (\"fnstcw %0\" : \"= m\" (old_cw));"));

ASSERT_EQUALS("asm ( \"\"ddd\"\" ) ;", tokenizeAndStringify(" __asm __volatile__ (\"ddd\") ;"));

ASSERT_EQUALS("asm ( \"\"ddd\"\" ) ;", tokenizeAndStringify(" __asm __volatile (\"ddd\") ;"));

ASSERT_EQUALS("asm ( \"\"mov ax,bx\"\" ) ;", tokenizeAndStringify("__asm__ volatile ( \"mov ax,bx\" );"));

ASSERT_EQUALS("asm ( \"mov ax , bx\" ) ; int a ;", tokenizeAndStringify("asm { mov ax,bx } int a;"));

ASSERT_EQUALS("asm\n\n( \"mov ax , bx\" ) ;", tokenizeAndStringify("__asm\nmov ax,bx\n__endasm;"));

ASSERT_EQUALS("asm\n\n( \"push b ; for if\" ) ;", tokenizeAndStringify("__asm\npush b ; for if\n__endasm;"));

// 'asm ( ) ;' should be in the same line

ASSERT_EQUALS(";\n\nasm ( \"\"mov ax,bx\"\" ) ;", tokenizeAndStringify(";\n\n__asm__ volatile ( \"mov ax,bx\" );"));

ASSERT_EQUALS("void func1 ( ) ;", tokenizeAndStringify("void func1() __asm__(\"...\") __attribute__();"));

}