townforge/Urho3D
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
upstream-octree
Urho3D/Source/ThirdParty/MojoShader/mojoshader_preprocessor.c

2363 lines
66 KiB
C
Raw Permalink Blame History

/**
* MojoShader; generate shader programs from bytecode of compiled
* Direct3D shaders.
*
* Please see the file LICENSE.txt in the source's root directory.
*
* This file written by Ryan C. Gordon.
*/
#define __MOJOSHADER_INTERNAL__ 1
#include "mojoshader_internal.h"
#if DEBUG_PREPROCESSOR
#define print_debug_token(token, len, val) \
MOJOSHADER_print_debug_token("PREPROCESSOR", token, len, val)
#else
#define print_debug_token(token, len, val)
#endif
#if DEBUG_LEXER
static Token debug_preprocessor_lexer(IncludeState *s)
{
const Token retval = preprocessor_lexer(s);
MOJOSHADER_print_debug_token("LEXER", s->token, s->tokenlen, retval);
return retval;
} // debug_preprocessor_lexer
#define preprocessor_lexer(s) debug_preprocessor_lexer(s)
#endif
#if DEBUG_TOKENIZER
static void print_debug_lexing_position(IncludeState *s)
{
if (s != NULL)
printf("NOW LEXING %s:%d ...\n", s->filename, s->line);
} // print_debug_lexing_position
#else
#define print_debug_lexing_position(s)
#endif
typedef struct Context
{
int isfail;
int out_of_memory;
char failstr[256];
int recursion_count;
int asm_comments;
int parsing_pragma;
Conditional *conditional_pool;
IncludeState *include_stack;
IncludeState *include_pool;
Define *define_hashtable[256];
Define *define_pool;
Define *file_macro;
Define *line_macro;
StringCache *filename_cache;
MOJOSHADER_includeOpen open_callback;
MOJOSHADER_includeClose close_callback;
MOJOSHADER_malloc malloc;
MOJOSHADER_free free;
void *malloc_data;
} Context;
// Convenience functions for allocators...
static inline void out_of_memory(Context *ctx)
{
ctx->out_of_memory = 1;
} // out_of_memory
static inline void *Malloc(Context *ctx, const size_t len)
{
void *retval = ctx->malloc((int) len, ctx->malloc_data);
if (retval == NULL)
out_of_memory(ctx);
return retval;
} // Malloc
static inline void Free(Context *ctx, void *ptr)
{
ctx->free(ptr, ctx->malloc_data);
} // Free
static void *MallocBridge(int bytes, void *data)
{
return Malloc((Context *) data, (size_t) bytes);
} // MallocBridge
static void FreeBridge(void *ptr, void *data)
{
Free((Context *) data, ptr);
} // FreeBridge
static inline char *StrDup(Context *ctx, const char *str)
{
char *retval = (char *) Malloc(ctx, strlen(str) + 1);
if (retval != NULL)
strcpy(retval, str);
return retval;
} // StrDup
static void failf(Context *ctx, const char *fmt, ...) ISPRINTF(2,3);
static void failf(Context *ctx, const char *fmt, ...)
{
ctx->isfail = 1;
va_list ap;
va_start(ap, fmt);
vsnprintf(ctx->failstr, sizeof (ctx->failstr), fmt, ap);
va_end(ap);
} // failf
static inline void fail(Context *ctx, const char *reason)
{
failf(ctx, "%s", reason);
} // fail
#if DEBUG_TOKENIZER
void MOJOSHADER_print_debug_token(const char *subsystem, const char *token,
const unsigned int tokenlen,
const Token tokenval)
{
printf("%s TOKEN: \"", subsystem);
unsigned int i;
for (i = 0; i < tokenlen; i++)
{
if (token[i] == '\n')
printf("\\n");
else if (token[i] == '\\')
printf("\\\\");
else
printf("%c", token[i]);
} // for
printf("\" (");
switch (tokenval)
{
#define TOKENCASE(x) case x: printf("%s", #x); break
TOKENCASE(TOKEN_UNKNOWN);
TOKENCASE(TOKEN_IDENTIFIER);
TOKENCASE(TOKEN_INT_LITERAL);
TOKENCASE(TOKEN_FLOAT_LITERAL);
TOKENCASE(TOKEN_STRING_LITERAL);
TOKENCASE(TOKEN_ADDASSIGN);
TOKENCASE(TOKEN_SUBASSIGN);
TOKENCASE(TOKEN_MULTASSIGN);
TOKENCASE(TOKEN_DIVASSIGN);
TOKENCASE(TOKEN_MODASSIGN);
TOKENCASE(TOKEN_XORASSIGN);
TOKENCASE(TOKEN_ANDASSIGN);
TOKENCASE(TOKEN_ORASSIGN);
TOKENCASE(TOKEN_INCREMENT);
TOKENCASE(TOKEN_DECREMENT);
TOKENCASE(TOKEN_RSHIFT);
TOKENCASE(TOKEN_LSHIFT);
TOKENCASE(TOKEN_ANDAND);
TOKENCASE(TOKEN_OROR);
TOKENCASE(TOKEN_LEQ);
TOKENCASE(TOKEN_GEQ);
TOKENCASE(TOKEN_EQL);
TOKENCASE(TOKEN_NEQ);
TOKENCASE(TOKEN_HASH);
TOKENCASE(TOKEN_HASHHASH);
TOKENCASE(TOKEN_PP_INCLUDE);
TOKENCASE(TOKEN_PP_LINE);
TOKENCASE(TOKEN_PP_DEFINE);
TOKENCASE(TOKEN_PP_UNDEF);
TOKENCASE(TOKEN_PP_IF);
TOKENCASE(TOKEN_PP_IFDEF);
TOKENCASE(TOKEN_PP_IFNDEF);
TOKENCASE(TOKEN_PP_ELSE);
TOKENCASE(TOKEN_PP_ELIF);
TOKENCASE(TOKEN_PP_ENDIF);
TOKENCASE(TOKEN_PP_ERROR);
TOKENCASE(TOKEN_PP_PRAGMA);
TOKENCASE(TOKEN_INCOMPLETE_COMMENT);
TOKENCASE(TOKEN_BAD_CHARS);
TOKENCASE(TOKEN_EOI);
TOKENCASE(TOKEN_PREPROCESSING_ERROR);
#undef TOKENCASE
case ((Token) '\n'):
printf("'\\n'");
break;
case ((Token) '\\'):
printf("'\\\\'");
break;
default:
assert(((int)tokenval) < 256);
printf("'%c'", (char) tokenval);
break;
} // switch
printf(")\n");
} // MOJOSHADER_print_debug_token
#endif
#if !MOJOSHADER_FORCE_INCLUDE_CALLBACKS
// !!! FIXME: most of these _MSC_VER should probably be _WINDOWS?
#ifdef _MSC_VER
#define WIN32_LEAN_AND_MEAN 1
#include <windows.h> // GL headers need this for WINGDIAPI definition.
#else
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#endif
int MOJOSHADER_internal_include_open(MOJOSHADER_includeType inctype,
const char *fname, const char *parent,
const char **outdata,
unsigned int *outbytes,
MOJOSHADER_malloc m, MOJOSHADER_free f,
void *d)
{
#ifdef _MSC_VER
WCHAR wpath[MAX_PATH];
if (!MultiByteToWideChar(CP_UTF8, 0, fname, -1, wpath, MAX_PATH))
return 0;
const DWORD share = FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE;
const HANDLE handle = CreateFileW(wpath, FILE_GENERIC_READ, share,
NULL, OPEN_EXISTING, NULL, NULL);
if (handle == INVALID_HANDLE_VALUE)
return 0;
const DWORD fileSize = GetFileSize(handle, NULL);
if (fileSize == INVALID_FILE_SIZE)
{
CloseHandle(handle);
return 0;
} // if
char *data = (char *) m(fileSize, d);
if (data == NULL)
{
CloseHandle(handle);
return 0;
} // if
DWORD readLength = 0;
if (!ReadFile(handle, data, fileSize, &readLength, NULL))
{
CloseHandle(handle);
f(data, d);
return 0;
} // if
CloseHandle(handle);
if (readLength != fileSize)
{
f(data, d);
return 0;
} // if
*outdata = data;
*outbytes = fileSize;
return 1;
#else
struct stat statbuf;
if (stat(fname, &statbuf) == -1)
return 0;
char *data = (char *) m(statbuf.st_size, d);
if (data == NULL)
return 0;
const int fd = open(fname, O_RDONLY);
if (fd == -1)
{
f(data, d);
return 0;
} // if
if (read(fd, data, statbuf.st_size) != statbuf.st_size)
{
f(data, d);
close(fd);
return 0;
} // if
close(fd);
*outdata = data;
*outbytes = (unsigned int) statbuf.st_size;
return 1;
#endif
} // MOJOSHADER_internal_include_open
void MOJOSHADER_internal_include_close(const char *data, MOJOSHADER_malloc m,
MOJOSHADER_free f, void *d)
{
f((void *) data, d);
} // MOJOSHADER_internal_include_close
#endif // !MOJOSHADER_FORCE_INCLUDE_CALLBACKS
// !!! FIXME: maybe use these pool magic elsewhere?
// !!! FIXME: maybe just get rid of this? (maybe the fragmentation isn't a big deal?)
// Pool stuff...
// ugh, I hate this macro salsa.
#define FREE_POOL(type, poolname) \
static void free_##poolname##_pool(Context *ctx) { \
type *item = ctx->poolname##_pool; \
while (item != NULL) { \
type *next = item->next; \
Free(ctx, item); \
item = next; \
} \
}
#define GET_POOL(type, poolname) \
static type *get_##poolname(Context *ctx) { \
type *retval = ctx->poolname##_pool; \
if (retval != NULL) \
ctx->poolname##_pool = retval->next; \
else \
retval = (type *) Malloc(ctx, sizeof (type)); \
if (retval != NULL) \
memset(retval, '\0', sizeof (type)); \
return retval; \
}
#define PUT_POOL(type, poolname) \
static void put_##poolname(Context *ctx, type *item) { \
item->next = ctx->poolname##_pool; \
ctx->poolname##_pool = item; \
}
#define IMPLEMENT_POOL(type, poolname) \
FREE_POOL(type, poolname) \
GET_POOL(type, poolname) \
PUT_POOL(type, poolname)
IMPLEMENT_POOL(Conditional, conditional)
IMPLEMENT_POOL(IncludeState, include)
IMPLEMENT_POOL(Define, define)
// Preprocessor define hashtable stuff...
// !!! FIXME: why isn't this using mojoshader_common.c's code?
// this is djb's xor hashing function.
static inline uint32 hash_string_djbxor(const char *sym)
{
register uint32 hash = 5381;
while (*sym)
hash = ((hash << 5) + hash) ^ *(sym++);
return hash;
} // hash_string_djbxor
static inline uint8 hash_define(const char *sym)
{
return (uint8) hash_string_djbxor(sym);
} // hash_define
static int add_define(Context *ctx, const char *sym, const char *val,
char **parameters, int paramcount)
{
const uint8 hash = hash_define(sym);
Define *bucket = ctx->define_hashtable[hash];
while (bucket)
{
if (strcmp(bucket->identifier, sym) == 0)
{
failf(ctx, "'%s' already defined", sym); // !!! FIXME: warning?
// !!! FIXME: gcc reports the location of previous #define here.
return 0;
} // if
bucket = bucket->next;
} // while
bucket = get_define(ctx);
if (bucket == NULL)
return 0;
bucket->definition = val;
bucket->original = NULL;
bucket->identifier = sym;
bucket->parameters = (const char **) parameters;
bucket->paramcount = paramcount;
bucket->next = ctx->define_hashtable[hash];
ctx->define_hashtable[hash] = bucket;
return 1;
} // add_define
static void free_define(Context *ctx, Define *def)
{
if (def != NULL)
{
int i;
for (i = 0; i < def->paramcount; i++)
Free(ctx, (void *) def->parameters[i]);
Free(ctx, (void *) def->parameters);
Free(ctx, (void *) def->identifier);
Free(ctx, (void *) def->definition);
Free(ctx, (void *) def->original);
put_define(ctx, def);
} // if
} // free_define
static int remove_define(Context *ctx, const char *sym)
{
const uint8 hash = hash_define(sym);
Define *bucket = ctx->define_hashtable[hash];
Define *prev = NULL;
while (bucket)
{
if (strcmp(bucket->identifier, sym) == 0)
{
if (prev == NULL)
ctx->define_hashtable[hash] = bucket->next;
else
prev->next = bucket->next;
free_define(ctx, bucket);
return 1;
} // if
prev = bucket;
bucket = bucket->next;
} // while
return 0;
} // remove_define
static const Define *find_define(Context *ctx, const char *sym)
{
if ( (ctx->file_macro) && (strcmp(sym, "__FILE__") == 0) )
{
Free(ctx, (char *) ctx->file_macro->definition);
const IncludeState *state = ctx->include_stack;
const char *fname = state ? state->filename : "";
const size_t len = strlen(fname) + 2;
char *str = (char *) Malloc(ctx, len);
if (!str)
return NULL;
str[0] = '\"';
memcpy(str + 1, fname, len - 2);
str[len - 1] = '\"';
ctx->file_macro->definition = str;
return ctx->file_macro;
} // if
else if ( (ctx->line_macro) && (strcmp(sym, "__LINE__") == 0) )
{
Free(ctx, (char *) ctx->line_macro->definition);
const IncludeState *state = ctx->include_stack;
const size_t bufsize = 32;
char *str = (char *) Malloc(ctx, bufsize);
if (!str)
return 0;
const size_t len = snprintf(str, bufsize, "%u", state->line);
assert(len < bufsize);
ctx->line_macro->definition = str;
return ctx->line_macro;
} // else
const uint8 hash = hash_define(sym);
Define *bucket = ctx->define_hashtable[hash];
while (bucket)
{
if (strcmp(bucket->identifier, sym) == 0)
return bucket;
bucket = bucket->next;
} // while
return NULL;
} // find_define
static const Define *find_define_by_token(Context *ctx)
{
IncludeState *state = ctx->include_stack;
assert(state->tokenval == TOKEN_IDENTIFIER);
char *sym = (char *) alloca(state->tokenlen+1);
memcpy(sym, state->token, state->tokenlen);
sym[state->tokenlen] = '\0';
return find_define(ctx, sym);
} // find_define_by_token
static const Define *find_macro_arg(const IncludeState *state,
const Define *defines)
{
const Define *def = NULL;
char *sym = (char *) alloca(state->tokenlen + 1);
memcpy(sym, state->token, state->tokenlen);
sym[state->tokenlen] = '\0';
for (def = defines; def != NULL; def = def->next)
{
assert(def->parameters == NULL); // args can't have args!
assert(def->paramcount == 0); // args can't have args!
if (strcmp(def->identifier, sym) == 0)
break;
} // while
return def;
} // find_macro_arg
static void put_all_defines(Context *ctx)
{
size_t i;
for (i = 0; i < STATICARRAYLEN(ctx->define_hashtable); i++)
{
Define *bucket = ctx->define_hashtable[i];
ctx->define_hashtable[i] = NULL;
while (bucket)
{
Define *next = bucket->next;
free_define(ctx, bucket);
bucket = next;
} // while
} // for
} // put_all_defines
static int push_source(Context *ctx, const char *fname, const char *source,
unsigned int srclen, unsigned int linenum,
MOJOSHADER_includeClose close_callback)
{
IncludeState *state = get_include(ctx);
if (state == NULL)
return 0;
if (fname != NULL)
{
state->filename = stringcache(ctx->filename_cache, fname);
if (state->filename == NULL)
{
put_include(ctx, state);
return 0;
} // if
} // if
state->close_callback = close_callback;
state->source_base = source;
state->source = source;
state->token = source;
state->tokenval = ((Token) '\n');
state->orig_length = srclen;
state->bytes_left = srclen;
state->line = linenum;
state->next = ctx->include_stack;
state->asm_comments = ctx->asm_comments;
print_debug_lexing_position(state);
ctx->include_stack = state;
return 1;
} // push_source
static void pop_source(Context *ctx)
{
IncludeState *state = ctx->include_stack;
assert(state != NULL); // more pops than pushes!
if (state == NULL)
return;
if (state->close_callback)
{
state->close_callback(state->source_base, ctx->malloc,
ctx->free, ctx->malloc_data);
} // if
// state->filename is a pointer to the filename cache; don't free it here!
Conditional *cond = state->conditional_stack;
while (cond)
{
Conditional *next = cond->next;
put_conditional(ctx, cond);
cond = next;
} // while
ctx->include_stack = state->next;
print_debug_lexing_position(ctx->include_stack);
put_include(ctx, state);
} // pop_source
static void close_define_include(const char *data, MOJOSHADER_malloc m,
MOJOSHADER_free f, void *d)
{
f((void *) data, d);
} // close_define_include
Preprocessor *preprocessor_start(const char *fname, const char *source,
unsigned int sourcelen,
MOJOSHADER_includeOpen open_callback,
MOJOSHADER_includeClose close_callback,
const MOJOSHADER_preprocessorDefine *defines,
unsigned int define_count, int asm_comments,
MOJOSHADER_malloc m, MOJOSHADER_free f, void *d)
{
int okay = 1;
unsigned int i = 0;
// the preprocessor is internal-only, so we verify all these are != NULL.
assert(m != NULL);
assert(f != NULL);
Context *ctx = (Context *) m(sizeof (Context), d);
if (ctx == NULL)
return NULL;
memset(ctx, '\0', sizeof (Context));
ctx->malloc = m;
ctx->free = f;
ctx->malloc_data = d;
ctx->open_callback = open_callback;
ctx->close_callback = close_callback;
ctx->asm_comments = asm_comments;
ctx->filename_cache = stringcache_create(MallocBridge, FreeBridge, ctx);
okay = ((okay) && (ctx->filename_cache != NULL));
ctx->file_macro = get_define(ctx);
okay = ((okay) && (ctx->file_macro != NULL));
if ((okay) && (ctx->file_macro))
okay = ((ctx->file_macro->identifier = StrDup(ctx, "__FILE__")) != 0);
ctx->line_macro = get_define(ctx);
okay = ((okay) && (ctx->line_macro != NULL));
if ((okay) && (ctx->line_macro))
okay = ((ctx->line_macro->identifier = StrDup(ctx, "__LINE__")) != 0);
// let the usual preprocessor parser sort these out.
char *define_include = NULL;
unsigned int define_include_len = 0;
if ((okay) && (define_count > 0))
{
Buffer *predefbuf = buffer_create(256, MallocBridge, FreeBridge, ctx);
okay = okay && (predefbuf != NULL);
for (i = 0; okay && (i < define_count); i++)
{
okay = okay && buffer_append_fmt(predefbuf, "#define %s %s\n",
defines[i].identifier, defines[i].definition);
} // for
define_include_len = buffer_size(predefbuf);
if (define_include_len > 0)
{
define_include = buffer_flatten(predefbuf);
okay = okay && (define_include != NULL);
} // if
buffer_destroy(predefbuf);
} // if
if ((okay) && (!push_source(ctx,fname,source,sourcelen,1,NULL)))
okay = 0;
if ((okay) && (define_include_len > 0))
{
assert(define_include != NULL);
okay = push_source(ctx, "<predefined macros>", define_include,
define_include_len, 1, close_define_include);
} // if
if (!okay)
{
preprocessor_end((Preprocessor *) ctx);
return NULL;
} // if
return (Preprocessor *) ctx;
} // preprocessor_start
void preprocessor_end(Preprocessor *_ctx)
{
Context *ctx = (Context *) _ctx;
if (ctx == NULL)
return;
while (ctx->include_stack != NULL)
pop_source(ctx);
put_all_defines(ctx);
if (ctx->filename_cache != NULL)
stringcache_destroy(ctx->filename_cache);
free_define(ctx, ctx->file_macro);
free_define(ctx, ctx->line_macro);
free_define_pool(ctx);
free_conditional_pool(ctx);
free_include_pool(ctx);
Free(ctx, ctx);
} // preprocessor_end
int preprocessor_outofmemory(Preprocessor *_ctx)
{
Context *ctx = (Context *) _ctx;
return ctx->out_of_memory;
} // preprocessor_outofmemory
static inline void pushback(IncludeState *state)
{
#if DEBUG_PREPROCESSOR
printf("PREPROCESSOR PUSHBACK\n");
#endif
assert(!state->pushedback);
state->pushedback = 1;
} // pushback
static Token lexer(IncludeState *state)
{
if (!state->pushedback)
return preprocessor_lexer(state);
state->pushedback = 0;
return state->tokenval;
} // lexer
// !!! FIXME: parsing fails on preprocessor directives should skip rest of line.
static int require_newline(IncludeState *state)
{
const Token token = lexer(state);
pushback(state); // rewind no matter what.
return ( (token == TOKEN_INCOMPLETE_COMMENT) || // call it an eol.
(token == ((Token) '\n')) || (token == TOKEN_EOI) );
} // require_newline
// !!! FIXME: didn't we implement this by hand elsewhere?
static int token_to_int(IncludeState *state)
{
assert(state->tokenval == TOKEN_INT_LITERAL);
char *buf = (char *) alloca(state->tokenlen+1);
memcpy(buf, state->token, state->tokenlen);
buf[state->tokenlen] = '\0';
return atoi(buf);
} // token_to_int
static void handle_pp_include(Context *ctx)
{
IncludeState *state = ctx->include_stack;
Token token = lexer(state);
MOJOSHADER_includeType incltype;
char *filename = NULL;
int bogus = 0;
if (token == TOKEN_STRING_LITERAL)
incltype = MOJOSHADER_INCLUDETYPE_LOCAL;
else if (token == ((Token) '<'))
{
incltype = MOJOSHADER_INCLUDETYPE_SYSTEM;
// can't use lexer, since every byte between the < > pair is
// considered part of the filename. :/
while (!bogus)
{
if ( !(bogus = (state->bytes_left == 0)) )
{
const char ch = *state->source;
if ( !(bogus = ((ch == '\r') || (ch == '\n'))) )
{
state->source++;
state->bytes_left--;
if (ch == '>')
break;
} // if
} // if
} // while
} // else if
else
{
bogus = 1;
} // else
if (!bogus)
{
state->token++; // skip '<' or '\"'...
const unsigned int len = ((unsigned int) (state->source-state->token));
filename = (char *) alloca(len);
memcpy(filename, state->token, len-1);
filename[len-1] = '\0';
bogus = !require_newline(state);
} // if
if (bogus)
{
fail(ctx, "Invalid #include directive");
return;
} // else
const char *newdata = NULL;
unsigned int newbytes = 0;
if ((ctx->open_callback == NULL) || (ctx->close_callback == NULL))
{
fail(ctx, "Saw #include, but no include callbacks defined");
return;
} // if
if (!ctx->open_callback(incltype, filename, state->source_base,
&newdata, &newbytes, ctx->malloc,
ctx->free, ctx->malloc_data))
{
fail(ctx, "Include callback failed"); // !!! FIXME: better error
return;
} // if
MOJOSHADER_includeClose callback = ctx->close_callback;
if (!push_source(ctx, filename, newdata, newbytes, 1, callback))
{
assert(ctx->out_of_memory);
ctx->close_callback(newdata, ctx->malloc, ctx->free, ctx->malloc_data);
} // if
} // handle_pp_include
static void handle_pp_line(Context *ctx)
{
IncludeState *state = ctx->include_stack;
char *filename = NULL;
int linenum = 0;
int bogus = 0;
if (lexer(state) != TOKEN_INT_LITERAL)
bogus = 1;
else
linenum = token_to_int(state);
if (!bogus)
{
Token t = lexer(state);
if (t == ((Token) '\n'))
{
state->line = linenum;
return;
}
bogus = (t != TOKEN_STRING_LITERAL);
}
if (!bogus)
{
state->token++; // skip '\"'...
filename = (char *) alloca(state->tokenlen);
memcpy(filename, state->token, state->tokenlen-1);
filename[state->tokenlen-1] = '\0';
bogus = !require_newline(state);
} // if
if (bogus)
{
fail(ctx, "Invalid #line directive");
return;
} // if
const char *cached = stringcache(ctx->filename_cache, filename);
state->filename = cached; // may be NULL if stringcache() failed.
state->line = linenum;
} // handle_pp_line
static void handle_pp_error(Context *ctx)
{
IncludeState *state = ctx->include_stack;
char *ptr = ctx->failstr;
int avail = sizeof (ctx->failstr) - 1;
int cpy = 0;
int done = 0;
const char *prefix = "#error";
const size_t prefixlen = strlen(prefix);
strcpy(ctx->failstr, prefix);
avail -= prefixlen;
ptr += prefixlen;
state->report_whitespace = 1;
while (!done)
{
const Token token = lexer(state);
switch (token)
{
case ((Token) '\n'):
state->line--; // make sure error is on the right line.
// fall through!
case TOKEN_INCOMPLETE_COMMENT:
case TOKEN_EOI:
pushback(state); // move back so we catch this later.
done = 1;
break;
case ((Token) ' '):
if (!avail)
break;
*(ptr++) = ' ';
avail--;
break;
default:
cpy = Min(avail, (int) state->tokenlen);
if (cpy)
memcpy(ptr, state->token, cpy);
ptr += cpy;
avail -= cpy;
break;
} // switch
} // while
*ptr = '\0';
state->report_whitespace = 0;
ctx->isfail = 1;
} // handle_pp_error
static void handle_pp_define(Context *ctx)
{
IncludeState *state = ctx->include_stack;
int done = 0;
if (lexer(state) != TOKEN_IDENTIFIER)
{
fail(ctx, "Macro names must be identifiers");
return;
} // if
char *definition = NULL;
char *sym = (char *) Malloc(ctx, state->tokenlen+1);
if (sym == NULL)
return;
memcpy(sym, state->token, state->tokenlen);
sym[state->tokenlen] = '\0';
if (strcmp(sym, "defined") == 0)
{
Free(ctx, sym);
fail(ctx, "'defined' cannot be used as a macro name");
return;
} // if
// Don't treat these symbols as special anymore if they get (re)#defined.
if (strcmp(sym, "__FILE__") == 0)
{
if (ctx->file_macro)
{
failf(ctx, "'%s' already defined", sym); // !!! FIXME: warning?
free_define(ctx, ctx->file_macro);
ctx->file_macro = NULL;
} // if
} // if
else if (strcmp(sym, "__LINE__") == 0)
{
if (ctx->line_macro)
{
failf(ctx, "'%s' already defined", sym); // !!! FIXME: warning?
free_define(ctx, ctx->line_macro);
ctx->line_macro = NULL;
} // if
} // else if
// #define a(b) is different than #define a (b) :(
state->report_whitespace = 1;
lexer(state);
state->report_whitespace = 0;
int params = 0;
char **idents = NULL;
static const char space = ' ';
if (state->tokenval == ((Token) ' '))
lexer(state); // skip it.
else if (state->tokenval == ((Token) '('))
{
IncludeState saved;
memcpy(&saved, state, sizeof (IncludeState));
while (1)
{
if (lexer(state) != TOKEN_IDENTIFIER)
break;
params++;
if (lexer(state) != ((Token) ','))
break;
} // while
if (state->tokenval != ((Token) ')'))
{
fail(ctx, "syntax error in macro parameter list");
goto handle_pp_define_failed;
} // if
if (params == 0) // special case for void args: "#define a() b"
params = -1;
else
{
idents = (char **) Malloc(ctx, sizeof (char *) * params);
if (idents == NULL)
goto handle_pp_define_failed;
// roll all the way back, do it again.
memcpy(state, &saved, sizeof (IncludeState));
memset(idents, '\0', sizeof (char *) * params);
int i;
for (i = 0; i < params; i++)
{
lexer(state);
assert(state->tokenval == TOKEN_IDENTIFIER);
char *dst = (char *) Malloc(ctx, state->tokenlen+1);
if (dst == NULL)
break;
memcpy(dst, state->token, state->tokenlen);
dst[state->tokenlen] = '\0';
idents[i] = dst;
if (i < (params-1))
{
lexer(state);
assert(state->tokenval == ((Token) ','));
} // if
} // for
if (i != params)
{
assert(ctx->out_of_memory);
goto handle_pp_define_failed;
} // if
lexer(state);
assert(state->tokenval == ((Token) ')'));
} // else
lexer(state);
} // else if
pushback(state);
Buffer *buffer = buffer_create(128, MallocBridge, FreeBridge, ctx);
state->report_whitespace = 1;
while ((!done) && (!ctx->out_of_memory))
{
const Token token = lexer(state);
switch (token)
{
case TOKEN_INCOMPLETE_COMMENT:
case TOKEN_EOI:
pushback(state); // move back so we catch this later.
done = 1;
break;
case ((Token) '\n'):
done = 1;
break;
case ((Token) ' '): // may not actually point to ' '.
assert(buffer_size(buffer) > 0);
buffer_append(buffer, &space, 1);
break;
default:
buffer_append(buffer, state->token, state->tokenlen);
break;
} // switch
} // while
state->report_whitespace = 0;
size_t buflen = buffer_size(buffer) + 1;
if (!ctx->out_of_memory)
definition = buffer_flatten(buffer);
buffer_destroy(buffer);
if (ctx->out_of_memory)
goto handle_pp_define_failed;
int hashhash_error = 0;
if ((buflen > 2) && (definition[0] == '#') && (definition[1] == '#'))
{
hashhash_error = 1;
buflen -= 2;
memmove(definition, definition + 2, buflen);
} // if
if (buflen > 2)
{
char *ptr = (definition + buflen) - 2;
if (*ptr == ' ')
{
ptr--;
buflen--;
} // if
if ((buflen > 2) && (ptr[0] == '#') && (ptr[-1] == '#'))
{
hashhash_error = 1;
buflen -= 2;
ptr[-1] = '\0';
} // if
} // if
if (hashhash_error)
fail(ctx, "'##' cannot appear at either end of a macro expansion");
assert(done);
if (!add_define(ctx, sym, definition, idents, params))
goto handle_pp_define_failed;
return;
handle_pp_define_failed:
Free(ctx, sym);
Free(ctx, definition);
if (idents != NULL)
{
while (params--)
Free(ctx, idents[params]);
} // if
Free(ctx, idents);
} // handle_pp_define
static void handle_pp_undef(Context *ctx)
{
IncludeState *state = ctx->include_stack;
if (lexer(state) != TOKEN_IDENTIFIER)
{
fail(ctx, "Macro names must be indentifiers");
return;
} // if
char *sym = (char *) alloca(state->tokenlen+1);
memcpy(sym, state->token, state->tokenlen);
sym[state->tokenlen] = '\0';
if (!require_newline(state))
{
fail(ctx, "Invalid #undef directive");
return;
} // if
if (strcmp(sym, "__FILE__") == 0)
{
if (ctx->file_macro)
{
failf(ctx, "undefining \"%s\"", sym); // !!! FIXME: should be warning.
free_define(ctx, ctx->file_macro);
ctx->file_macro = NULL;
} // if
} // if
else if (strcmp(sym, "__LINE__") == 0)
{
if (ctx->line_macro)
{
failf(ctx, "undefining \"%s\"", sym); // !!! FIXME: should be warning.
free_define(ctx, ctx->line_macro);
ctx->line_macro = NULL;
} // if
} // if
remove_define(ctx, sym);
} // handle_pp_undef
static Conditional *_handle_pp_ifdef(Context *ctx, const Token type)
{
IncludeState *state = ctx->include_stack;
assert((type == TOKEN_PP_IFDEF) || (type == TOKEN_PP_IFNDEF));
if (lexer(state) != TOKEN_IDENTIFIER)
{
fail(ctx, "Macro names must be indentifiers");
return NULL;
} // if
char *sym = (char *) alloca(state->tokenlen+1);
memcpy(sym, state->token, state->tokenlen);
sym[state->tokenlen] = '\0';
if (!require_newline(state))
{
if (type == TOKEN_PP_IFDEF)
fail(ctx, "Invalid #ifdef directive");
else
fail(ctx, "Invalid #ifndef directive");
return NULL;
} // if
Conditional *conditional = get_conditional(ctx);
assert((conditional != NULL) || (ctx->out_of_memory));
if (conditional == NULL)
return NULL;
Conditional *parent = state->conditional_stack;
const int found = (find_define(ctx, sym) != NULL);
const int chosen = (type == TOKEN_PP_IFDEF) ? found : !found;
const int skipping = ( (((parent) && (parent->skipping))) || (!chosen) );
conditional->type = type;
conditional->linenum = state->line - 1;
conditional->skipping = skipping;
conditional->chosen = chosen;
conditional->next = parent;
state->conditional_stack = conditional;
return conditional;
} // _handle_pp_ifdef
static inline void handle_pp_ifdef(Context *ctx)
{
_handle_pp_ifdef(ctx, TOKEN_PP_IFDEF);
} // handle_pp_ifdef
static inline void handle_pp_ifndef(Context *ctx)
{
_handle_pp_ifdef(ctx, TOKEN_PP_IFNDEF);
} // handle_pp_ifndef
static int replace_and_push_macro(Context *ctx, const Define *def,
const Define *params)
{
char *final = NULL;
// We push the #define and lex it, building a buffer with argument
// replacement, stringification, and concatenation.
Buffer *buffer = buffer_create(128, MallocBridge, FreeBridge, ctx);
if (buffer == NULL)
return 0;
IncludeState *state = ctx->include_stack;
if (!push_source(ctx, state->filename, def->definition,
strlen(def->definition), state->line, NULL))
{
buffer_destroy(buffer);
return 0;
} // if
state = ctx->include_stack;
while (lexer(state) != TOKEN_EOI)
{
int wantorig = 0;
const Define *arg = NULL;
// put a space between tokens if we're not concatenating.
if (state->tokenval == TOKEN_HASHHASH) // concatenate?
{
wantorig = 1;
lexer(state);
assert(state->tokenval != TOKEN_EOI);
} // if
else
{
if (buffer_size(buffer) > 0)
{
if (!buffer_append(buffer, " ", 1))
goto replace_and_push_macro_failed;
} // if
} // else
const char *data = state->token;
unsigned int len = state->tokenlen;
if (state->tokenval == TOKEN_HASH) // stringify?
{
lexer(state);
assert(state->tokenval != TOKEN_EOI); // we checked for this.
if (!buffer_append(buffer, "\"", 1))
goto replace_and_push_macro_failed;
if (state->tokenval == TOKEN_IDENTIFIER)
{
arg = find_macro_arg(state, params);
if (arg != NULL)
{
data = arg->original;
len = strlen(data);
} // if
} // if
if (!buffer_append(buffer, data, len))
goto replace_and_push_macro_failed;
if (!buffer_append(buffer, "\"", 1))
goto replace_and_push_macro_failed;
continue;
} // if
if (state->tokenval == TOKEN_IDENTIFIER)
{
arg = find_macro_arg(state, params);
if (arg != NULL)
{
if (!wantorig)
{
wantorig = (lexer(state) == TOKEN_HASHHASH);
pushback(state);
} // if
data = wantorig ? arg->original : arg->definition;
len = strlen(data);
} // if
} // if
if (!buffer_append(buffer, data, len))
goto replace_and_push_macro_failed;
} // while
final = buffer_flatten(buffer);
if (!final)
goto replace_and_push_macro_failed;
buffer_destroy(buffer);
pop_source(ctx); // ditch the macro.
state = ctx->include_stack;
if (!push_source(ctx, state->filename, final, strlen(final), state->line,
close_define_include))
{
Free(ctx, final);
return 0;
} // if
return 1;
replace_and_push_macro_failed:
pop_source(ctx);
buffer_destroy(buffer);
return 0;
} // replace_and_push_macro
static int handle_macro_args(Context *ctx, const char *sym, const Define *def)
{
int retval = 0;
IncludeState *state = ctx->include_stack;
Define *params = NULL;
const int expected = (def->paramcount < 0) ? 0 : def->paramcount;
int saw_params = 0;
IncludeState saved; // can't pushback, we need the original token.
memcpy(&saved, state, sizeof (IncludeState));
if (lexer(state) != ((Token) '('))
{
memcpy(state, &saved, sizeof (IncludeState));
goto handle_macro_args_failed; // gcc abandons replacement, too.
} // if
state->report_whitespace = 1;
int void_call = 0;
int paren = 1;
while (paren > 0)
{
Buffer *buffer = buffer_create(128, MallocBridge, FreeBridge, ctx);
Buffer *origbuffer = buffer_create(128, MallocBridge, FreeBridge, ctx);
Token t = lexer(state);
assert(!void_call);
while (1)
{
const char *origexpr = state->token;
unsigned int origexprlen = state->tokenlen;
const char *expr = state->token;
unsigned int exprlen = state->tokenlen;
if (t == ((Token) '('))
paren++;
else if (t == ((Token) ')'))
{
paren--;
if (paren < 1) // end of macro?
break;
} // else if
else if (t == ((Token) ','))
{
if (paren == 1) // new macro arg?
break;
} // else if
else if (t == ((Token) ' '))
{
// don't add whitespace to the start, so we recognize
// void calls correctly.
origexpr = expr = " ";
origexprlen = (buffer_size(origbuffer) == 0) ? 0 : 1;
exprlen = (buffer_size(buffer) == 0) ? 0 : 1;
} // else if
else if (t == TOKEN_IDENTIFIER)
{
const Define *def = find_define_by_token(ctx);
// don't replace macros with arguments so they replace correctly, later.
if ((def) && (def->paramcount == 0))
{
expr = def->definition;
exprlen = strlen(def->definition);
} // if
} // else if
else if ((t == TOKEN_INCOMPLETE_COMMENT) || (t == TOKEN_EOI))
{
pushback(state);
fail(ctx, "Unterminated macro list");
goto handle_macro_args_failed;
} // else if
assert(expr != NULL);
if (!buffer_append(buffer, expr, exprlen))
goto handle_macro_args_failed;
if (!buffer_append(origbuffer, origexpr, origexprlen))
goto handle_macro_args_failed;
t = lexer(state);
} // while
if (buffer_size(buffer) == 0)
void_call = ((saw_params == 0) && (paren == 0));
if (saw_params < expected)
{
const int origdeflen = (int) buffer_size(origbuffer);
char *origdefinition = buffer_flatten(origbuffer);
const int deflen = (int) buffer_size(buffer);
char *definition = buffer_flatten(buffer);
Define *p = get_define(ctx);
if ((!origdefinition) || (!definition) || (!p))
{
Free(ctx, origdefinition);
Free(ctx, definition);
buffer_destroy(origbuffer);
buffer_destroy(buffer);
free_define(ctx, p);
goto handle_macro_args_failed;
} // if
// trim any whitespace from the end of the string...
int i;
for (i = deflen - 1; i >= 0; i--)
{
if (definition[i] == ' ')
definition[i] = '\0';
else
break;
} // for
for (i = origdeflen - 1; i >= 0; i--)
{
if (origdefinition[i] == ' ')
origdefinition[i] = '\0';
else
break;
} // for
p->identifier = def->parameters[saw_params];
p->definition = definition;
p->original = origdefinition;
p->next = params;
params = p;
} // if
buffer_destroy(buffer);
buffer_destroy(origbuffer);
saw_params++;
} // while
assert(paren == 0);
// "a()" should match "#define a()" ...
if ((expected == 0) && (saw_params == 1) && (void_call))
{
assert(params == NULL);
saw_params = 0;
} // if
if (saw_params != expected)
{
failf(ctx, "macro '%s' passed %d arguments, but requires %d",
sym, saw_params, expected);
goto handle_macro_args_failed;
} // if
// this handles arg replacement and the '##' and '#' operators.
retval = replace_and_push_macro(ctx, def, params);
handle_macro_args_failed:
while (params)
{
Define *next = params->next;
params->identifier = NULL;
free_define(ctx, params);
params = next;
} // while
state->report_whitespace = 0;
return retval;
} // handle_macro_args
static int handle_pp_identifier(Context *ctx)
{
if (ctx->recursion_count++ >= 256) // !!! FIXME: gcc can figure this out.
{
fail(ctx, "Recursing macros");
return 0;
} // if
IncludeState *state = ctx->include_stack;
const char *fname = state->filename;
const unsigned int line = state->line;
char *sym = (char *) alloca(state->tokenlen+1);
memcpy(sym, state->token, state->tokenlen);
sym[state->tokenlen] = '\0';
// Is this identifier #defined?
const Define *def = find_define(ctx, sym);
if (def == NULL)
return 0; // just send the token through unchanged.
else if (def->paramcount != 0)
return handle_macro_args(ctx, sym, def);
const size_t deflen = strlen(def->definition);
return push_source(ctx, fname, def->definition, deflen, line, NULL);
} // handle_pp_identifier
static int find_precedence(const Token token)
{
// operator precedence, left and right associative...
typedef struct { int precedence; Token token; } Precedence;
static const Precedence ops[] = {
{ 0, TOKEN_OROR }, { 1, TOKEN_ANDAND }, { 2, ((Token) '|') },
{ 3, ((Token) '^') }, { 4, ((Token) '&') }, { 5, TOKEN_NEQ },
{ 6, TOKEN_EQL }, { 7, ((Token) '<') }, { 7, ((Token) '>') },
{ 7, TOKEN_LEQ }, { 7, TOKEN_GEQ }, { 8, TOKEN_LSHIFT },
{ 8, TOKEN_RSHIFT }, { 9, ((Token) '-') }, { 9, ((Token) '+') },
{ 10, ((Token) '%') }, { 10, ((Token) '/') }, { 10, ((Token) '*') },
{ 11, TOKEN_PP_UNARY_PLUS }, { 11, TOKEN_PP_UNARY_MINUS },
{ 11, ((Token) '!') }, { 11, ((Token) '~') },
};
size_t i;
for (i = 0; i < STATICARRAYLEN(ops); i++)
{
if (ops[i].token == token)
return ops[i].precedence;
} // for
return -1;
} // find_precedence
// !!! FIXME: we're using way too much stack space here...
typedef struct RpnTokens
{
int isoperator;
int value;
} RpnTokens;
static long interpret_rpn(const RpnTokens *tokens, int tokencount, int *error)
{
long stack[128];
size_t stacksize = 0;
*error = 1;
#define NEED_X_TOKENS(x) do { if (stacksize < x) return 0; } while (0)
#define BINARY_OPERATION(op) do { \
NEED_X_TOKENS(2); \
stack[stacksize-2] = stack[stacksize-2] op stack[stacksize-1]; \
stacksize--; \
} while (0)
#define UNARY_OPERATION(op) do { \
NEED_X_TOKENS(1); \
stack[stacksize-1] = op stack[stacksize-1]; \
} while (0)
while (tokencount-- > 0)
{
if (!tokens->isoperator)
{
assert(stacksize < STATICARRAYLEN(stack));
stack[stacksize++] = (long) tokens->value;
tokens++;
continue;
} // if
// operators.
switch (tokens->value)
{
case '!': UNARY_OPERATION(!); break;
case '~': UNARY_OPERATION(~); break;
case TOKEN_PP_UNARY_MINUS: UNARY_OPERATION(-); break;
case TOKEN_PP_UNARY_PLUS: UNARY_OPERATION(+); break;
case TOKEN_OROR: BINARY_OPERATION(||); break;
case TOKEN_ANDAND: BINARY_OPERATION(&&); break;
case '|': BINARY_OPERATION(|); break;
case '^': BINARY_OPERATION(^); break;
case '&': BINARY_OPERATION(&); break;
case TOKEN_NEQ: BINARY_OPERATION(!=); break;
case TOKEN_EQL: BINARY_OPERATION(==); break;
case '<': BINARY_OPERATION(<); break;
case '>': BINARY_OPERATION(>); break;
case TOKEN_LEQ: BINARY_OPERATION(<=); break;
case TOKEN_GEQ: BINARY_OPERATION(>=); break;
case TOKEN_LSHIFT: BINARY_OPERATION(<<); break;
case TOKEN_RSHIFT: BINARY_OPERATION(>>); break;
case '-': BINARY_OPERATION(-); break;
case '+': BINARY_OPERATION(+); break;
case '%': BINARY_OPERATION(%); break;
case '/': BINARY_OPERATION(/); break;
case '*': BINARY_OPERATION(*); break;
default: return 0;
} // switch
tokens++;
} // while
#undef NEED_X_TOKENS
#undef BINARY_OPERATION
#undef UNARY_OPERATION
if (stacksize != 1)
return 0;
*error = 0;
return stack[0];
} // interpret_rpn
// http://en.wikipedia.org/wiki/Shunting_yard_algorithm
// Convert from infix to postfix, then use this for constant folding.
// Everything that parses should fold down to a constant value: any
// identifiers that aren't resolved as macros become zero. Anything we
// don't explicitly expect becomes a parsing error.
// returns 1 (true), 0 (false), or -1 (error)
static int reduce_pp_expression(Context *ctx)
{
IncludeState *orig_state = ctx->include_stack;
RpnTokens output[128];
Token stack[64];
Token previous_token = TOKEN_UNKNOWN;
size_t outputsize = 0;
size_t stacksize = 0;
int matched = 0;
int done = 0;
#define ADD_TO_OUTPUT(op, val) \
assert(outputsize < STATICARRAYLEN(output)); \
output[outputsize].isoperator = op; \
output[outputsize].value = val; \
outputsize++;
#define PUSH_TO_STACK(t) \
assert(stacksize < STATICARRAYLEN(stack)); \
stack[stacksize] = t; \
stacksize++;
while (!done)
{
IncludeState *state = ctx->include_stack;
Token token = lexer(state);
int isleft = 1;
int precedence = -1;
if ( (token == ((Token) '!')) || (token == ((Token) '~')) )
isleft = 0;
else if (token == ((Token) '-'))
{
if ((isleft = (previous_token == TOKEN_INT_LITERAL)) == 0)
token = TOKEN_PP_UNARY_MINUS;
} // else if
else if (token == ((Token) '+'))
{
if ((isleft = (previous_token == TOKEN_INT_LITERAL)) == 0)
token = TOKEN_PP_UNARY_PLUS;
} // else if
if (token != TOKEN_IDENTIFIER)
ctx->recursion_count = 0;
switch (token)
{
case TOKEN_EOI:
if (state != orig_state) // end of a substate, or the expr?
{
pop_source(ctx);
continue; // substate, go again with the parent state.
} // if
done = 1; // the expression itself is done.
break;
case ((Token) '\n'):
done = 1;
break; // we're done!
case TOKEN_IDENTIFIER:
if (handle_pp_identifier(ctx))
continue; // go again with new IncludeState.
if ( (state->tokenlen == 7) &&
(memcmp(state->token, "defined", 7) == 0) )
{
token = lexer(state);
const int paren = (token == ((Token) '('));
if (paren) // gcc doesn't let us nest parens here, either.
token = lexer(state);
if (token != TOKEN_IDENTIFIER)
{
fail(ctx, "operator 'defined' requires an identifier");
return -1;
} // if
const int found = (find_define_by_token(ctx) != NULL);
if (paren)
{
if (lexer(state) != ((Token) ')'))
{
fail(ctx, "Unmatched ')'");
return -1;
} // if
} // if
ADD_TO_OUTPUT(0, found);
continue;
} // if
// can't replace identifier with a number? It becomes zero.
token = TOKEN_INT_LITERAL;
ADD_TO_OUTPUT(0, 0);
break;
case TOKEN_INT_LITERAL:
ADD_TO_OUTPUT(0, token_to_int(state));
break;
case ((Token) '('):
PUSH_TO_STACK((Token) '(');
break;
case ((Token) ')'):
matched = 0;
while (stacksize > 0)
{
const Token t = stack[--stacksize];
if (t == ((Token) '('))
{
matched = 1;
break;
} // if
ADD_TO_OUTPUT(1, t);
} // while
if (!matched)
{
fail(ctx, "Unmatched ')'");
return -1;
} // if
break;
default:
precedence = find_precedence(token);
// bogus token, or two operators together.
if (precedence < 0)
{
pushback(state);
fail(ctx, "Invalid expression");
return -1;
} // if
else // it's an operator.
{
while (stacksize > 0)
{
const Token t = stack[stacksize-1];
const int p = find_precedence(t);
if ( (p >= 0) &&
( ((isleft) && (precedence <= p)) ||
((!isleft) && (precedence < p)) ) )
{
stacksize--;
ADD_TO_OUTPUT(1, t);
} // if
else
{
break;
} // else
} // while
PUSH_TO_STACK(token);
} // else
break;
} // switch
previous_token = token;
} // while
while (stacksize > 0)
{
const Token t = stack[--stacksize];
if (t == ((Token) '('))
{
fail(ctx, "Unmatched ')'");
return -1;
} // if
ADD_TO_OUTPUT(1, t);
} // while
#undef ADD_TO_OUTPUT
#undef PUSH_TO_STACK
// okay, you now have some validated data in reverse polish notation.
#if DEBUG_PREPROCESSOR
printf("PREPROCESSOR EXPRESSION RPN:");
int i = 0;
for (i = 0; i < outputsize; i++)
{
if (!output[i].isoperator)
printf(" %d", output[i].value);
else
{
switch (output[i].value)
{
case TOKEN_OROR: printf(" ||"); break;
case TOKEN_ANDAND: printf(" &&"); break;
case TOKEN_NEQ: printf(" !="); break;
case TOKEN_EQL: printf(" =="); break;
case TOKEN_LEQ: printf(" <="); break;
case TOKEN_GEQ: printf(" >="); break;
case TOKEN_LSHIFT: printf(" <<"); break;
case TOKEN_RSHIFT: printf(" >>"); break;
case TOKEN_PP_UNARY_PLUS: printf(" +"); break;
case TOKEN_PP_UNARY_MINUS: printf(" -"); break;
default: printf(" %c", output[i].value); break;
} // switch
} // else
} // for
printf("\n");
#endif
int error = 0;
const long val = interpret_rpn(output, outputsize, &error);
#if DEBUG_PREPROCESSOR
printf("PREPROCESSOR RPN RESULT: %ld%s\n", val, error ? " (ERROR)" : "");
#endif
if (error)
{
fail(ctx, "Invalid expression");
return -1;
} // if
return ((val) ? 1 : 0);
} // reduce_pp_expression
static Conditional *handle_pp_if(Context *ctx)
{
IncludeState *state = ctx->include_stack;
const int result = reduce_pp_expression(ctx);
if (result == -1)
return NULL;
Conditional *conditional = get_conditional(ctx);
assert((conditional != NULL) || (ctx->out_of_memory));
if (conditional == NULL)
return NULL;
Conditional *parent = state->conditional_stack;
const int chosen = result;
const int skipping = ( (((parent) && (parent->skipping))) || (!chosen) );
conditional->type = TOKEN_PP_IF;
conditional->linenum = state->line - 1;
conditional->skipping = skipping;
conditional->chosen = chosen;
conditional->next = parent;
state->conditional_stack = conditional;
return conditional;
} // handle_pp_if
static void handle_pp_elif(Context *ctx)
{
const int rc = reduce_pp_expression(ctx);
if (rc == -1)
return;
IncludeState *state = ctx->include_stack;
Conditional *cond = state->conditional_stack;
if (cond == NULL)
fail(ctx, "#elif without #if");
else if (cond->type == TOKEN_PP_ELSE)
fail(ctx, "#elif after #else");
else
{
const Conditional *parent = cond->next;
cond->type = TOKEN_PP_ELIF;
cond->skipping = (parent && parent->skipping) || cond->chosen || !rc;
if (!cond->chosen)
cond->chosen = rc;
} // else
} // handle_pp_elif
static void handle_pp_else(Context *ctx)
{
IncludeState *state = ctx->include_stack;
Conditional *cond = state->conditional_stack;
if (!require_newline(state))
fail(ctx, "Invalid #else directive");
else if (cond == NULL)
fail(ctx, "#else without #if");
else if (cond->type == TOKEN_PP_ELSE)
fail(ctx, "#else after #else");
else
{
const Conditional *parent = cond->next;
cond->type = TOKEN_PP_ELSE;
cond->skipping = (parent && parent->skipping) || cond->chosen;
if (!cond->chosen)
cond->chosen = 1;
} // else
} // handle_pp_else
static void handle_pp_endif(Context *ctx)
{
IncludeState *state = ctx->include_stack;
Conditional *cond = state->conditional_stack;
if (!require_newline(state))
fail(ctx, "Invalid #endif directive");
else if (cond == NULL)
fail(ctx, "Unmatched #endif");
else
{
state->conditional_stack = cond->next; // pop it.
put_conditional(ctx, cond);
} // else
} // handle_pp_endif
static void unterminated_pp_condition(Context *ctx)
{
IncludeState *state = ctx->include_stack;
Conditional *cond = state->conditional_stack;
// !!! FIXME: report the line number where the #if is, not the EOI.
switch (cond->type)
{
case TOKEN_PP_IF: fail(ctx, "Unterminated #if"); break;
case TOKEN_PP_IFDEF: fail(ctx, "Unterminated #ifdef"); break;
case TOKEN_PP_IFNDEF: fail(ctx, "Unterminated #ifndef"); break;
case TOKEN_PP_ELSE: fail(ctx, "Unterminated #else"); break;
case TOKEN_PP_ELIF: fail(ctx, "Unterminated #elif"); break;
default: assert(0 && "Shouldn't hit this case"); break;
} // switch
// pop this conditional, we'll report the next error next time...
state->conditional_stack = cond->next; // pop it.
put_conditional(ctx, cond);
} // unterminated_pp_condition
static inline const char *_preprocessor_nexttoken(Preprocessor *_ctx,
unsigned int *_len, Token *_token)
{
Context *ctx = (Context *) _ctx;
while (1)
{
if (ctx->isfail)
{
ctx->isfail = 0;
*_token = TOKEN_PREPROCESSING_ERROR;
*_len = strlen(ctx->failstr);
return ctx->failstr;
} // if
IncludeState *state = ctx->include_stack;
if (state == NULL)
{
*_token = TOKEN_EOI;
*_len = 0;
return NULL; // we're done!
} // if
const Conditional *cond = state->conditional_stack;
const int skipping = ((cond != NULL) && (cond->skipping));
const Token token = lexer(state);
if (token != TOKEN_IDENTIFIER)
ctx->recursion_count = 0;
if (token == TOKEN_EOI)
{
assert(state->bytes_left == 0);
if (state->conditional_stack != NULL)
{
unterminated_pp_condition(ctx);
continue; // returns an error.
} // if
pop_source(ctx);
continue; // pick up again after parent's #include line.
} // if
else if (token == TOKEN_INCOMPLETE_COMMENT)
{
fail(ctx, "Incomplete multiline comment");
continue; // will return at top of loop.
} // else if
else if (token == TOKEN_PP_IFDEF)
{
handle_pp_ifdef(ctx);
continue; // get the next thing.
} // else if
else if (token == TOKEN_PP_IFNDEF)
{
handle_pp_ifndef(ctx);
continue; // get the next thing.
} // else if
else if (token == TOKEN_PP_IF)
{
handle_pp_if(ctx);
continue; // get the next thing.
} // else if
else if (token == TOKEN_PP_ELIF)
{
handle_pp_elif(ctx);
continue; // get the next thing.
} // else if
else if (token == TOKEN_PP_ENDIF)
{
handle_pp_endif(ctx);
continue; // get the next thing.
} // else if
else if (token == TOKEN_PP_ELSE)
{
handle_pp_else(ctx);
continue; // get the next thing.
} // else if
// NOTE: Conditionals must be above (skipping) test.
else if (skipping)
continue; // just keep dumping tokens until we get end of block.
else if (token == TOKEN_PP_INCLUDE)
{
handle_pp_include(ctx);
continue; // will return error or use new top of include_stack.
} // else if
else if (token == TOKEN_PP_LINE)
{
handle_pp_line(ctx);
continue; // get the next thing.
} // else if
else if (token == TOKEN_PP_ERROR)
{
handle_pp_error(ctx);
continue; // will return at top of loop.
} // else if
else if (token == TOKEN_PP_DEFINE)
{
handle_pp_define(ctx);
continue; // will return at top of loop.
} // else if
else if (token == TOKEN_PP_UNDEF)
{
handle_pp_undef(ctx);
continue; // will return at top of loop.
} // else if
else if (token == TOKEN_PP_PRAGMA)
{
ctx->parsing_pragma = 1;
} // else if
if (token == TOKEN_IDENTIFIER)
{
if (handle_pp_identifier(ctx))
continue; // pushed the include_stack.
} // else if
else if (token == ((Token) '\n'))
{
print_debug_lexing_position(state);
if (ctx->parsing_pragma) // let this one through.
ctx->parsing_pragma = 0;
else
{
// preprocessor is line-oriented, nothing else gets newlines.
continue; // get the next thing.
} // else
} // else if
assert(!skipping);
*_token = token;
*_len = state->tokenlen;
return state->token;
} // while
assert(0 && "shouldn't hit this code");
*_token = TOKEN_UNKNOWN;
*_len = 0;
return NULL;
} // _preprocessor_nexttoken
const char *preprocessor_nexttoken(Preprocessor *ctx, unsigned int *len,
Token *token)
{
const char *retval = _preprocessor_nexttoken(ctx, len, token);
print_debug_token(retval, *len, *token);
return retval;
} // preprocessor_nexttoken
const char *preprocessor_sourcepos(Preprocessor *_ctx, unsigned int *pos)
{
Context *ctx = (Context *) _ctx;
if (ctx->include_stack == NULL)
{
*pos = 0;
return NULL;
} // if
*pos = ctx->include_stack->line;
return ctx->include_stack->filename;
} // preprocessor_sourcepos
static void indent_buffer(Buffer *buffer, int n, const int newline)
{
static char spaces[4] = { ' ', ' ', ' ', ' ' };
if (newline)
{
while (n--)
{
if (!buffer_append(buffer, spaces, sizeof (spaces)))
return;
} // while
} // if
else
{
if (!buffer_append(buffer, spaces, 1))
return;
} // else
} // indent_buffer
static const MOJOSHADER_preprocessData out_of_mem_data_preprocessor = {
1, &MOJOSHADER_out_of_mem_error, 0, 0, 0, 0, 0
};
// public API...
const MOJOSHADER_preprocessData *MOJOSHADER_preprocess(const char *filename,
const char *source, unsigned int sourcelen,
const MOJOSHADER_preprocessorDefine *defines,
unsigned int define_count,
MOJOSHADER_includeOpen include_open,
MOJOSHADER_includeClose include_close,
MOJOSHADER_malloc m, MOJOSHADER_free f, void *d)
{
MOJOSHADER_preprocessData *retval = NULL;
Preprocessor *pp = NULL;
ErrorList *errors = NULL;
Buffer *buffer = NULL;
Token token = TOKEN_UNKNOWN;
const char *tokstr = NULL;
int nl = 1;
int indent = 0;
unsigned int len = 0;
char *output = NULL;
int errcount = 0;
size_t total_bytes = 0;
// !!! FIXME: what's wrong with ENDLINE_STR?
#ifdef _WINDOWS
static const char endline[] = { '\r', '\n' };
#else
static const char endline[] = { '\n' };
#endif
if (!m) m = MOJOSHADER_internal_malloc;
if (!f) f = MOJOSHADER_internal_free;
if (!include_open) include_open = MOJOSHADER_internal_include_open;
if (!include_close) include_close = MOJOSHADER_internal_include_close;
pp = preprocessor_start(filename, source, sourcelen,
include_open, include_close,
defines, define_count, 0, m, f, d);
if (pp == NULL)
goto preprocess_out_of_mem;
errors = errorlist_create(MallocBridge, FreeBridge, pp);
if (errors == NULL)
goto preprocess_out_of_mem;
buffer = buffer_create(4096, MallocBridge, FreeBridge, pp);
if (buffer == NULL)
goto preprocess_out_of_mem;
while ((tokstr = preprocessor_nexttoken(pp, &len, &token)) != NULL)
{
int isnewline = 0;
assert(token != TOKEN_EOI);
if (preprocessor_outofmemory(pp))
goto preprocess_out_of_mem;
// Microsoft's preprocessor is weird.
// It ignores newlines, and then inserts its own around certain
// tokens. For example, after a semicolon. This allows HLSL code to
// be mostly readable, instead of a stream of tokens.
if ( (token == ((Token) '}')) || (token == ((Token) ';')) )
{
if ( (token == ((Token) '}')) && (indent > 0) )
indent--;
indent_buffer(buffer, indent, nl);
buffer_append(buffer, tokstr, len);
buffer_append(buffer, endline, sizeof (endline));
isnewline = 1;
} // if
else if (token == ((Token) '\n'))
{
buffer_append(buffer, endline, sizeof (endline));
isnewline = 1;
} // else if
else if (token == ((Token) '{'))
{
buffer_append(buffer, endline, sizeof (endline));
indent_buffer(buffer, indent, 1);
buffer_append(buffer, "{", 1);
buffer_append(buffer, endline, sizeof (endline));
indent++;
isnewline = 1;
} // else if
else if (token == TOKEN_PREPROCESSING_ERROR)
{
unsigned int pos = 0;
const char *fname = preprocessor_sourcepos(pp, &pos);
errorlist_add(errors, fname, (int) pos, tokstr);
} // else if
else
{
indent_buffer(buffer, indent, nl);
buffer_append(buffer, tokstr, len);
} // else
nl = isnewline;
} // while
assert(token == TOKEN_EOI);
total_bytes = buffer_size(buffer);
output = buffer_flatten(buffer);
buffer_destroy(buffer);
buffer = NULL; // don't free this pointer again.
if (output == NULL)
goto preprocess_out_of_mem;
retval = (MOJOSHADER_preprocessData *) m(sizeof (*retval), d);
if (retval == NULL)
goto preprocess_out_of_mem;
memset(retval, '\0', sizeof (*retval));
errcount = errorlist_count(errors);
if (errcount > 0)
{
retval->error_count = errcount;
retval->errors = errorlist_flatten(errors);
if (retval->errors == NULL)
goto preprocess_out_of_mem;
} // if
retval->output = output;
retval->output_len = total_bytes;
retval->malloc = m;
retval->free = f;
retval->malloc_data = d;
errorlist_destroy(errors);
preprocessor_end(pp);
return retval;
preprocess_out_of_mem:
if (retval != NULL)
f(retval->errors, d);
f(retval, d);
f(output, d);
buffer_destroy(buffer);
errorlist_destroy(errors);
preprocessor_end(pp);
return &out_of_mem_data_preprocessor;
} // MOJOSHADER_preprocess
void MOJOSHADER_freePreprocessData(const MOJOSHADER_preprocessData *_data)
{
MOJOSHADER_preprocessData *data = (MOJOSHADER_preprocessData *) _data;
if ((data == NULL) || (data == &out_of_mem_data_preprocessor))
return;
MOJOSHADER_free f = (data->free == NULL) ? MOJOSHADER_internal_free : data->free;
void *d = data->malloc_data;
int i;
f((void *) data->output, d);
for (i = 0; i < data->error_count; i++)
{
f((void *) data->errors[i].error, d);
f((void *) data->errors[i].filename, d);
} // for
f(data->errors, d);
f(data, d);
} // MOJOSHADER_freePreprocessData
// end of mojoshader_preprocessor.c ...
Reference in New Issue View Git Blame Copy Permalink