LLVM API Documentation

JITEmitter.cpp

Go to the documentation of this file.
00001 //===-- JITEmitter.cpp - Write machine code to executable memory ----------===//
00002 //
00003 //                     The LLVM Compiler Infrastructure
00004 //
00005 // This file is distributed under the University of Illinois Open Source
00006 // License. See LICENSE.TXT for details.
00007 //
00008 //===----------------------------------------------------------------------===//
00009 //
00010 // This file defines a MachineCodeEmitter object that is used by the JIT to
00011 // write machine code to memory and remember where relocatable values are.
00012 //
00013 //===----------------------------------------------------------------------===//
00014 
00015 #define DEBUG_TYPE "jit"
00016 #include "JIT.h"
00017 #include "JITDwarfEmitter.h"
00018 #include "llvm/Constants.h"
00019 #include "llvm/Module.h"
00020 #include "llvm/DerivedTypes.h"
00021 #include "llvm/CodeGen/JITCodeEmitter.h"
00022 #include "llvm/CodeGen/MachineFunction.h"
00023 #include "llvm/CodeGen/MachineConstantPool.h"
00024 #include "llvm/CodeGen/MachineJumpTableInfo.h"
00025 #include "llvm/CodeGen/MachineModuleInfo.h"
00026 #include "llvm/CodeGen/MachineRelocation.h"
00027 #include "llvm/ExecutionEngine/GenericValue.h"
00028 #include "llvm/ExecutionEngine/JITEventListener.h"
00029 #include "llvm/ExecutionEngine/JITMemoryManager.h"
00030 #include "llvm/CodeGen/MachineCodeInfo.h"
00031 #include "llvm/Target/TargetData.h"
00032 #include "llvm/Target/TargetJITInfo.h"
00033 #include "llvm/Target/TargetMachine.h"
00034 #include "llvm/Target/TargetOptions.h"
00035 #include "llvm/Support/Debug.h"
00036 #include "llvm/Support/ErrorHandling.h"
00037 #include "llvm/Support/MutexGuard.h"
00038 #include "llvm/Support/ValueHandle.h"
00039 #include "llvm/Support/raw_ostream.h"
00040 #include "llvm/System/Disassembler.h"
00041 #include "llvm/System/Memory.h"
00042 #include "llvm/Target/TargetInstrInfo.h"
00043 #include "llvm/ADT/SmallPtrSet.h"
00044 #include "llvm/ADT/SmallVector.h"
00045 #include "llvm/ADT/Statistic.h"
00046 #include <algorithm>
00047 #ifndef NDEBUG
00048 #include <iomanip>
00049 #endif
00050 using namespace llvm;
00051 
00052 STATISTIC(NumBytes, "Number of bytes of machine code compiled");
00053 STATISTIC(NumRelos, "Number of relocations applied");
00054 STATISTIC(NumRetries, "Number of retries with more memory");
00055 static JIT *TheJIT = 0;
00056 
00057 
00058 //===----------------------------------------------------------------------===//
00059 // JIT lazy compilation code.
00060 //
00061 namespace {
00062   class JITResolverState {
00063   public:
00064     typedef std::map<AssertingVH<Function>, void*> FunctionToStubMapTy;
00065     typedef std::map<void*, Function*> StubToFunctionMapTy;
00066     typedef std::map<AssertingVH<GlobalValue>, void*> GlobalToIndirectSymMapTy;
00067   private:
00068     /// FunctionToStubMap - Keep track of the stub created for a particular
00069     /// function so that we can reuse them if necessary.
00070     FunctionToStubMapTy FunctionToStubMap;
00071 
00072     /// StubToFunctionMap - Keep track of the function that each stub
00073     /// corresponds to.
00074     StubToFunctionMapTy StubToFunctionMap;
00075 
00076     /// GlobalToIndirectSymMap - Keep track of the indirect symbol created for a
00077     /// particular GlobalVariable so that we can reuse them if necessary.
00078     GlobalToIndirectSymMapTy GlobalToIndirectSymMap;
00079 
00080   public:
00081     FunctionToStubMapTy& getFunctionToStubMap(const MutexGuard& locked) {
00082       assert(locked.holds(TheJIT->lock));
00083       return FunctionToStubMap;
00084     }
00085 
00086     StubToFunctionMapTy& getStubToFunctionMap(const MutexGuard& locked) {
00087       assert(locked.holds(TheJIT->lock));
00088       return StubToFunctionMap;
00089     }
00090 
00091     GlobalToIndirectSymMapTy& getGlobalToIndirectSymMap(const MutexGuard& locked) {
00092       assert(locked.holds(TheJIT->lock));
00093       return GlobalToIndirectSymMap;
00094     }
00095   };
00096 
00097   /// JITResolver - Keep track of, and resolve, call sites for functions that
00098   /// have not yet been compiled.
00099   class JITResolver {
00100     typedef JITResolverState::FunctionToStubMapTy FunctionToStubMapTy;
00101     typedef JITResolverState::StubToFunctionMapTy StubToFunctionMapTy;
00102     typedef JITResolverState::GlobalToIndirectSymMapTy GlobalToIndirectSymMapTy;
00103 
00104     /// LazyResolverFn - The target lazy resolver function that we actually
00105     /// rewrite instructions to use.
00106     TargetJITInfo::LazyResolverFn LazyResolverFn;
00107 
00108     JITResolverState state;
00109 
00110     /// ExternalFnToStubMap - This is the equivalent of FunctionToStubMap for
00111     /// external functions.
00112     std::map<void*, void*> ExternalFnToStubMap;
00113 
00114     /// revGOTMap - map addresses to indexes in the GOT
00115     std::map<void*, unsigned> revGOTMap;
00116     unsigned nextGOTIndex;
00117 
00118     static JITResolver *TheJITResolver;
00119   public:
00120     explicit JITResolver(JIT &jit) : nextGOTIndex(0) {
00121       TheJIT = &jit;
00122 
00123       LazyResolverFn = jit.getJITInfo().getLazyResolverFunction(JITCompilerFn);
00124       assert(TheJITResolver == 0 && "Multiple JIT resolvers?");
00125       TheJITResolver = this;
00126     }
00127     
00128     ~JITResolver() {
00129       TheJITResolver = 0;
00130     }
00131 
00132     /// getFunctionStubIfAvailable - This returns a pointer to a function stub
00133     /// if it has already been created.
00134     void *getFunctionStubIfAvailable(Function *F);
00135 
00136     /// getFunctionStub - This returns a pointer to a function stub, creating
00137     /// one on demand as needed.  If empty is true, create a function stub
00138     /// pointing at address 0, to be filled in later.
00139     void *getFunctionStub(Function *F);
00140 
00141     /// getExternalFunctionStub - Return a stub for the function at the
00142     /// specified address, created lazily on demand.
00143     void *getExternalFunctionStub(void *FnAddr);
00144 
00145     /// getGlobalValueIndirectSym - Return an indirect symbol containing the
00146     /// specified GV address.
00147     void *getGlobalValueIndirectSym(GlobalValue *V, void *GVAddress);
00148 
00149     /// AddCallbackAtLocation - If the target is capable of rewriting an
00150     /// instruction without the use of a stub, record the location of the use so
00151     /// we know which function is being used at the location.
00152     void *AddCallbackAtLocation(Function *F, void *Location) {
00153       MutexGuard locked(TheJIT->lock);
00154       /// Get the target-specific JIT resolver function.
00155       state.getStubToFunctionMap(locked)[Location] = F;
00156       return (void*)(intptr_t)LazyResolverFn;
00157     }
00158     
00159     void getRelocatableGVs(SmallVectorImpl<GlobalValue*> &GVs,
00160                            SmallVectorImpl<void*> &Ptrs);
00161     
00162     GlobalValue *invalidateStub(void *Stub);
00163 
00164     /// getGOTIndexForAddress - Return a new or existing index in the GOT for
00165     /// an address.  This function only manages slots, it does not manage the
00166     /// contents of the slots or the memory associated with the GOT.
00167     unsigned getGOTIndexForAddr(void *addr);
00168 
00169     /// JITCompilerFn - This function is called to resolve a stub to a compiled
00170     /// address.  If the LLVM Function corresponding to the stub has not yet
00171     /// been compiled, this function compiles it first.
00172     static void *JITCompilerFn(void *Stub);
00173   };
00174 }
00175 
00176 JITResolver *JITResolver::TheJITResolver = 0;
00177 
00178 /// getFunctionStubIfAvailable - This returns a pointer to a function stub
00179 /// if it has already been created.
00180 void *JITResolver::getFunctionStubIfAvailable(Function *F) {
00181   MutexGuard locked(TheJIT->lock);
00182 
00183   // If we already have a stub for this function, recycle it.
00184   void *&Stub = state.getFunctionToStubMap(locked)[F];
00185   return Stub;
00186 }
00187 
00188 /// getFunctionStub - This returns a pointer to a function stub, creating
00189 /// one on demand as needed.
00190 void *JITResolver::getFunctionStub(Function *F) {
00191   MutexGuard locked(TheJIT->lock);
00192 
00193   // If we already have a stub for this function, recycle it.
00194   void *&Stub = state.getFunctionToStubMap(locked)[F];
00195   if (Stub) return Stub;
00196 
00197   // Call the lazy resolver function unless we are JIT'ing non-lazily, in which
00198   // case we must resolve the symbol now.
00199   void *Actual =  TheJIT->isLazyCompilationDisabled() 
00200     ? (void *)0 : (void *)(intptr_t)LazyResolverFn;
00201    
00202   // If this is an external declaration, attempt to resolve the address now
00203   // to place in the stub.
00204   if (F->isDeclaration() && !F->hasNotBeenReadFromBitcode()) {
00205     Actual = TheJIT->getPointerToFunction(F);
00206 
00207     // If we resolved the symbol to a null address (eg. a weak external)
00208     // don't emit a stub. Return a null pointer to the application.  If dlsym
00209     // stubs are enabled, not being able to resolve the address is not
00210     // meaningful.
00211     if (!Actual && !TheJIT->areDlsymStubsEnabled()) return 0;
00212   }
00213 
00214   // Codegen a new stub, calling the lazy resolver or the actual address of the
00215   // external function, if it was resolved.
00216   Stub = TheJIT->getJITInfo().emitFunctionStub(F, Actual,
00217                                                *TheJIT->getCodeEmitter());
00218 
00219   if (Actual != (void*)(intptr_t)LazyResolverFn) {
00220     // If we are getting the stub for an external function, we really want the
00221     // address of the stub in the GlobalAddressMap for the JIT, not the address
00222     // of the external function.
00223     TheJIT->updateGlobalMapping(F, Stub);
00224   }
00225 
00226   DEBUG(errs() << "JIT: Stub emitted at [" << Stub << "] for function '"
00227         << F->getName() << "'\n");
00228 
00229   // Finally, keep track of the stub-to-Function mapping so that the
00230   // JITCompilerFn knows which function to compile!
00231   state.getStubToFunctionMap(locked)[Stub] = F;
00232   
00233   // If we are JIT'ing non-lazily but need to call a function that does not
00234   // exist yet, add it to the JIT's work list so that we can fill in the stub
00235   // address later.
00236   if (!Actual && TheJIT->isLazyCompilationDisabled())
00237     if (!F->isDeclaration() || F->hasNotBeenReadFromBitcode())
00238       TheJIT->addPendingFunction(F);
00239   
00240   return Stub;
00241 }
00242 
00243 /// getGlobalValueIndirectSym - Return a lazy pointer containing the specified
00244 /// GV address.
00245 void *JITResolver::getGlobalValueIndirectSym(GlobalValue *GV, void *GVAddress) {
00246   MutexGuard locked(TheJIT->lock);
00247 
00248   // If we already have a stub for this global variable, recycle it.
00249   void *&IndirectSym = state.getGlobalToIndirectSymMap(locked)[GV];
00250   if (IndirectSym) return IndirectSym;
00251 
00252   // Otherwise, codegen a new indirect symbol.
00253   IndirectSym = TheJIT->getJITInfo().emitGlobalValueIndirectSym(GV, GVAddress,
00254                                                      *TheJIT->getCodeEmitter());
00255 
00256   DEBUG(errs() << "JIT: Indirect symbol emitted at [" << IndirectSym 
00257         << "] for GV '" << GV->getName() << "'\n");
00258 
00259   return IndirectSym;
00260 }
00261 
00262 /// getExternalFunctionStub - Return a stub for the function at the
00263 /// specified address, created lazily on demand.
00264 void *JITResolver::getExternalFunctionStub(void *FnAddr) {
00265   // If we already have a stub for this function, recycle it.
00266   void *&Stub = ExternalFnToStubMap[FnAddr];
00267   if (Stub) return Stub;
00268 
00269   Stub = TheJIT->getJITInfo().emitFunctionStub(0, FnAddr,
00270                                                *TheJIT->getCodeEmitter());
00271 
00272   DOUT << "JIT: Stub emitted at [" << Stub
00273        << "] for external function at '" << FnAddr << "'\n";
00274   return Stub;
00275 }
00276 
00277 unsigned JITResolver::getGOTIndexForAddr(void* addr) {
00278   unsigned idx = revGOTMap[addr];
00279   if (!idx) {
00280     idx = ++nextGOTIndex;
00281     revGOTMap[addr] = idx;
00282     DOUT << "JIT: Adding GOT entry " << idx << " for addr [" << addr << "]\n";
00283   }
00284   return idx;
00285 }
00286 
00287 void JITResolver::getRelocatableGVs(SmallVectorImpl<GlobalValue*> &GVs,
00288                                     SmallVectorImpl<void*> &Ptrs) {
00289   MutexGuard locked(TheJIT->lock);
00290   
00291   FunctionToStubMapTy &FM = state.getFunctionToStubMap(locked);
00292   GlobalToIndirectSymMapTy &GM = state.getGlobalToIndirectSymMap(locked);
00293   
00294   for (FunctionToStubMapTy::iterator i = FM.begin(), e = FM.end(); i != e; ++i){
00295     Function *F = i->first;
00296     if (F->isDeclaration() && F->hasExternalLinkage()) {
00297       GVs.push_back(i->first);
00298       Ptrs.push_back(i->second);
00299     }
00300   }
00301   for (GlobalToIndirectSymMapTy::iterator i = GM.begin(), e = GM.end();
00302        i != e; ++i) {
00303     GVs.push_back(i->first);
00304     Ptrs.push_back(i->second);
00305   }
00306 }
00307 
00308 GlobalValue *JITResolver::invalidateStub(void *Stub) {
00309   MutexGuard locked(TheJIT->lock);
00310   
00311   FunctionToStubMapTy &FM = state.getFunctionToStubMap(locked);
00312   StubToFunctionMapTy &SM = state.getStubToFunctionMap(locked);
00313   GlobalToIndirectSymMapTy &GM = state.getGlobalToIndirectSymMap(locked);
00314   
00315   // Look up the cheap way first, to see if it's a function stub we are
00316   // invalidating.  If so, remove it from both the forward and reverse maps.
00317   if (SM.find(Stub) != SM.end()) {
00318     Function *F = SM[Stub];
00319     SM.erase(Stub);
00320     FM.erase(F);
00321     return F;
00322   }
00323   
00324   // Otherwise, it might be an indirect symbol stub.  Find it and remove it.
00325   for (GlobalToIndirectSymMapTy::iterator i = GM.begin(), e = GM.end();
00326        i != e; ++i) {
00327     if (i->second != Stub)
00328       continue;
00329     GlobalValue *GV = i->first;
00330     GM.erase(i);
00331     return GV;
00332   }
00333   
00334   // Lastly, check to see if it's in the ExternalFnToStubMap.
00335   for (std::map<void *, void *>::iterator i = ExternalFnToStubMap.begin(),
00336        e = ExternalFnToStubMap.end(); i != e; ++i) {
00337     if (i->second != Stub)
00338       continue;
00339     ExternalFnToStubMap.erase(i);
00340     break;
00341   }
00342   
00343   return 0;
00344 }
00345 
00346 /// JITCompilerFn - This function is called when a lazy compilation stub has
00347 /// been entered.  It looks up which function this stub corresponds to, compiles
00348 /// it if necessary, then returns the resultant function pointer.
00349 void *JITResolver::JITCompilerFn(void *Stub) {
00350   JITResolver &JR = *TheJITResolver;
00351   
00352   Function* F = 0;
00353   void* ActualPtr = 0;
00354 
00355   {
00356     // Only lock for getting the Function. The call getPointerToFunction made
00357     // in this function might trigger function materializing, which requires
00358     // JIT lock to be unlocked.
00359     MutexGuard locked(TheJIT->lock);
00360 
00361     // The address given to us for the stub may not be exactly right, it might be
00362     // a little bit after the stub.  As such, use upper_bound to find it.
00363     StubToFunctionMapTy::iterator I =
00364       JR.state.getStubToFunctionMap(locked).upper_bound(Stub);
00365     assert(I != JR.state.getStubToFunctionMap(locked).begin() &&
00366            "This is not a known stub!");
00367     F = (--I)->second;
00368     ActualPtr = I->first;
00369   }
00370 
00371   // If we have already code generated the function, just return the address.
00372   void *Result = TheJIT->getPointerToGlobalIfAvailable(F);
00373   
00374   if (!Result) {
00375     // Otherwise we don't have it, do lazy compilation now.
00376     
00377     // If lazy compilation is disabled, emit a useful error message and abort.
00378     if (TheJIT->isLazyCompilationDisabled()) {
00379       llvm_report_error("LLVM JIT requested to do lazy compilation of function '"
00380                         + F->getName() + "' when lazy compiles are disabled!");
00381     }
00382   
00383     // We might like to remove the stub from the StubToFunction map.
00384     // We can't do that! Multiple threads could be stuck, waiting to acquire the
00385     // lock above. As soon as the 1st function finishes compiling the function,
00386     // the next one will be released, and needs to be able to find the function
00387     // it needs to call.
00388     //JR.state.getStubToFunctionMap(locked).erase(I);
00389 
00390     DEBUG(errs() << "JIT: Lazily resolving function '" << F->getName()
00391           << "' In stub ptr = " << Stub << " actual ptr = "
00392           << ActualPtr << "\n");
00393 
00394     Result = TheJIT->getPointerToFunction(F);
00395   }
00396   
00397   // Reacquire the lock to erase the stub in the map.
00398   MutexGuard locked(TheJIT->lock);
00399 
00400   // We don't need to reuse this stub in the future, as F is now compiled.
00401   JR.state.getFunctionToStubMap(locked).erase(F);
00402 
00403   // FIXME: We could rewrite all references to this stub if we knew them.
00404 
00405   // What we will do is set the compiled function address to map to the
00406   // same GOT entry as the stub so that later clients may update the GOT
00407   // if they see it still using the stub address.
00408   // Note: this is done so the Resolver doesn't have to manage GOT memory
00409   // Do this without allocating map space if the target isn't using a GOT
00410   if(JR.revGOTMap.find(Stub) != JR.revGOTMap.end())
00411     JR.revGOTMap[Result] = JR.revGOTMap[Stub];
00412 
00413   return Result;
00414 }
00415 
00416 //===----------------------------------------------------------------------===//
00417 // JITEmitter code.
00418 //
00419 namespace {
00420   /// JITEmitter - The JIT implementation of the MachineCodeEmitter, which is
00421   /// used to output functions to memory for execution.
00422   class JITEmitter : public JITCodeEmitter {
00423     JITMemoryManager *MemMgr;
00424 
00425     // When outputting a function stub in the context of some other function, we
00426     // save BufferBegin/BufferEnd/CurBufferPtr here.
00427     uint8_t *SavedBufferBegin, *SavedBufferEnd, *SavedCurBufferPtr;
00428 
00429     // When reattempting to JIT a function after running out of space, we store
00430     // the estimated size of the function we're trying to JIT here, so we can
00431     // ask the memory manager for at least this much space.  When we
00432     // successfully emit the function, we reset this back to zero.
00433     uintptr_t SizeEstimate;
00434 
00435     /// Relocations - These are the relocations that the function needs, as
00436     /// emitted.
00437     std::vector<MachineRelocation> Relocations;
00438     
00439     /// MBBLocations - This vector is a mapping from MBB ID's to their address.
00440     /// It is filled in by the StartMachineBasicBlock callback and queried by
00441     /// the getMachineBasicBlockAddress callback.
00442     std::vector<uintptr_t> MBBLocations;
00443 
00444     /// ConstantPool - The constant pool for the current function.
00445     ///
00446     MachineConstantPool *ConstantPool;
00447 
00448     /// ConstantPoolBase - A pointer to the first entry in the constant pool.
00449     ///
00450     void *ConstantPoolBase;
00451 
00452     /// ConstPoolAddresses - Addresses of individual constant pool entries.
00453     ///
00454     SmallVector<uintptr_t, 8> ConstPoolAddresses;
00455 
00456     /// JumpTable - The jump tables for the current function.
00457     ///
00458     MachineJumpTableInfo *JumpTable;
00459     
00460     /// JumpTableBase - A pointer to the first entry in the jump table.
00461     ///
00462     void *JumpTableBase;
00463 
00464     /// Resolver - This contains info about the currently resolved functions.
00465     JITResolver Resolver;
00466     
00467     /// DE - The dwarf emitter for the jit.
00468     JITDwarfEmitter *DE;
00469 
00470     /// LabelLocations - This vector is a mapping from Label ID's to their 
00471     /// address.
00472     std::vector<uintptr_t> LabelLocations;
00473 
00474     /// MMI - Machine module info for exception informations
00475     MachineModuleInfo* MMI;
00476 
00477     // GVSet - a set to keep track of which globals have been seen
00478     SmallPtrSet<const GlobalVariable*, 8> GVSet;
00479 
00480     // CurFn - The llvm function being emitted.  Only valid during 
00481     // finishFunction().
00482     const Function *CurFn;
00483 
00484     /// Information about emitted code, which is passed to the
00485     /// JITEventListeners.  This is reset in startFunction and used in
00486     /// finishFunction.
00487     JITEvent_EmittedFunctionDetails EmissionDetails;
00488 
00489     // CurFnStubUses - For a given Function, a vector of stubs that it
00490     // references.  This facilitates the JIT detecting that a stub is no
00491     // longer used, so that it may be deallocated.
00492     DenseMap<const Function *, SmallVector<void*, 1> > CurFnStubUses;
00493     
00494     // StubFnRefs - For a given pointer to a stub, a set of Functions which
00495     // reference the stub.  When the count of a stub's references drops to zero,
00496     // the stub is unused.
00497     DenseMap<void *, SmallPtrSet<const Function*, 1> > StubFnRefs;
00498     
00499     // ExtFnStubs - A map of external function names to stubs which have entries
00500     // in the JITResolver's ExternalFnToStubMap.
00501     StringMap<void *> ExtFnStubs;
00502 
00503     DebugLocTuple PrevDLT;
00504 
00505   public:
00506     JITEmitter(JIT &jit, JITMemoryManager *JMM)
00507         : SizeEstimate(0), Resolver(jit), MMI(0), CurFn(0) {
00508       MemMgr = JMM ? JMM : JITMemoryManager::CreateDefaultMemManager();
00509       if (jit.getJITInfo().needsGOT()) {
00510         MemMgr->AllocateGOT();
00511         DOUT << "JIT is managing a GOT\n";
00512       }
00513 
00514       if (DwarfExceptionHandling) DE = new JITDwarfEmitter(jit);
00515     }
00516     ~JITEmitter() { 
00517       delete MemMgr;
00518       if (DwarfExceptionHandling) delete DE;
00519     }
00520 
00521     /// classof - Methods for support type inquiry through isa, cast, and
00522     /// dyn_cast:
00523     ///
00524     static inline bool classof(const JITEmitter*) { return true; }
00525     static inline bool classof(const MachineCodeEmitter*) { return true; }
00526     
00527     JITResolver &getJITResolver() { return Resolver; }
00528 
00529     virtual void startFunction(MachineFunction &F);
00530     virtual bool finishFunction(MachineFunction &F);
00531     
00532     void emitConstantPool(MachineConstantPool *MCP);
00533     void initJumpTableInfo(MachineJumpTableInfo *MJTI);
00534     void emitJumpTableInfo(MachineJumpTableInfo *MJTI);
00535     
00536     virtual void startGVStub(const GlobalValue* GV, unsigned StubSize,
00537                                    unsigned Alignment = 1);
00538     virtual void startGVStub(const GlobalValue* GV, void *Buffer,
00539                              unsigned StubSize);
00540     virtual void* finishGVStub(const GlobalValue *GV);
00541 
00542     /// allocateSpace - Reserves space in the current block if any, or
00543     /// allocate a new one of the given size.
00544     virtual void *allocateSpace(uintptr_t Size, unsigned Alignment);
00545 
00546     /// allocateGlobal - Allocate memory for a global.  Unlike allocateSpace,
00547     /// this method does not allocate memory in the current output buffer,
00548     /// because a global may live longer than the current function.
00549     virtual void *allocateGlobal(uintptr_t Size, unsigned Alignment);
00550 
00551     virtual void addRelocation(const MachineRelocation &MR) {
00552       Relocations.push_back(MR);
00553     }
00554     
00555     virtual void StartMachineBasicBlock(MachineBasicBlock *MBB) {
00556       if (MBBLocations.size() <= (unsigned)MBB->getNumber())
00557         MBBLocations.resize((MBB->getNumber()+1)*2);
00558       MBBLocations[MBB->getNumber()] = getCurrentPCValue();
00559       DOUT << "JIT: Emitting BB" << MBB->getNumber() << " at ["
00560            << (void*) getCurrentPCValue() << "]\n";
00561     }
00562 
00563     virtual uintptr_t getConstantPoolEntryAddress(unsigned Entry) const;
00564     virtual uintptr_t getJumpTableEntryAddress(unsigned Entry) const;
00565 
00566     virtual uintptr_t getMachineBasicBlockAddress(MachineBasicBlock *MBB) const {
00567       assert(MBBLocations.size() > (unsigned)MBB->getNumber() && 
00568              MBBLocations[MBB->getNumber()] && "MBB not emitted!");
00569       return MBBLocations[MBB->getNumber()];
00570     }
00571 
00572     /// retryWithMoreMemory - Log a retry and deallocate all memory for the
00573     /// given function.  Increase the minimum allocation size so that we get
00574     /// more memory next time.
00575     void retryWithMoreMemory(MachineFunction &F);
00576 
00577     /// deallocateMemForFunction - Deallocate all memory for the specified
00578     /// function body.
00579     void deallocateMemForFunction(const Function *F);
00580 
00581     /// AddStubToCurrentFunction - Mark the current function being JIT'd as
00582     /// using the stub at the specified address. Allows
00583     /// deallocateMemForFunction to also remove stubs no longer referenced.
00584     void AddStubToCurrentFunction(void *Stub);
00585     
00586     /// getExternalFnStubs - Accessor for the JIT to find stubs emitted for
00587     /// MachineRelocations that reference external functions by name.
00588     const StringMap<void*> &getExternalFnStubs() const { return ExtFnStubs; }
00589     
00590     virtual void processDebugLoc(DebugLoc DL);
00591 
00592     virtual void emitLabel(uint64_t LabelID) {
00593       if (LabelLocations.size() <= LabelID)
00594         LabelLocations.resize((LabelID+1)*2);
00595       LabelLocations[LabelID] = getCurrentPCValue();
00596     }
00597 
00598     virtual uintptr_t getLabelAddress(uint64_t LabelID) const {
00599       assert(LabelLocations.size() > (unsigned)LabelID && 
00600              LabelLocations[LabelID] && "Label not emitted!");
00601       return LabelLocations[LabelID];
00602     }
00603  
00604     virtual void setModuleInfo(MachineModuleInfo* Info) {
00605       MMI = Info;
00606       if (DwarfExceptionHandling) DE->setModuleInfo(Info);
00607     }
00608 
00609     void setMemoryExecutable() {
00610       MemMgr->setMemoryExecutable();
00611     }
00612     
00613     JITMemoryManager *getMemMgr() const { return MemMgr; }
00614 
00615   private:
00616     void *getPointerToGlobal(GlobalValue *GV, void *Reference, bool NoNeedStub);
00617     void *getPointerToGVIndirectSym(GlobalValue *V, void *Reference,
00618                                     bool NoNeedStub);
00619     unsigned addSizeOfGlobal(const GlobalVariable *GV, unsigned Size);
00620     unsigned addSizeOfGlobalsInConstantVal(const Constant *C, unsigned Size);
00621     unsigned addSizeOfGlobalsInInitializer(const Constant *Init, unsigned Size);
00622     unsigned GetSizeOfGlobalsInBytes(MachineFunction &MF);
00623   };
00624 }
00625 
00626 void *JITEmitter::getPointerToGlobal(GlobalValue *V, void *Reference,
00627                                      bool DoesntNeedStub) {
00628   if (GlobalVariable *GV = dyn_cast<GlobalVariable>(V))
00629     return TheJIT->getOrEmitGlobalVariable(GV);
00630 
00631   if (GlobalAlias *GA = dyn_cast<GlobalAlias>(V))
00632     return TheJIT->getPointerToGlobal(GA->resolveAliasedGlobal(false));
00633 
00634   // If we have already compiled the function, return a pointer to its body.
00635   Function *F = cast<Function>(V);
00636   void *ResultPtr;
00637   if (!DoesntNeedStub && !TheJIT->isLazyCompilationDisabled()) {
00638     // Return the function stub if it's already created.
00639     ResultPtr = Resolver.getFunctionStubIfAvailable(F);
00640     if (ResultPtr)
00641       AddStubToCurrentFunction(ResultPtr);
00642   } else {
00643     ResultPtr = TheJIT->getPointerToGlobalIfAvailable(F);
00644   }
00645   if (ResultPtr) return ResultPtr;
00646 
00647   // If this is an external function pointer, we can force the JIT to
00648   // 'compile' it, which really just adds it to the map.  In dlsym mode, 
00649   // external functions are forced through a stub, regardless of reloc type.
00650   if (F->isDeclaration() && !F->hasNotBeenReadFromBitcode() &&
00651       DoesntNeedStub && !TheJIT->areDlsymStubsEnabled())
00652     return TheJIT->getPointerToFunction(F);
00653 
00654   // Okay, the function has not been compiled yet, if the target callback
00655   // mechanism is capable of rewriting the instruction directly, prefer to do
00656   // that instead of emitting a stub.  This uses the lazy resolver, so is not
00657   // legal if lazy compilation is disabled.
00658   if (DoesntNeedStub && !TheJIT->isLazyCompilationDisabled())
00659     return Resolver.AddCallbackAtLocation(F, Reference);
00660 
00661   // Otherwise, we have to emit a stub.
00662   void *StubAddr = Resolver.getFunctionStub(F);
00663 
00664   // Add the stub to the current function's list of referenced stubs, so we can
00665   // deallocate them if the current function is ever freed.  It's possible to
00666   // return null from getFunctionStub in the case of a weak extern that fails
00667   // to resolve.
00668   if (StubAddr)
00669     AddStubToCurrentFunction(StubAddr);
00670 
00671   return StubAddr;
00672 }
00673 
00674 void *JITEmitter::getPointerToGVIndirectSym(GlobalValue *V, void *Reference,
00675                                             bool NoNeedStub) {
00676   // Make sure GV is emitted first, and create a stub containing the fully
00677   // resolved address.
00678   void *GVAddress = getPointerToGlobal(V, Reference, true);
00679   void *StubAddr = Resolver.getGlobalValueIndirectSym(V, GVAddress);
00680   
00681   // Add the stub to the current function's list of referenced stubs, so we can
00682   // deallocate them if the current function is ever freed.
00683   AddStubToCurrentFunction(StubAddr);
00684   
00685   return StubAddr;
00686 }
00687 
00688 void JITEmitter::AddStubToCurrentFunction(void *StubAddr) {
00689   if (!TheJIT->areDlsymStubsEnabled())
00690     return;
00691   
00692   assert(CurFn && "Stub added to current function, but current function is 0!");
00693   
00694   SmallVectorImpl<void*> &StubsUsed = CurFnStubUses[CurFn];
00695   StubsUsed.push_back(StubAddr);
00696 
00697   SmallPtrSet<const Function *, 1> &FnRefs = StubFnRefs[StubAddr];
00698   FnRefs.insert(CurFn);
00699 }
00700 
00701 void JITEmitter::processDebugLoc(DebugLoc DL) {
00702   if (!DL.isUnknown()) {
00703     DebugLocTuple CurDLT = EmissionDetails.MF->getDebugLocTuple(DL);
00704 
00705     if (CurDLT.CompileUnit != 0 && PrevDLT != CurDLT) {
00706       JITEvent_EmittedFunctionDetails::LineStart NextLine;
00707       NextLine.Address = getCurrentPCValue();
00708       NextLine.Loc = DL;
00709       EmissionDetails.LineStarts.push_back(NextLine);
00710     }
00711 
00712     PrevDLT = CurDLT;
00713   }
00714 }
00715 
00716 static unsigned GetConstantPoolSizeInBytes(MachineConstantPool *MCP,
00717                                            const TargetData *TD) {
00718   const std::vector<MachineConstantPoolEntry> &Constants = MCP->getConstants();
00719   if (Constants.empty()) return 0;
00720 
00721   unsigned Size = 0;
00722   for (unsigned i = 0, e = Constants.size(); i != e; ++i) {
00723     MachineConstantPoolEntry CPE = Constants[i];
00724     unsigned AlignMask = CPE.getAlignment() - 1;
00725     Size = (Size + AlignMask) & ~AlignMask;
00726     const Type *Ty = CPE.getType();
00727     Size += TD->getTypeAllocSize(Ty);
00728   }
00729   return Size;
00730 }
00731 
00732 static unsigned GetJumpTableSizeInBytes(MachineJumpTableInfo *MJTI) {
00733   const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
00734   if (JT.empty()) return 0;
00735   
00736   unsigned NumEntries = 0;
00737   for (unsigned i = 0, e = JT.size(); i != e; ++i)
00738     NumEntries += JT[i].MBBs.size();
00739 
00740   unsigned EntrySize = MJTI->getEntrySize();
00741 
00742   return NumEntries * EntrySize;
00743 }
00744 
00745 static uintptr_t RoundUpToAlign(uintptr_t Size, unsigned Alignment) {
00746   if (Alignment == 0) Alignment = 1;
00747   // Since we do not know where the buffer will be allocated, be pessimistic. 
00748   return Size + Alignment;
00749 }
00750 
00751 /// addSizeOfGlobal - add the size of the global (plus any alignment padding)
00752 /// into the running total Size.
00753 
00754 unsigned JITEmitter::addSizeOfGlobal(const GlobalVariable *GV, unsigned Size) {
00755   const Type *ElTy = GV->getType()->getElementType();
00756   size_t GVSize = (size_t)TheJIT->getTargetData()->getTypeAllocSize(ElTy);
00757   size_t GVAlign = 
00758       (size_t)TheJIT->getTargetData()->getPreferredAlignment(GV);
00759   DOUT << "JIT: Adding in size " << GVSize << " alignment " << GVAlign;
00760   DEBUG(GV->dump());
00761   // Assume code section ends with worst possible alignment, so first
00762   // variable needs maximal padding.
00763   if (Size==0)
00764     Size = 1;
00765   Size = ((Size+GVAlign-1)/GVAlign)*GVAlign;
00766   Size += GVSize;
00767   return Size;
00768 }
00769 
00770 /// addSizeOfGlobalsInConstantVal - find any globals that we haven't seen yet
00771 /// but are referenced from the constant; put them in GVSet and add their
00772 /// size into the running total Size.
00773 
00774 unsigned JITEmitter::addSizeOfGlobalsInConstantVal(const Constant *C, 
00775                                               unsigned Size) {
00776   // If its undefined, return the garbage.
00777   if (isa<UndefValue>(C))
00778     return Size;
00779 
00780   // If the value is a ConstantExpr
00781   if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
00782     Constant *Op0 = CE->getOperand(0);
00783     switch (CE->getOpcode()) {
00784     case Instruction::GetElementPtr:
00785     case Instruction::Trunc:
00786     case Instruction::ZExt:
00787     case Instruction::SExt:
00788     case Instruction::FPTrunc:
00789     case Instruction::FPExt:
00790     case Instruction::UIToFP:
00791     case Instruction::SIToFP:
00792     case Instruction::FPToUI:
00793     case Instruction::FPToSI:
00794     case Instruction::PtrToInt:
00795     case Instruction::IntToPtr:
00796     case Instruction::BitCast: {
00797       Size = addSizeOfGlobalsInConstantVal(Op0, Size);
00798       break;
00799     }
00800     case Instruction::Add:
00801     case Instruction::FAdd:
00802     case Instruction::Sub:
00803     case Instruction::FSub:
00804     case Instruction::Mul:
00805     case Instruction::FMul:
00806     case Instruction::UDiv:
00807     case Instruction::SDiv:
00808     case Instruction::URem:
00809     case Instruction::SRem:
00810     case Instruction::And:
00811     case Instruction::Or:
00812     case Instruction::Xor: {
00813       Size = addSizeOfGlobalsInConstantVal(Op0, Size);
00814       Size = addSizeOfGlobalsInConstantVal(CE->getOperand(1), Size);
00815       break;
00816     }
00817     default: {
00818        std::string msg;
00819        raw_string_ostream Msg(msg);
00820        Msg << "ConstantExpr not handled: " << *CE;
00821        llvm_report_error(Msg.str());
00822     }
00823     }
00824   }
00825 
00826   if (C->getType()->getTypeID() == Type::PointerTyID)
00827     if (const GlobalVariable* GV = dyn_cast<GlobalVariable>(C))
00828       if (GVSet.insert(GV))
00829         Size = addSizeOfGlobal(GV, Size);
00830 
00831   return Size;
00832 }
00833 
00834 /// addSizeOfGLobalsInInitializer - handle any globals that we haven't seen yet
00835 /// but are referenced from the given initializer.
00836 
00837 unsigned JITEmitter::addSizeOfGlobalsInInitializer(const Constant *Init, 
00838                                               unsigned Size) {
00839   if (!isa<UndefValue>(Init) &&
00840       !isa<ConstantVector>(Init) &&
00841       !isa<ConstantAggregateZero>(Init) &&
00842       !isa<ConstantArray>(Init) &&
00843       !isa<ConstantStruct>(Init) &&
00844       Init->getType()->isFirstClassType())
00845     Size = addSizeOfGlobalsInConstantVal(Init, Size);
00846   return Size;
00847 }
00848 
00849 /// GetSizeOfGlobalsInBytes - walk the code for the function, looking for
00850 /// globals; then walk the initializers of those globals looking for more.
00851 /// If their size has not been considered yet, add it into the running total
00852 /// Size.
00853 
00854 unsigned JITEmitter::GetSizeOfGlobalsInBytes(MachineFunction &MF) {
00855   unsigned Size = 0;
00856   GVSet.clear();
00857 
00858   for (MachineFunction::iterator MBB = MF.begin(), E = MF.end(); 
00859        MBB != E; ++MBB) {
00860     for (MachineBasicBlock::const_iterator I = MBB->begin(), E = MBB->end();
00861          I != E; ++I) {
00862       const TargetInstrDesc &Desc = I->getDesc();
00863       const MachineInstr &MI = *I;
00864       unsigned NumOps = Desc.getNumOperands();
00865       for (unsigned CurOp = 0; CurOp < NumOps; CurOp++) {
00866         const MachineOperand &MO = MI.getOperand(CurOp);
00867         if (MO.isGlobal()) {
00868           GlobalValue* V = MO.getGlobal();
00869           const GlobalVariable *GV = dyn_cast<const GlobalVariable>(V);
00870           if (!GV)
00871             continue;
00872           // If seen in previous function, it will have an entry here.
00873           if (TheJIT->getPointerToGlobalIfAvailable(GV))
00874             continue;
00875           // If seen earlier in this function, it will have an entry here.
00876           // FIXME: it should be possible to combine these tables, by
00877           // assuming the addresses of the new globals in this module
00878           // start at 0 (or something) and adjusting them after codegen
00879           // complete.  Another possibility is to grab a marker bit in GV.
00880           if (GVSet.insert(GV))
00881             // A variable as yet unseen.  Add in its size.
00882             Size = addSizeOfGlobal(GV, Size);
00883         }
00884       }
00885     }
00886   }
00887   DOUT << "JIT: About to look through initializers\n";
00888   // Look for more globals that are referenced only from initializers.
00889   // GVSet.end is computed each time because the set can grow as we go.
00890   for (SmallPtrSet<const GlobalVariable *, 8>::iterator I = GVSet.begin(); 
00891        I != GVSet.end(); I++) {
00892     const GlobalVariable* GV = *I;
00893     if (GV->hasInitializer())
00894       Size = addSizeOfGlobalsInInitializer(GV->getInitializer(), Size);
00895   }
00896 
00897   return Size;
00898 }
00899 
00900 void JITEmitter::startFunction(MachineFunction &F) {
00901   DEBUG(errs() << "JIT: Starting CodeGen of Function "
00902         << F.getFunction()->getName() << "\n");
00903 
00904   uintptr_t ActualSize = 0;
00905   // Set the memory writable, if it's not already
00906   MemMgr->setMemoryWritable();
00907   if (MemMgr->NeedsExactSize()) {
00908     DOUT << "JIT: ExactSize\n";
00909     const TargetInstrInfo* TII = F.getTarget().getInstrInfo();
00910     MachineJumpTableInfo *MJTI = F.getJumpTableInfo();
00911     MachineConstantPool *MCP = F.getConstantPool();
00912     
00913     // Ensure the constant pool/jump table info is at least 4-byte aligned.
00914     ActualSize = RoundUpToAlign(ActualSize, 16);
00915     
00916     // Add the alignment of the constant pool
00917     ActualSize = RoundUpToAlign(ActualSize, MCP->getConstantPoolAlignment());
00918 
00919     // Add the constant pool size
00920     ActualSize += GetConstantPoolSizeInBytes(MCP, TheJIT->getTargetData());
00921 
00922     // Add the aligment of the jump table info
00923     ActualSize = RoundUpToAlign(ActualSize, MJTI->getAlignment());
00924 
00925     // Add the jump table size
00926     ActualSize += GetJumpTableSizeInBytes(MJTI);
00927     
00928     // Add the alignment for the function
00929     ActualSize = RoundUpToAlign(ActualSize,
00930                                 std::max(F.getFunction()->getAlignment(), 8U));
00931 
00932     // Add the function size
00933     ActualSize += TII->GetFunctionSizeInBytes(F);
00934 
00935     DOUT << "JIT: ActualSize before globals " << ActualSize << "\n";
00936     // Add the size of the globals that will be allocated after this function.
00937     // These are all the ones referenced from this function that were not
00938     // previously allocated.
00939     ActualSize += GetSizeOfGlobalsInBytes(F);
00940     DOUT << "JIT: ActualSize after globals " << ActualSize << "\n";
00941   } else if (SizeEstimate > 0) {
00942     // SizeEstimate will be non-zero on reallocation attempts.
00943     ActualSize = SizeEstimate;
00944   }
00945 
00946   BufferBegin = CurBufferPtr = MemMgr->startFunctionBody(F.getFunction(),
00947                                                          ActualSize);
00948   BufferEnd = BufferBegin+ActualSize;
00949   
00950   // Ensure the constant pool/jump table info is at least 4-byte aligned.
00951   emitAlignment(16);
00952 
00953   emitConstantPool(F.getConstantPool());
00954   initJumpTableInfo(F.getJumpTableInfo());
00955 
00956   // About to start emitting the machine code for the function.
00957   emitAlignment(std::max(F.getFunction()->getAlignment(), 8U));
00958   TheJIT->updateGlobalMapping(F.getFunction(), CurBufferPtr);
00959 
00960   MBBLocations.clear();
00961 
00962   EmissionDetails.MF = &F;
00963   EmissionDetails.LineStarts.clear();
00964 }
00965 
00966 bool JITEmitter::finishFunction(MachineFunction &F) {
00967   if (CurBufferPtr == BufferEnd) {
00968     // We must call endFunctionBody before retrying, because
00969     // deallocateMemForFunction requires it.
00970     MemMgr->endFunctionBody(F.getFunction(), BufferBegin, CurBufferPtr);
00971     retryWithMoreMemory(F);
00972     return true;
00973   }
00974 
00975   emitJumpTableInfo(F.getJumpTableInfo());
00976 
00977   // FnStart is the start of the text, not the start of the constant pool and
00978   // other per-function data.
00979   uint8_t *FnStart =
00980     (uint8_t *)TheJIT->getPointerToGlobalIfAvailable(F.getFunction());
00981 
00982   // FnEnd is the end of the function's machine code.
00983   uint8_t *FnEnd = CurBufferPtr;
00984 
00985   if (!Relocations.empty()) {
00986     CurFn = F.getFunction();
00987     NumRelos += Relocations.size();
00988 
00989     // Resolve the relocations to concrete pointers.
00990     for (unsigned i = 0, e = Relocations.size(); i != e; ++i) {
00991       MachineRelocation &MR = Relocations[i];
00992       void *ResultPtr = 0;
00993       if (!MR.letTargetResolve()) {
00994         if (MR.isExternalSymbol()) {
00995           ResultPtr = TheJIT->getPointerToNamedFunction(MR.getExternalSymbol(),
00996                                                         false);
00997           DOUT << "JIT: Map \'" << MR.getExternalSymbol() << "\' to ["
00998                << ResultPtr << "]\n";  
00999 
01000           // If the target REALLY wants a stub for this function, emit it now.
01001           if (!MR.doesntNeedStub()) {
01002             if (!TheJIT->areDlsymStubsEnabled()) {
01003               ResultPtr = Resolver.getExternalFunctionStub(ResultPtr);
01004             } else {
01005               void *&Stub = ExtFnStubs[MR.getExternalSymbol()];
01006               if (!Stub) {
01007                 Stub = Resolver.getExternalFunctionStub((void *)&Stub);
01008                 AddStubToCurrentFunction(Stub);
01009               }
01010               ResultPtr = Stub;
01011             }
01012           }
01013         } else if (MR.isGlobalValue()) {
01014           ResultPtr = getPointerToGlobal(MR.getGlobalValue(),
01015                                          BufferBegin+MR.getMachineCodeOffset(),
01016                                          MR.doesntNeedStub());
01017         } else if (MR.isIndirectSymbol()) {
01018           ResultPtr = getPointerToGVIndirectSym(MR.getGlobalValue(),
01019                                           BufferBegin+MR.getMachineCodeOffset(),
01020                                           MR.doesntNeedStub());
01021         } else if (MR.isBasicBlock()) {
01022           ResultPtr = (void*)getMachineBasicBlockAddress(MR.getBasicBlock());
01023         } else if (MR.isConstantPoolIndex()) {
01024           ResultPtr = (void*)getConstantPoolEntryAddress(MR.getConstantPoolIndex());
01025         } else {
01026           assert(MR.isJumpTableIndex());
01027           ResultPtr=(void*)getJumpTableEntryAddress(MR.getJumpTableIndex());
01028         }
01029 
01030         MR.setResultPointer(ResultPtr);
01031       }
01032 
01033       // if we are managing the GOT and the relocation wants an index,
01034       // give it one
01035       if (MR.isGOTRelative() && MemMgr->isManagingGOT()) {
01036         unsigned idx = Resolver.getGOTIndexForAddr(ResultPtr);
01037         MR.setGOTIndex(idx);
01038         if (((void**)MemMgr->getGOTBase())[idx] != ResultPtr) {
01039           DOUT << "JIT: GOT was out of date for " << ResultPtr
01040                << " pointing at " << ((void**)MemMgr->getGOTBase())[idx]
01041                << "\n";
01042           ((void**)MemMgr->getGOTBase())[idx] = ResultPtr;
01043         }
01044       }
01045     }
01046 
01047     CurFn = 0;
01048     TheJIT->getJITInfo().relocate(BufferBegin, &Relocations[0],
01049                                   Relocations.size(), MemMgr->getGOTBase());
01050   }
01051 
01052   // Update the GOT entry for F to point to the new code.
01053   if (MemMgr->isManagingGOT()) {
01054     unsigned idx = Resolver.getGOTIndexForAddr((void*)BufferBegin);
01055     if (((void**)MemMgr->getGOTBase())[idx] != (void*)BufferBegin) {
01056       DOUT << "JIT: GOT was out of date for " << (void*)BufferBegin
01057            << " pointing at " << ((void**)MemMgr->getGOTBase())[idx] << "\n";
01058       ((void**)MemMgr->getGOTBase())[idx] = (void*)BufferBegin;
01059     }
01060   }
01061 
01062   // CurBufferPtr may have moved beyond FnEnd, due to memory allocation for
01063   // global variables that were referenced in the relocations.
01064   MemMgr->endFunctionBody(F.getFunction(), BufferBegin, CurBufferPtr);
01065 
01066   if (CurBufferPtr == BufferEnd) {
01067     retryWithMoreMemory(F);
01068     return true;
01069   } else {
01070     // Now that we've succeeded in emitting the function, reset the
01071     // SizeEstimate back down to zero.
01072     SizeEstimate = 0;
01073   }
01074 
01075   BufferBegin = CurBufferPtr = 0;
01076   NumBytes += FnEnd-FnStart;
01077 
01078   // Invalidate the icache if necessary.
01079   sys::Memory::InvalidateInstructionCache(FnStart, FnEnd-FnStart);
01080 
01081   TheJIT->NotifyFunctionEmitted(*F.getFunction(), FnStart, FnEnd-FnStart,
01082                                 EmissionDetails);
01083 
01084   DEBUG(errs() << "JIT: Finished CodeGen of [" << (void*)FnStart
01085         << "] Function: " << F.getFunction()->getName()
01086         << ": " << (FnEnd-FnStart) << " bytes of text, "
01087         << Relocations.size() << " relocations\n");
01088 
01089   Relocations.clear();
01090   ConstPoolAddresses.clear();
01091 
01092   // Mark code region readable and executable if it's not so already.
01093   MemMgr->setMemoryExecutable();
01094 
01095 #ifndef NDEBUG
01096   {
01097     if (sys::hasDisassembler()) {
01098       DOUT << "JIT: Disassembled code:\n";
01099       DOUT << sys::disassembleBuffer(FnStart, FnEnd-FnStart, (uintptr_t)FnStart);
01100     } else {
01101       DOUT << "JIT: Binary code:\n";
01102       DOUT << std::hex;
01103       uint8_t* q = FnStart;
01104       for (int i = 0; q < FnEnd; q += 4, ++i) {
01105         if (i == 4)
01106           i = 0;
01107         if (i == 0)
01108           DOUT << "JIT: " << std::setw(8) << std::setfill('0')
01109                << (long)(q - FnStart) << ": ";
01110         bool Done = false;
01111         for (int j = 3; j >= 0; --j) {
01112           if (q + j >= FnEnd)
01113             Done = true;
01114           else
01115             DOUT << std::setw(2) << std::setfill('0') << (unsigned short)q[j];
01116         }
01117         if (Done)
01118           break;
01119         DOUT << ' ';
01120         if (i == 3)
01121           DOUT << '\n';
01122       }
01123       DOUT << std::dec;
01124       DOUT<< '\n';
01125     }
01126   }
01127 #endif
01128   if (DwarfExceptionHandling) {
01129     uintptr_t ActualSize = 0;
01130     SavedBufferBegin = BufferBegin;
01131     SavedBufferEnd = BufferEnd;
01132     SavedCurBufferPtr = CurBufferPtr;
01133     
01134     if (MemMgr->NeedsExactSize()) {
01135       ActualSize = DE->GetDwarfTableSizeInBytes(F, *this, FnStart, FnEnd);
01136     }
01137 
01138     BufferBegin = CurBufferPtr = MemMgr->startExceptionTable(F.getFunction(),
01139                                                              ActualSize);
01140     BufferEnd = BufferBegin+ActualSize;
01141     uint8_t* FrameRegister = DE->EmitDwarfTable(F, *this, FnStart, FnEnd);
01142     MemMgr->endExceptionTable(F.getFunction(), BufferBegin, CurBufferPtr,
01143                               FrameRegister);
01144     BufferBegin = SavedBufferBegin;
01145     BufferEnd = SavedBufferEnd;
01146     CurBufferPtr = SavedCurBufferPtr;
01147 
01148     TheJIT->RegisterTable(FrameRegister);
01149   }
01150 
01151   if (MMI)
01152     MMI->EndFunction();
01153  
01154   return false;
01155 }
01156 
01157 void JITEmitter::retryWithMoreMemory(MachineFunction &F) {
01158   DOUT << "JIT: Ran out of space for native code.  Reattempting.\n";
01159   Relocations.clear();  // Clear the old relocations or we'll reapply them.
01160   ConstPoolAddresses.clear();
01161   ++NumRetries;
01162   deallocateMemForFunction(F.getFunction());
01163   // Try again with at least twice as much free space.
01164   SizeEstimate = (uintptr_t)(2 * (BufferEnd - BufferBegin));
01165 }
01166 
01167 /// deallocateMemForFunction - Deallocate all memory for the specified
01168 /// function body.  Also drop any references the function has to stubs.
01169 void JITEmitter::deallocateMemForFunction(const Function *F) {
01170   MemMgr->deallocateMemForFunction(F);
01171 
01172   // If the function did not reference any stubs, return.
01173   if (CurFnStubUses.find(F) == CurFnStubUses.end())
01174     return;
01175   
01176   // For each referenced stub, erase the reference to this function, and then
01177   // erase the list of referenced stubs.
01178   SmallVectorImpl<void *> &StubList = CurFnStubUses[F];
01179   for (unsigned i = 0, e = StubList.size(); i != e; ++i) {
01180     void *Stub = StubList[i];
01181     
01182     // If we already invalidated this stub for this function, continue.
01183     if (StubFnRefs.count(Stub) == 0)
01184       continue;
01185       
01186     SmallPtrSet<const Function *, 1> &FnRefs = StubFnRefs[Stub];
01187     FnRefs.erase(F);
01188     
01189     // If this function was the last reference to the stub, invalidate the stub
01190     // in the JITResolver.  Were there a memory manager deallocateStub routine,
01191     // we could call that at this point too.
01192     if (FnRefs.empty()) {
01193       DOUT << "\nJIT: Invalidated Stub at [" << Stub << "]\n";
01194       StubFnRefs.erase(Stub);
01195 
01196       // Invalidate the stub.  If it is a GV stub, update the JIT's global
01197       // mapping for that GV to zero, otherwise, search the string map of
01198       // external function names to stubs and remove the entry for this stub.
01199       GlobalValue *GV = Resolver.invalidateStub(Stub);
01200       if (GV) {
01201         TheJIT->updateGlobalMapping(GV, 0);
01202       } else {
01203         for (StringMapIterator<void*> i = ExtFnStubs.begin(),
01204              e = ExtFnStubs.end(); i != e; ++i) {
01205           if (i->second == Stub) {
01206             ExtFnStubs.erase(i);
01207             break;
01208           }
01209         }
01210       }
01211     }
01212   }
01213   CurFnStubUses.erase(F);
01214 }
01215 
01216 
01217 void* JITEmitter::allocateSpace(uintptr_t Size, unsigned Alignment) {
01218   if (BufferBegin)
01219     return JITCodeEmitter::allocateSpace(Size, Alignment);
01220 
01221   // create a new memory block if there is no active one.
01222   // care must be taken so that BufferBegin is invalidated when a
01223   // block is trimmed
01224   BufferBegin = CurBufferPtr = MemMgr->allocateSpace(Size, Alignment);
01225   BufferEnd = BufferBegin+Size;
01226   return CurBufferPtr;
01227 }
01228 
01229 void* JITEmitter::allocateGlobal(uintptr_t Size, unsigned Alignment) {
01230   // Delegate this call through the memory manager.
01231   return MemMgr->allocateGlobal(Size, Alignment);
01232 }
01233 
01234 void JITEmitter::emitConstantPool(MachineConstantPool *MCP) {
01235   if (TheJIT->getJITInfo().hasCustomConstantPool())
01236     return;
01237 
01238   const std::vector<MachineConstantPoolEntry> &Constants = MCP->getConstants();
01239   if (Constants.empty()) return;
01240 
01241   unsigned Size = GetConstantPoolSizeInBytes(MCP, TheJIT->getTargetData());
01242   unsigned Align = MCP->getConstantPoolAlignment();
01243   ConstantPoolBase = allocateSpace(Size, Align);
01244   ConstantPool = MCP;
01245 
01246   if (ConstantPoolBase == 0) return;  // Buffer overflow.
01247 
01248   DOUT << "JIT: Emitted constant pool at [" << ConstantPoolBase
01249        << "] (size: " << Size << ", alignment: " << Align << ")\n";
01250 
01251   // Initialize the memory for all of the constant pool entries.
01252   unsigned Offset = 0;
01253   for (unsigned i = 0, e = Constants.size(); i != e; ++i) {
01254     MachineConstantPoolEntry CPE = Constants[i];
01255     unsigned AlignMask = CPE.getAlignment() - 1;
01256     Offset = (Offset + AlignMask) & ~AlignMask;
01257 
01258     uintptr_t CAddr = (uintptr_t)ConstantPoolBase + Offset;
01259     ConstPoolAddresses.push_back(CAddr);
01260     if (CPE.isMachineConstantPoolEntry()) {
01261       // FIXME: add support to lower machine constant pool values into bytes!
01262       llvm_report_error("Initialize memory with machine specific constant pool"
01263                         "entry has not been implemented!");
01264     }
01265     TheJIT->InitializeMemory(CPE.Val.ConstVal, (void*)CAddr);
01266     DOUT << "JIT:   CP" << i << " at [0x"
01267          << std::hex << CAddr << std::dec << "]\n";
01268 
01269     const Type *Ty = CPE.Val.ConstVal->getType();
01270     Offset += TheJIT->getTargetData()->getTypeAllocSize(Ty);
01271   }
01272 }
01273 
01274 void JITEmitter::initJumpTableInfo(MachineJumpTableInfo *MJTI) {
01275   if (TheJIT->getJITInfo().hasCustomJumpTables())
01276     return;
01277 
01278   const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
01279   if (JT.empty()) return;
01280   
01281   unsigned NumEntries = 0;
01282   for (unsigned i = 0, e = JT.size(); i != e; ++i)
01283     NumEntries += JT[i].MBBs.size();
01284 
01285   unsigned EntrySize = MJTI->getEntrySize();
01286 
01287   // Just allocate space for all the jump tables now.  We will fix up the actual
01288   // MBB entries in the tables after we emit the code for each block, since then
01289   // we will know the final locations of the MBBs in memory.
01290   JumpTable = MJTI;
01291   JumpTableBase = allocateSpace(NumEntries * EntrySize, MJTI->getAlignment());
01292 }
01293 
01294 void JITEmitter::emitJumpTableInfo(MachineJumpTableInfo *MJTI) {
01295   if (TheJIT->getJITInfo().hasCustomJumpTables())
01296     return;
01297 
01298   const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
01299   if (JT.empty() || JumpTableBase == 0) return;
01300   
01301   if (TargetMachine::getRelocationModel() == Reloc::PIC_) {
01302     assert(MJTI->getEntrySize() == 4 && "Cross JIT'ing?");
01303     // For each jump table, place the offset from the beginning of the table
01304     // to the target address.
01305     int *SlotPtr = (int*)JumpTableBase;
01306 
01307     for (unsigned i = 0, e = JT.size(); i != e; ++i) {
01308       const std::vector<MachineBasicBlock*> &MBBs = JT[i].MBBs;
01309       // Store the offset of the basic block for this jump table slot in the
01310       // memory we allocated for the jump table in 'initJumpTableInfo'
01311       uintptr_t Base = (uintptr_t)SlotPtr;
01312       for (unsigned mi = 0, me = MBBs.size(); mi != me; ++mi) {
01313         uintptr_t MBBAddr = getMachineBasicBlockAddress(MBBs[mi]);
01314         *SlotPtr++ = TheJIT->getJITInfo().getPICJumpTableEntry(MBBAddr, Base);
01315       }
01316     }
01317   } else {
01318     assert(MJTI->getEntrySize() == sizeof(void*) && "Cross JIT'ing?");
01319     
01320     // For each jump table, map each target in the jump table to the address of 
01321     // an emitted MachineBasicBlock.
01322     intptr_t *SlotPtr = (intptr_t*)JumpTableBase;
01323 
01324     for (unsigned i = 0, e = JT.size(); i != e; ++i) {
01325       const std::vector<MachineBasicBlock*> &MBBs = JT[i].MBBs;
01326       // Store the address of the basic block for this jump table slot in the
01327       // memory we allocated for the jump table in 'initJumpTableInfo'
01328       for (unsigned mi = 0, me = MBBs.size(); mi != me; ++mi)
01329         *SlotPtr++ = getMachineBasicBlockAddress(MBBs[mi]);
01330     }
01331   }
01332 }
01333 
01334 void JITEmitter::startGVStub(const GlobalValue* GV, unsigned StubSize,
01335                              unsigned Alignment) {
01336   SavedBufferBegin = BufferBegin;
01337   SavedBufferEnd = BufferEnd;
01338   SavedCurBufferPtr = CurBufferPtr;
01339   
01340   BufferBegin = CurBufferPtr = MemMgr->allocateStub(GV, StubSize, Alignment);
01341   BufferEnd = BufferBegin+StubSize+1;
01342 }
01343 
01344 void JITEmitter::startGVStub(const GlobalValue* GV, void *Buffer,
01345                              unsigned StubSize) {
01346   SavedBufferBegin = BufferBegin;
01347   SavedBufferEnd = BufferEnd;
01348   SavedCurBufferPtr = CurBufferPtr;
01349   
01350   BufferBegin = CurBufferPtr = (uint8_t *)Buffer;
01351   BufferEnd = BufferBegin+StubSize+1;
01352 }
01353 
01354 void *JITEmitter::finishGVStub(const GlobalValue* GV) {
01355   NumBytes += getCurrentPCOffset();
01356   std::swap(SavedBufferBegin, BufferBegin);
01357   BufferEnd = SavedBufferEnd;
01358   CurBufferPtr = SavedCurBufferPtr;
01359   return SavedBufferBegin;
01360 }
01361 
01362 // getConstantPoolEntryAddress - Return the address of the 'ConstantNum' entry
01363 // in the constant pool that was last emitted with the 'emitConstantPool'
01364 // method.
01365 //
01366 uintptr_t JITEmitter::getConstantPoolEntryAddress(unsigned ConstantNum) const {
01367   assert(ConstantNum < ConstantPool->getConstants().size() &&
01368          "Invalid ConstantPoolIndex!");
01369   return ConstPoolAddresses[ConstantNum];
01370 }
01371 
01372 // getJumpTableEntryAddress - Return the address of the JumpTable with index
01373 // 'Index' in the jumpp table that was last initialized with 'initJumpTableInfo'
01374 //
01375 uintptr_t JITEmitter::getJumpTableEntryAddress(unsigned Index) const {
01376   const std::vector<MachineJumpTableEntry> &JT = JumpTable->getJumpTables();
01377   assert(Index < JT.size() && "Invalid jump table index!");
01378   
01379   unsigned Offset = 0;
01380   unsigned EntrySize = JumpTable->getEntrySize();
01381   
01382   for (unsigned i = 0; i < Index; ++i)
01383     Offset += JT[i].MBBs.size();
01384   
01385    Offset *= EntrySize;
01386   
01387   return (uintptr_t)((char *)JumpTableBase + Offset);
01388 }
01389 
01390 //===----------------------------------------------------------------------===//
01391 //  Public interface to this file
01392 //===----------------------------------------------------------------------===//
01393 
01394 JITCodeEmitter *JIT::createEmitter(JIT &jit, JITMemoryManager *JMM) {
01395   return new JITEmitter(jit, JMM);
01396 }
01397 
01398 // getPointerToNamedFunction - This function is used as a global wrapper to
01399 // JIT::getPointerToNamedFunction for the purpose of resolving symbols when
01400 // bugpoint is debugging the JIT. In that scenario, we are loading an .so and
01401 // need to resolve function(s) that are being mis-codegenerated, so we need to
01402 // resolve their addresses at runtime, and this is the way to do it.
01403 extern "C" {
01404   void *getPointerToNamedFunction(const char *Name) {
01405     if (Function *F = TheJIT->FindFunctionNamed(Name))
01406       return TheJIT->getPointerToFunction(F);
01407     return TheJIT->getPointerToNamedFunction(Name);
01408   }
01409 }
01410 
01411 // getPointerToFunctionOrStub - If the specified function has been
01412 // code-gen'd, return a pointer to the function.  If not, compile it, or use
01413 // a stub to implement lazy compilation if available.
01414 //
01415 void *JIT::getPointerToFunctionOrStub(Function *F) {
01416   // If we have already code generated the function, just return the address.
01417   if (void *Addr = getPointerToGlobalIfAvailable(F))
01418     return Addr;
01419   
01420   // Get a stub if the target supports it.
01421   assert(isa<JITEmitter>(JCE) && "Unexpected MCE?");
01422   JITEmitter *JE = cast<JITEmitter>(getCodeEmitter());
01423   return JE->getJITResolver().getFunctionStub(F);
01424 }
01425 
01426 void JIT::updateFunctionStub(Function *F) {
01427   // Get the empty stub we generated earlier.
01428   assert(isa<JITEmitter>(JCE) && "Unexpected MCE?");
01429   JITEmitter *JE = cast<JITEmitter>(getCodeEmitter());
01430   void *Stub = JE->getJITResolver().getFunctionStub(F);
01431 
01432   // Tell the target jit info to rewrite the stub at the specified address,
01433   // rather than creating a new one.
01434   void *Addr = getPointerToGlobalIfAvailable(F);
01435   getJITInfo().emitFunctionStubAtAddr(F, Addr, Stub, *getCodeEmitter());
01436 }
01437 
01438 /// updateDlsymStubTable - Emit the data necessary to relocate the stubs
01439 /// that were emitted during code generation.
01440 ///
01441 void JIT::updateDlsymStubTable() {
01442   assert(isa<JITEmitter>(JCE) && "Unexpected MCE?");
01443   JITEmitter *JE = cast<JITEmitter>(getCodeEmitter());
01444   
01445   SmallVector<GlobalValue*, 8> GVs;
01446   SmallVector<void*, 8> Ptrs;
01447   const StringMap<void *> &ExtFns = JE->getExternalFnStubs();
01448 
01449   JE->getJITResolver().getRelocatableGVs(GVs, Ptrs);
01450 
01451   unsigned nStubs = GVs.size() + ExtFns.size();
01452   
01453   // If there are no relocatable stubs, return.
01454   if (nStubs == 0)
01455     return;
01456 
01457   // If there are no new relocatable stubs, return.
01458   void *CurTable = JE->getMemMgr()->getDlsymTable();
01459   if (CurTable && (*(unsigned *)CurTable == nStubs))
01460     return;
01461   
01462   // Calculate the size of the stub info
01463   unsigned offset = 4 + 4 * nStubs + sizeof(intptr_t) * nStubs;
01464   
01465   SmallVector<unsigned, 8> Offsets;
01466   for (unsigned i = 0; i != GVs.size(); ++i) {
01467     Offsets.push_back(offset);
01468     offset += GVs[i]->getName().size() + 1;
01469   }
01470   for (StringMapConstIterator<void*> i = ExtFns.begin(), e = ExtFns.end(); 
01471        i != e; ++i) {
01472     Offsets.push_back(offset);
01473     offset += strlen(i->first()) + 1;
01474   }
01475   
01476   // Allocate space for the new "stub", which contains the dlsym table.
01477   JE->startGVStub(0, offset, 4);
01478   
01479   // Emit the number of records
01480   JE->emitInt32(nStubs);
01481   
01482   // Emit the string offsets
01483   for (unsigned i = 0; i != nStubs; ++i)
01484     JE->emitInt32(Offsets[i]);
01485   
01486   // Emit the pointers.  Verify that they are at least 2-byte aligned, and set
01487   // the low bit to 0 == GV, 1 == Function, so that the client code doing the
01488   // relocation can write the relocated pointer at the appropriate place in
01489   // the stub.
01490   for (unsigned i = 0; i != GVs.size(); ++i) {
01491     intptr_t Ptr = (intptr_t)Ptrs[i];
01492     assert((Ptr & 1) == 0 && "Stub pointers must be at least 2-byte aligned!");
01493     
01494     if (isa<Function>(GVs[i]))
01495       Ptr |= (intptr_t)1;
01496            
01497     if (sizeof(Ptr) == 8)
01498       JE->emitInt64(Ptr);
01499     else
01500       JE->emitInt32(Ptr);
01501   }
01502   for (StringMapConstIterator<void*> i = ExtFns.begin(), e = ExtFns.end(); 
01503        i != e; ++i) {
01504     intptr_t Ptr = (intptr_t)i->second | 1;
01505 
01506     if (sizeof(Ptr) == 8)
01507       JE->emitInt64(Ptr);
01508     else
01509       JE->emitInt32(Ptr);
01510   }
01511   
01512   // Emit the strings.
01513   for (unsigned i = 0; i != GVs.size(); ++i)
01514     JE->emitString(GVs[i]->getName());
01515   for (StringMapConstIterator<void*> i = ExtFns.begin(), e = ExtFns.end(); 
01516        i != e; ++i)
01517     JE->emitString(i->first());
01518   
01519   // Tell the JIT memory manager where it is.  The JIT Memory Manager will
01520   // deallocate space for the old one, if one existed.
01521   JE->getMemMgr()->SetDlsymTable(JE->finishGVStub(0));
01522 }
01523 
01524 /// freeMachineCodeForFunction - release machine code memory for given Function.
01525 ///
01526 void JIT::freeMachineCodeForFunction(Function *F) {
01527 
01528   // Delete translation for this from the ExecutionEngine, so it will get
01529   // retranslated next time it is used.
01530   void *OldPtr = updateGlobalMapping(F, 0);
01531 
01532   if (OldPtr)
01533     TheJIT->NotifyFreeingMachineCode(*F, OldPtr);
01534 
01535   // Free the actual memory for the function body and related stuff.
01536   assert(isa<JITEmitter>(JCE) && "Unexpected MCE?");
01537   cast<JITEmitter>(JCE)->deallocateMemForFunction(F);
01538 }