LLVM API Documentation
00001 //===-- JITEmitter.cpp - Write machine code to executable memory ----------===// 00002 // 00003 // The LLVM Compiler Infrastructure 00004 // 00005 // This file is distributed under the University of Illinois Open Source 00006 // License. See LICENSE.TXT for details. 00007 // 00008 //===----------------------------------------------------------------------===// 00009 // 00010 // This file defines a MachineCodeEmitter object that is used by the JIT to 00011 // write machine code to memory and remember where relocatable values are. 00012 // 00013 //===----------------------------------------------------------------------===// 00014 00015 #define DEBUG_TYPE "jit" 00016 #include "JIT.h" 00017 #include "JITDwarfEmitter.h" 00018 #include "llvm/Constants.h" 00019 #include "llvm/Module.h" 00020 #include "llvm/DerivedTypes.h" 00021 #include "llvm/CodeGen/JITCodeEmitter.h" 00022 #include "llvm/CodeGen/MachineFunction.h" 00023 #include "llvm/CodeGen/MachineConstantPool.h" 00024 #include "llvm/CodeGen/MachineJumpTableInfo.h" 00025 #include "llvm/CodeGen/MachineModuleInfo.h" 00026 #include "llvm/CodeGen/MachineRelocation.h" 00027 #include "llvm/ExecutionEngine/GenericValue.h" 00028 #include "llvm/ExecutionEngine/JITEventListener.h" 00029 #include "llvm/ExecutionEngine/JITMemoryManager.h" 00030 #include "llvm/CodeGen/MachineCodeInfo.h" 00031 #include "llvm/Target/TargetData.h" 00032 #include "llvm/Target/TargetJITInfo.h" 00033 #include "llvm/Target/TargetMachine.h" 00034 #include "llvm/Target/TargetOptions.h" 00035 #include "llvm/Support/Debug.h" 00036 #include "llvm/Support/ErrorHandling.h" 00037 #include "llvm/Support/MutexGuard.h" 00038 #include "llvm/Support/ValueHandle.h" 00039 #include "llvm/Support/raw_ostream.h" 00040 #include "llvm/System/Disassembler.h" 00041 #include "llvm/System/Memory.h" 00042 #include "llvm/Target/TargetInstrInfo.h" 00043 #include "llvm/ADT/SmallPtrSet.h" 00044 #include "llvm/ADT/SmallVector.h" 00045 #include "llvm/ADT/Statistic.h" 00046 #include <algorithm> 00047 #ifndef NDEBUG 00048 #include <iomanip> 00049 #endif 00050 using namespace llvm; 00051 00052 STATISTIC(NumBytes, "Number of bytes of machine code compiled"); 00053 STATISTIC(NumRelos, "Number of relocations applied"); 00054 STATISTIC(NumRetries, "Number of retries with more memory"); 00055 static JIT *TheJIT = 0; 00056 00057 00058 //===----------------------------------------------------------------------===// 00059 // JIT lazy compilation code. 00060 // 00061 namespace { 00062 class JITResolverState { 00063 public: 00064 typedef std::map<AssertingVH<Function>, void*> FunctionToStubMapTy; 00065 typedef std::map<void*, Function*> StubToFunctionMapTy; 00066 typedef std::map<AssertingVH<GlobalValue>, void*> GlobalToIndirectSymMapTy; 00067 private: 00068 /// FunctionToStubMap - Keep track of the stub created for a particular 00069 /// function so that we can reuse them if necessary. 00070 FunctionToStubMapTy FunctionToStubMap; 00071 00072 /// StubToFunctionMap - Keep track of the function that each stub 00073 /// corresponds to. 00074 StubToFunctionMapTy StubToFunctionMap; 00075 00076 /// GlobalToIndirectSymMap - Keep track of the indirect symbol created for a 00077 /// particular GlobalVariable so that we can reuse them if necessary. 00078 GlobalToIndirectSymMapTy GlobalToIndirectSymMap; 00079 00080 public: 00081 FunctionToStubMapTy& getFunctionToStubMap(const MutexGuard& locked) { 00082 assert(locked.holds(TheJIT->lock)); 00083 return FunctionToStubMap; 00084 } 00085 00086 StubToFunctionMapTy& getStubToFunctionMap(const MutexGuard& locked) { 00087 assert(locked.holds(TheJIT->lock)); 00088 return StubToFunctionMap; 00089 } 00090 00091 GlobalToIndirectSymMapTy& getGlobalToIndirectSymMap(const MutexGuard& locked) { 00092 assert(locked.holds(TheJIT->lock)); 00093 return GlobalToIndirectSymMap; 00094 } 00095 }; 00096 00097 /// JITResolver - Keep track of, and resolve, call sites for functions that 00098 /// have not yet been compiled. 00099 class JITResolver { 00100 typedef JITResolverState::FunctionToStubMapTy FunctionToStubMapTy; 00101 typedef JITResolverState::StubToFunctionMapTy StubToFunctionMapTy; 00102 typedef JITResolverState::GlobalToIndirectSymMapTy GlobalToIndirectSymMapTy; 00103 00104 /// LazyResolverFn - The target lazy resolver function that we actually 00105 /// rewrite instructions to use. 00106 TargetJITInfo::LazyResolverFn LazyResolverFn; 00107 00108 JITResolverState state; 00109 00110 /// ExternalFnToStubMap - This is the equivalent of FunctionToStubMap for 00111 /// external functions. 00112 std::map<void*, void*> ExternalFnToStubMap; 00113 00114 /// revGOTMap - map addresses to indexes in the GOT 00115 std::map<void*, unsigned> revGOTMap; 00116 unsigned nextGOTIndex; 00117 00118 static JITResolver *TheJITResolver; 00119 public: 00120 explicit JITResolver(JIT &jit) : nextGOTIndex(0) { 00121 TheJIT = &jit; 00122 00123 LazyResolverFn = jit.getJITInfo().getLazyResolverFunction(JITCompilerFn); 00124 assert(TheJITResolver == 0 && "Multiple JIT resolvers?"); 00125 TheJITResolver = this; 00126 } 00127 00128 ~JITResolver() { 00129 TheJITResolver = 0; 00130 } 00131 00132 /// getFunctionStubIfAvailable - This returns a pointer to a function stub 00133 /// if it has already been created. 00134 void *getFunctionStubIfAvailable(Function *F); 00135 00136 /// getFunctionStub - This returns a pointer to a function stub, creating 00137 /// one on demand as needed. If empty is true, create a function stub 00138 /// pointing at address 0, to be filled in later. 00139 void *getFunctionStub(Function *F); 00140 00141 /// getExternalFunctionStub - Return a stub for the function at the 00142 /// specified address, created lazily on demand. 00143 void *getExternalFunctionStub(void *FnAddr); 00144 00145 /// getGlobalValueIndirectSym - Return an indirect symbol containing the 00146 /// specified GV address. 00147 void *getGlobalValueIndirectSym(GlobalValue *V, void *GVAddress); 00148 00149 /// AddCallbackAtLocation - If the target is capable of rewriting an 00150 /// instruction without the use of a stub, record the location of the use so 00151 /// we know which function is being used at the location. 00152 void *AddCallbackAtLocation(Function *F, void *Location) { 00153 MutexGuard locked(TheJIT->lock); 00154 /// Get the target-specific JIT resolver function. 00155 state.getStubToFunctionMap(locked)[Location] = F; 00156 return (void*)(intptr_t)LazyResolverFn; 00157 } 00158 00159 void getRelocatableGVs(SmallVectorImpl<GlobalValue*> &GVs, 00160 SmallVectorImpl<void*> &Ptrs); 00161 00162 GlobalValue *invalidateStub(void *Stub); 00163 00164 /// getGOTIndexForAddress - Return a new or existing index in the GOT for 00165 /// an address. This function only manages slots, it does not manage the 00166 /// contents of the slots or the memory associated with the GOT. 00167 unsigned getGOTIndexForAddr(void *addr); 00168 00169 /// JITCompilerFn - This function is called to resolve a stub to a compiled 00170 /// address. If the LLVM Function corresponding to the stub has not yet 00171 /// been compiled, this function compiles it first. 00172 static void *JITCompilerFn(void *Stub); 00173 }; 00174 } 00175 00176 JITResolver *JITResolver::TheJITResolver = 0; 00177 00178 /// getFunctionStubIfAvailable - This returns a pointer to a function stub 00179 /// if it has already been created. 00180 void *JITResolver::getFunctionStubIfAvailable(Function *F) { 00181 MutexGuard locked(TheJIT->lock); 00182 00183 // If we already have a stub for this function, recycle it. 00184 void *&Stub = state.getFunctionToStubMap(locked)[F]; 00185 return Stub; 00186 } 00187 00188 /// getFunctionStub - This returns a pointer to a function stub, creating 00189 /// one on demand as needed. 00190 void *JITResolver::getFunctionStub(Function *F) { 00191 MutexGuard locked(TheJIT->lock); 00192 00193 // If we already have a stub for this function, recycle it. 00194 void *&Stub = state.getFunctionToStubMap(locked)[F]; 00195 if (Stub) return Stub; 00196 00197 // Call the lazy resolver function unless we are JIT'ing non-lazily, in which 00198 // case we must resolve the symbol now. 00199 void *Actual = TheJIT->isLazyCompilationDisabled() 00200 ? (void *)0 : (void *)(intptr_t)LazyResolverFn; 00201 00202 // If this is an external declaration, attempt to resolve the address now 00203 // to place in the stub. 00204 if (F->isDeclaration() && !F->hasNotBeenReadFromBitcode()) { 00205 Actual = TheJIT->getPointerToFunction(F); 00206 00207 // If we resolved the symbol to a null address (eg. a weak external) 00208 // don't emit a stub. Return a null pointer to the application. If dlsym 00209 // stubs are enabled, not being able to resolve the address is not 00210 // meaningful. 00211 if (!Actual && !TheJIT->areDlsymStubsEnabled()) return 0; 00212 } 00213 00214 // Codegen a new stub, calling the lazy resolver or the actual address of the 00215 // external function, if it was resolved. 00216 Stub = TheJIT->getJITInfo().emitFunctionStub(F, Actual, 00217 *TheJIT->getCodeEmitter()); 00218 00219 if (Actual != (void*)(intptr_t)LazyResolverFn) { 00220 // If we are getting the stub for an external function, we really want the 00221 // address of the stub in the GlobalAddressMap for the JIT, not the address 00222 // of the external function. 00223 TheJIT->updateGlobalMapping(F, Stub); 00224 } 00225 00226 DEBUG(errs() << "JIT: Stub emitted at [" << Stub << "] for function '" 00227 << F->getName() << "'\n"); 00228 00229 // Finally, keep track of the stub-to-Function mapping so that the 00230 // JITCompilerFn knows which function to compile! 00231 state.getStubToFunctionMap(locked)[Stub] = F; 00232 00233 // If we are JIT'ing non-lazily but need to call a function that does not 00234 // exist yet, add it to the JIT's work list so that we can fill in the stub 00235 // address later. 00236 if (!Actual && TheJIT->isLazyCompilationDisabled()) 00237 if (!F->isDeclaration() || F->hasNotBeenReadFromBitcode()) 00238 TheJIT->addPendingFunction(F); 00239 00240 return Stub; 00241 } 00242 00243 /// getGlobalValueIndirectSym - Return a lazy pointer containing the specified 00244 /// GV address. 00245 void *JITResolver::getGlobalValueIndirectSym(GlobalValue *GV, void *GVAddress) { 00246 MutexGuard locked(TheJIT->lock); 00247 00248 // If we already have a stub for this global variable, recycle it. 00249 void *&IndirectSym = state.getGlobalToIndirectSymMap(locked)[GV]; 00250 if (IndirectSym) return IndirectSym; 00251 00252 // Otherwise, codegen a new indirect symbol. 00253 IndirectSym = TheJIT->getJITInfo().emitGlobalValueIndirectSym(GV, GVAddress, 00254 *TheJIT->getCodeEmitter()); 00255 00256 DEBUG(errs() << "JIT: Indirect symbol emitted at [" << IndirectSym 00257 << "] for GV '" << GV->getName() << "'\n"); 00258 00259 return IndirectSym; 00260 } 00261 00262 /// getExternalFunctionStub - Return a stub for the function at the 00263 /// specified address, created lazily on demand. 00264 void *JITResolver::getExternalFunctionStub(void *FnAddr) { 00265 // If we already have a stub for this function, recycle it. 00266 void *&Stub = ExternalFnToStubMap[FnAddr]; 00267 if (Stub) return Stub; 00268 00269 Stub = TheJIT->getJITInfo().emitFunctionStub(0, FnAddr, 00270 *TheJIT->getCodeEmitter()); 00271 00272 DOUT << "JIT: Stub emitted at [" << Stub 00273 << "] for external function at '" << FnAddr << "'\n"; 00274 return Stub; 00275 } 00276 00277 unsigned JITResolver::getGOTIndexForAddr(void* addr) { 00278 unsigned idx = revGOTMap[addr]; 00279 if (!idx) { 00280 idx = ++nextGOTIndex; 00281 revGOTMap[addr] = idx; 00282 DOUT << "JIT: Adding GOT entry " << idx << " for addr [" << addr << "]\n"; 00283 } 00284 return idx; 00285 } 00286 00287 void JITResolver::getRelocatableGVs(SmallVectorImpl<GlobalValue*> &GVs, 00288 SmallVectorImpl<void*> &Ptrs) { 00289 MutexGuard locked(TheJIT->lock); 00290 00291 FunctionToStubMapTy &FM = state.getFunctionToStubMap(locked); 00292 GlobalToIndirectSymMapTy &GM = state.getGlobalToIndirectSymMap(locked); 00293 00294 for (FunctionToStubMapTy::iterator i = FM.begin(), e = FM.end(); i != e; ++i){ 00295 Function *F = i->first; 00296 if (F->isDeclaration() && F->hasExternalLinkage()) { 00297 GVs.push_back(i->first); 00298 Ptrs.push_back(i->second); 00299 } 00300 } 00301 for (GlobalToIndirectSymMapTy::iterator i = GM.begin(), e = GM.end(); 00302 i != e; ++i) { 00303 GVs.push_back(i->first); 00304 Ptrs.push_back(i->second); 00305 } 00306 } 00307 00308 GlobalValue *JITResolver::invalidateStub(void *Stub) { 00309 MutexGuard locked(TheJIT->lock); 00310 00311 FunctionToStubMapTy &FM = state.getFunctionToStubMap(locked); 00312 StubToFunctionMapTy &SM = state.getStubToFunctionMap(locked); 00313 GlobalToIndirectSymMapTy &GM = state.getGlobalToIndirectSymMap(locked); 00314 00315 // Look up the cheap way first, to see if it's a function stub we are 00316 // invalidating. If so, remove it from both the forward and reverse maps. 00317 if (SM.find(Stub) != SM.end()) { 00318 Function *F = SM[Stub]; 00319 SM.erase(Stub); 00320 FM.erase(F); 00321 return F; 00322 } 00323 00324 // Otherwise, it might be an indirect symbol stub. Find it and remove it. 00325 for (GlobalToIndirectSymMapTy::iterator i = GM.begin(), e = GM.end(); 00326 i != e; ++i) { 00327 if (i->second != Stub) 00328 continue; 00329 GlobalValue *GV = i->first; 00330 GM.erase(i); 00331 return GV; 00332 } 00333 00334 // Lastly, check to see if it's in the ExternalFnToStubMap. 00335 for (std::map<void *, void *>::iterator i = ExternalFnToStubMap.begin(), 00336 e = ExternalFnToStubMap.end(); i != e; ++i) { 00337 if (i->second != Stub) 00338 continue; 00339 ExternalFnToStubMap.erase(i); 00340 break; 00341 } 00342 00343 return 0; 00344 } 00345 00346 /// JITCompilerFn - This function is called when a lazy compilation stub has 00347 /// been entered. It looks up which function this stub corresponds to, compiles 00348 /// it if necessary, then returns the resultant function pointer. 00349 void *JITResolver::JITCompilerFn(void *Stub) { 00350 JITResolver &JR = *TheJITResolver; 00351 00352 Function* F = 0; 00353 void* ActualPtr = 0; 00354 00355 { 00356 // Only lock for getting the Function. The call getPointerToFunction made 00357 // in this function might trigger function materializing, which requires 00358 // JIT lock to be unlocked. 00359 MutexGuard locked(TheJIT->lock); 00360 00361 // The address given to us for the stub may not be exactly right, it might be 00362 // a little bit after the stub. As such, use upper_bound to find it. 00363 StubToFunctionMapTy::iterator I = 00364 JR.state.getStubToFunctionMap(locked).upper_bound(Stub); 00365 assert(I != JR.state.getStubToFunctionMap(locked).begin() && 00366 "This is not a known stub!"); 00367 F = (--I)->second; 00368 ActualPtr = I->first; 00369 } 00370 00371 // If we have already code generated the function, just return the address. 00372 void *Result = TheJIT->getPointerToGlobalIfAvailable(F); 00373 00374 if (!Result) { 00375 // Otherwise we don't have it, do lazy compilation now. 00376 00377 // If lazy compilation is disabled, emit a useful error message and abort. 00378 if (TheJIT->isLazyCompilationDisabled()) { 00379 llvm_report_error("LLVM JIT requested to do lazy compilation of function '" 00380 + F->getName() + "' when lazy compiles are disabled!"); 00381 } 00382 00383 // We might like to remove the stub from the StubToFunction map. 00384 // We can't do that! Multiple threads could be stuck, waiting to acquire the 00385 // lock above. As soon as the 1st function finishes compiling the function, 00386 // the next one will be released, and needs to be able to find the function 00387 // it needs to call. 00388 //JR.state.getStubToFunctionMap(locked).erase(I); 00389 00390 DEBUG(errs() << "JIT: Lazily resolving function '" << F->getName() 00391 << "' In stub ptr = " << Stub << " actual ptr = " 00392 << ActualPtr << "\n"); 00393 00394 Result = TheJIT->getPointerToFunction(F); 00395 } 00396 00397 // Reacquire the lock to erase the stub in the map. 00398 MutexGuard locked(TheJIT->lock); 00399 00400 // We don't need to reuse this stub in the future, as F is now compiled. 00401 JR.state.getFunctionToStubMap(locked).erase(F); 00402 00403 // FIXME: We could rewrite all references to this stub if we knew them. 00404 00405 // What we will do is set the compiled function address to map to the 00406 // same GOT entry as the stub so that later clients may update the GOT 00407 // if they see it still using the stub address. 00408 // Note: this is done so the Resolver doesn't have to manage GOT memory 00409 // Do this without allocating map space if the target isn't using a GOT 00410 if(JR.revGOTMap.find(Stub) != JR.revGOTMap.end()) 00411 JR.revGOTMap[Result] = JR.revGOTMap[Stub]; 00412 00413 return Result; 00414 } 00415 00416 //===----------------------------------------------------------------------===// 00417 // JITEmitter code. 00418 // 00419 namespace { 00420 /// JITEmitter - The JIT implementation of the MachineCodeEmitter, which is 00421 /// used to output functions to memory for execution. 00422 class JITEmitter : public JITCodeEmitter { 00423 JITMemoryManager *MemMgr; 00424 00425 // When outputting a function stub in the context of some other function, we 00426 // save BufferBegin/BufferEnd/CurBufferPtr here. 00427 uint8_t *SavedBufferBegin, *SavedBufferEnd, *SavedCurBufferPtr; 00428 00429 // When reattempting to JIT a function after running out of space, we store 00430 // the estimated size of the function we're trying to JIT here, so we can 00431 // ask the memory manager for at least this much space. When we 00432 // successfully emit the function, we reset this back to zero. 00433 uintptr_t SizeEstimate; 00434 00435 /// Relocations - These are the relocations that the function needs, as 00436 /// emitted. 00437 std::vector<MachineRelocation> Relocations; 00438 00439 /// MBBLocations - This vector is a mapping from MBB ID's to their address. 00440 /// It is filled in by the StartMachineBasicBlock callback and queried by 00441 /// the getMachineBasicBlockAddress callback. 00442 std::vector<uintptr_t> MBBLocations; 00443 00444 /// ConstantPool - The constant pool for the current function. 00445 /// 00446 MachineConstantPool *ConstantPool; 00447 00448 /// ConstantPoolBase - A pointer to the first entry in the constant pool. 00449 /// 00450 void *ConstantPoolBase; 00451 00452 /// ConstPoolAddresses - Addresses of individual constant pool entries. 00453 /// 00454 SmallVector<uintptr_t, 8> ConstPoolAddresses; 00455 00456 /// JumpTable - The jump tables for the current function. 00457 /// 00458 MachineJumpTableInfo *JumpTable; 00459 00460 /// JumpTableBase - A pointer to the first entry in the jump table. 00461 /// 00462 void *JumpTableBase; 00463 00464 /// Resolver - This contains info about the currently resolved functions. 00465 JITResolver Resolver; 00466 00467 /// DE - The dwarf emitter for the jit. 00468 JITDwarfEmitter *DE; 00469 00470 /// LabelLocations - This vector is a mapping from Label ID's to their 00471 /// address. 00472 std::vector<uintptr_t> LabelLocations; 00473 00474 /// MMI - Machine module info for exception informations 00475 MachineModuleInfo* MMI; 00476 00477 // GVSet - a set to keep track of which globals have been seen 00478 SmallPtrSet<const GlobalVariable*, 8> GVSet; 00479 00480 // CurFn - The llvm function being emitted. Only valid during 00481 // finishFunction(). 00482 const Function *CurFn; 00483 00484 /// Information about emitted code, which is passed to the 00485 /// JITEventListeners. This is reset in startFunction and used in 00486 /// finishFunction. 00487 JITEvent_EmittedFunctionDetails EmissionDetails; 00488 00489 // CurFnStubUses - For a given Function, a vector of stubs that it 00490 // references. This facilitates the JIT detecting that a stub is no 00491 // longer used, so that it may be deallocated. 00492 DenseMap<const Function *, SmallVector<void*, 1> > CurFnStubUses; 00493 00494 // StubFnRefs - For a given pointer to a stub, a set of Functions which 00495 // reference the stub. When the count of a stub's references drops to zero, 00496 // the stub is unused. 00497 DenseMap<void *, SmallPtrSet<const Function*, 1> > StubFnRefs; 00498 00499 // ExtFnStubs - A map of external function names to stubs which have entries 00500 // in the JITResolver's ExternalFnToStubMap. 00501 StringMap<void *> ExtFnStubs; 00502 00503 DebugLocTuple PrevDLT; 00504 00505 public: 00506 JITEmitter(JIT &jit, JITMemoryManager *JMM) 00507 : SizeEstimate(0), Resolver(jit), MMI(0), CurFn(0) { 00508 MemMgr = JMM ? JMM : JITMemoryManager::CreateDefaultMemManager(); 00509 if (jit.getJITInfo().needsGOT()) { 00510 MemMgr->AllocateGOT(); 00511 DOUT << "JIT is managing a GOT\n"; 00512 } 00513 00514 if (DwarfExceptionHandling) DE = new JITDwarfEmitter(jit); 00515 } 00516 ~JITEmitter() { 00517 delete MemMgr; 00518 if (DwarfExceptionHandling) delete DE; 00519 } 00520 00521 /// classof - Methods for support type inquiry through isa, cast, and 00522 /// dyn_cast: 00523 /// 00524 static inline bool classof(const JITEmitter*) { return true; } 00525 static inline bool classof(const MachineCodeEmitter*) { return true; } 00526 00527 JITResolver &getJITResolver() { return Resolver; } 00528 00529 virtual void startFunction(MachineFunction &F); 00530 virtual bool finishFunction(MachineFunction &F); 00531 00532 void emitConstantPool(MachineConstantPool *MCP); 00533 void initJumpTableInfo(MachineJumpTableInfo *MJTI); 00534 void emitJumpTableInfo(MachineJumpTableInfo *MJTI); 00535 00536 virtual void startGVStub(const GlobalValue* GV, unsigned StubSize, 00537 unsigned Alignment = 1); 00538 virtual void startGVStub(const GlobalValue* GV, void *Buffer, 00539 unsigned StubSize); 00540 virtual void* finishGVStub(const GlobalValue *GV); 00541 00542 /// allocateSpace - Reserves space in the current block if any, or 00543 /// allocate a new one of the given size. 00544 virtual void *allocateSpace(uintptr_t Size, unsigned Alignment); 00545 00546 /// allocateGlobal - Allocate memory for a global. Unlike allocateSpace, 00547 /// this method does not allocate memory in the current output buffer, 00548 /// because a global may live longer than the current function. 00549 virtual void *allocateGlobal(uintptr_t Size, unsigned Alignment); 00550 00551 virtual void addRelocation(const MachineRelocation &MR) { 00552 Relocations.push_back(MR); 00553 } 00554 00555 virtual void StartMachineBasicBlock(MachineBasicBlock *MBB) { 00556 if (MBBLocations.size() <= (unsigned)MBB->getNumber()) 00557 MBBLocations.resize((MBB->getNumber()+1)*2); 00558 MBBLocations[MBB->getNumber()] = getCurrentPCValue(); 00559 DOUT << "JIT: Emitting BB" << MBB->getNumber() << " at [" 00560 << (void*) getCurrentPCValue() << "]\n"; 00561 } 00562 00563 virtual uintptr_t getConstantPoolEntryAddress(unsigned Entry) const; 00564 virtual uintptr_t getJumpTableEntryAddress(unsigned Entry) const; 00565 00566 virtual uintptr_t getMachineBasicBlockAddress(MachineBasicBlock *MBB) const { 00567 assert(MBBLocations.size() > (unsigned)MBB->getNumber() && 00568 MBBLocations[MBB->getNumber()] && "MBB not emitted!"); 00569 return MBBLocations[MBB->getNumber()]; 00570 } 00571 00572 /// retryWithMoreMemory - Log a retry and deallocate all memory for the 00573 /// given function. Increase the minimum allocation size so that we get 00574 /// more memory next time. 00575 void retryWithMoreMemory(MachineFunction &F); 00576 00577 /// deallocateMemForFunction - Deallocate all memory for the specified 00578 /// function body. 00579 void deallocateMemForFunction(const Function *F); 00580 00581 /// AddStubToCurrentFunction - Mark the current function being JIT'd as 00582 /// using the stub at the specified address. Allows 00583 /// deallocateMemForFunction to also remove stubs no longer referenced. 00584 void AddStubToCurrentFunction(void *Stub); 00585 00586 /// getExternalFnStubs - Accessor for the JIT to find stubs emitted for 00587 /// MachineRelocations that reference external functions by name. 00588 const StringMap<void*> &getExternalFnStubs() const { return ExtFnStubs; } 00589 00590 virtual void processDebugLoc(DebugLoc DL); 00591 00592 virtual void emitLabel(uint64_t LabelID) { 00593 if (LabelLocations.size() <= LabelID) 00594 LabelLocations.resize((LabelID+1)*2); 00595 LabelLocations[LabelID] = getCurrentPCValue(); 00596 } 00597 00598 virtual uintptr_t getLabelAddress(uint64_t LabelID) const { 00599 assert(LabelLocations.size() > (unsigned)LabelID && 00600 LabelLocations[LabelID] && "Label not emitted!"); 00601 return LabelLocations[LabelID]; 00602 } 00603 00604 virtual void setModuleInfo(MachineModuleInfo* Info) { 00605 MMI = Info; 00606 if (DwarfExceptionHandling) DE->setModuleInfo(Info); 00607 } 00608 00609 void setMemoryExecutable() { 00610 MemMgr->setMemoryExecutable(); 00611 } 00612 00613 JITMemoryManager *getMemMgr() const { return MemMgr; } 00614 00615 private: 00616 void *getPointerToGlobal(GlobalValue *GV, void *Reference, bool NoNeedStub); 00617 void *getPointerToGVIndirectSym(GlobalValue *V, void *Reference, 00618 bool NoNeedStub); 00619 unsigned addSizeOfGlobal(const GlobalVariable *GV, unsigned Size); 00620 unsigned addSizeOfGlobalsInConstantVal(const Constant *C, unsigned Size); 00621 unsigned addSizeOfGlobalsInInitializer(const Constant *Init, unsigned Size); 00622 unsigned GetSizeOfGlobalsInBytes(MachineFunction &MF); 00623 }; 00624 } 00625 00626 void *JITEmitter::getPointerToGlobal(GlobalValue *V, void *Reference, 00627 bool DoesntNeedStub) { 00628 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(V)) 00629 return TheJIT->getOrEmitGlobalVariable(GV); 00630 00631 if (GlobalAlias *GA = dyn_cast<GlobalAlias>(V)) 00632 return TheJIT->getPointerToGlobal(GA->resolveAliasedGlobal(false)); 00633 00634 // If we have already compiled the function, return a pointer to its body. 00635 Function *F = cast<Function>(V); 00636 void *ResultPtr; 00637 if (!DoesntNeedStub && !TheJIT->isLazyCompilationDisabled()) { 00638 // Return the function stub if it's already created. 00639 ResultPtr = Resolver.getFunctionStubIfAvailable(F); 00640 if (ResultPtr) 00641 AddStubToCurrentFunction(ResultPtr); 00642 } else { 00643 ResultPtr = TheJIT->getPointerToGlobalIfAvailable(F); 00644 } 00645 if (ResultPtr) return ResultPtr; 00646 00647 // If this is an external function pointer, we can force the JIT to 00648 // 'compile' it, which really just adds it to the map. In dlsym mode, 00649 // external functions are forced through a stub, regardless of reloc type. 00650 if (F->isDeclaration() && !F->hasNotBeenReadFromBitcode() && 00651 DoesntNeedStub && !TheJIT->areDlsymStubsEnabled()) 00652 return TheJIT->getPointerToFunction(F); 00653 00654 // Okay, the function has not been compiled yet, if the target callback 00655 // mechanism is capable of rewriting the instruction directly, prefer to do 00656 // that instead of emitting a stub. This uses the lazy resolver, so is not 00657 // legal if lazy compilation is disabled. 00658 if (DoesntNeedStub && !TheJIT->isLazyCompilationDisabled()) 00659 return Resolver.AddCallbackAtLocation(F, Reference); 00660 00661 // Otherwise, we have to emit a stub. 00662 void *StubAddr = Resolver.getFunctionStub(F); 00663 00664 // Add the stub to the current function's list of referenced stubs, so we can 00665 // deallocate them if the current function is ever freed. It's possible to 00666 // return null from getFunctionStub in the case of a weak extern that fails 00667 // to resolve. 00668 if (StubAddr) 00669 AddStubToCurrentFunction(StubAddr); 00670 00671 return StubAddr; 00672 } 00673 00674 void *JITEmitter::getPointerToGVIndirectSym(GlobalValue *V, void *Reference, 00675 bool NoNeedStub) { 00676 // Make sure GV is emitted first, and create a stub containing the fully 00677 // resolved address. 00678 void *GVAddress = getPointerToGlobal(V, Reference, true); 00679 void *StubAddr = Resolver.getGlobalValueIndirectSym(V, GVAddress); 00680 00681 // Add the stub to the current function's list of referenced stubs, so we can 00682 // deallocate them if the current function is ever freed. 00683 AddStubToCurrentFunction(StubAddr); 00684 00685 return StubAddr; 00686 } 00687 00688 void JITEmitter::AddStubToCurrentFunction(void *StubAddr) { 00689 if (!TheJIT->areDlsymStubsEnabled()) 00690 return; 00691 00692 assert(CurFn && "Stub added to current function, but current function is 0!"); 00693 00694 SmallVectorImpl<void*> &StubsUsed = CurFnStubUses[CurFn]; 00695 StubsUsed.push_back(StubAddr); 00696 00697 SmallPtrSet<const Function *, 1> &FnRefs = StubFnRefs[StubAddr]; 00698 FnRefs.insert(CurFn); 00699 } 00700 00701 void JITEmitter::processDebugLoc(DebugLoc DL) { 00702 if (!DL.isUnknown()) { 00703 DebugLocTuple CurDLT = EmissionDetails.MF->getDebugLocTuple(DL); 00704 00705 if (CurDLT.CompileUnit != 0 && PrevDLT != CurDLT) { 00706 JITEvent_EmittedFunctionDetails::LineStart NextLine; 00707 NextLine.Address = getCurrentPCValue(); 00708 NextLine.Loc = DL; 00709 EmissionDetails.LineStarts.push_back(NextLine); 00710 } 00711 00712 PrevDLT = CurDLT; 00713 } 00714 } 00715 00716 static unsigned GetConstantPoolSizeInBytes(MachineConstantPool *MCP, 00717 const TargetData *TD) { 00718 const std::vector<MachineConstantPoolEntry> &Constants = MCP->getConstants(); 00719 if (Constants.empty()) return 0; 00720 00721 unsigned Size = 0; 00722 for (unsigned i = 0, e = Constants.size(); i != e; ++i) { 00723 MachineConstantPoolEntry CPE = Constants[i]; 00724 unsigned AlignMask = CPE.getAlignment() - 1; 00725 Size = (Size + AlignMask) & ~AlignMask; 00726 const Type *Ty = CPE.getType(); 00727 Size += TD->getTypeAllocSize(Ty); 00728 } 00729 return Size; 00730 } 00731 00732 static unsigned GetJumpTableSizeInBytes(MachineJumpTableInfo *MJTI) { 00733 const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables(); 00734 if (JT.empty()) return 0; 00735 00736 unsigned NumEntries = 0; 00737 for (unsigned i = 0, e = JT.size(); i != e; ++i) 00738 NumEntries += JT[i].MBBs.size(); 00739 00740 unsigned EntrySize = MJTI->getEntrySize(); 00741 00742 return NumEntries * EntrySize; 00743 } 00744 00745 static uintptr_t RoundUpToAlign(uintptr_t Size, unsigned Alignment) { 00746 if (Alignment == 0) Alignment = 1; 00747 // Since we do not know where the buffer will be allocated, be pessimistic. 00748 return Size + Alignment; 00749 } 00750 00751 /// addSizeOfGlobal - add the size of the global (plus any alignment padding) 00752 /// into the running total Size. 00753 00754 unsigned JITEmitter::addSizeOfGlobal(const GlobalVariable *GV, unsigned Size) { 00755 const Type *ElTy = GV->getType()->getElementType(); 00756 size_t GVSize = (size_t)TheJIT->getTargetData()->getTypeAllocSize(ElTy); 00757 size_t GVAlign = 00758 (size_t)TheJIT->getTargetData()->getPreferredAlignment(GV); 00759 DOUT << "JIT: Adding in size " << GVSize << " alignment " << GVAlign; 00760 DEBUG(GV->dump()); 00761 // Assume code section ends with worst possible alignment, so first 00762 // variable needs maximal padding. 00763 if (Size==0) 00764 Size = 1; 00765 Size = ((Size+GVAlign-1)/GVAlign)*GVAlign; 00766 Size += GVSize; 00767 return Size; 00768 } 00769 00770 /// addSizeOfGlobalsInConstantVal - find any globals that we haven't seen yet 00771 /// but are referenced from the constant; put them in GVSet and add their 00772 /// size into the running total Size. 00773 00774 unsigned JITEmitter::addSizeOfGlobalsInConstantVal(const Constant *C, 00775 unsigned Size) { 00776 // If its undefined, return the garbage. 00777 if (isa<UndefValue>(C)) 00778 return Size; 00779 00780 // If the value is a ConstantExpr 00781 if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) { 00782 Constant *Op0 = CE->getOperand(0); 00783 switch (CE->getOpcode()) { 00784 case Instruction::GetElementPtr: 00785 case Instruction::Trunc: 00786 case Instruction::ZExt: 00787 case Instruction::SExt: 00788 case Instruction::FPTrunc: 00789 case Instruction::FPExt: 00790 case Instruction::UIToFP: 00791 case Instruction::SIToFP: 00792 case Instruction::FPToUI: 00793 case Instruction::FPToSI: 00794 case Instruction::PtrToInt: 00795 case Instruction::IntToPtr: 00796 case Instruction::BitCast: { 00797 Size = addSizeOfGlobalsInConstantVal(Op0, Size); 00798 break; 00799 } 00800 case Instruction::Add: 00801 case Instruction::FAdd: 00802 case Instruction::Sub: 00803 case Instruction::FSub: 00804 case Instruction::Mul: 00805 case Instruction::FMul: 00806 case Instruction::UDiv: 00807 case Instruction::SDiv: 00808 case Instruction::URem: 00809 case Instruction::SRem: 00810 case Instruction::And: 00811 case Instruction::Or: 00812 case Instruction::Xor: { 00813 Size = addSizeOfGlobalsInConstantVal(Op0, Size); 00814 Size = addSizeOfGlobalsInConstantVal(CE->getOperand(1), Size); 00815 break; 00816 } 00817 default: { 00818 std::string msg; 00819 raw_string_ostream Msg(msg); 00820 Msg << "ConstantExpr not handled: " << *CE; 00821 llvm_report_error(Msg.str()); 00822 } 00823 } 00824 } 00825 00826 if (C->getType()->getTypeID() == Type::PointerTyID) 00827 if (const GlobalVariable* GV = dyn_cast<GlobalVariable>(C)) 00828 if (GVSet.insert(GV)) 00829 Size = addSizeOfGlobal(GV, Size); 00830 00831 return Size; 00832 } 00833 00834 /// addSizeOfGLobalsInInitializer - handle any globals that we haven't seen yet 00835 /// but are referenced from the given initializer. 00836 00837 unsigned JITEmitter::addSizeOfGlobalsInInitializer(const Constant *Init, 00838 unsigned Size) { 00839 if (!isa<UndefValue>(Init) && 00840 !isa<ConstantVector>(Init) && 00841 !isa<ConstantAggregateZero>(Init) && 00842 !isa<ConstantArray>(Init) && 00843 !isa<ConstantStruct>(Init) && 00844 Init->getType()->isFirstClassType()) 00845 Size = addSizeOfGlobalsInConstantVal(Init, Size); 00846 return Size; 00847 } 00848 00849 /// GetSizeOfGlobalsInBytes - walk the code for the function, looking for 00850 /// globals; then walk the initializers of those globals looking for more. 00851 /// If their size has not been considered yet, add it into the running total 00852 /// Size. 00853 00854 unsigned JITEmitter::GetSizeOfGlobalsInBytes(MachineFunction &MF) { 00855 unsigned Size = 0; 00856 GVSet.clear(); 00857 00858 for (MachineFunction::iterator MBB = MF.begin(), E = MF.end(); 00859 MBB != E; ++MBB) { 00860 for (MachineBasicBlock::const_iterator I = MBB->begin(), E = MBB->end(); 00861 I != E; ++I) { 00862 const TargetInstrDesc &Desc = I->getDesc(); 00863 const MachineInstr &MI = *I; 00864 unsigned NumOps = Desc.getNumOperands(); 00865 for (unsigned CurOp = 0; CurOp < NumOps; CurOp++) { 00866 const MachineOperand &MO = MI.getOperand(CurOp); 00867 if (MO.isGlobal()) { 00868 GlobalValue* V = MO.getGlobal(); 00869 const GlobalVariable *GV = dyn_cast<const GlobalVariable>(V); 00870 if (!GV) 00871 continue; 00872 // If seen in previous function, it will have an entry here. 00873 if (TheJIT->getPointerToGlobalIfAvailable(GV)) 00874 continue; 00875 // If seen earlier in this function, it will have an entry here. 00876 // FIXME: it should be possible to combine these tables, by 00877 // assuming the addresses of the new globals in this module 00878 // start at 0 (or something) and adjusting them after codegen 00879 // complete. Another possibility is to grab a marker bit in GV. 00880 if (GVSet.insert(GV)) 00881 // A variable as yet unseen. Add in its size. 00882 Size = addSizeOfGlobal(GV, Size); 00883 } 00884 } 00885 } 00886 } 00887 DOUT << "JIT: About to look through initializers\n"; 00888 // Look for more globals that are referenced only from initializers. 00889 // GVSet.end is computed each time because the set can grow as we go. 00890 for (SmallPtrSet<const GlobalVariable *, 8>::iterator I = GVSet.begin(); 00891 I != GVSet.end(); I++) { 00892 const GlobalVariable* GV = *I; 00893 if (GV->hasInitializer()) 00894 Size = addSizeOfGlobalsInInitializer(GV->getInitializer(), Size); 00895 } 00896 00897 return Size; 00898 } 00899 00900 void JITEmitter::startFunction(MachineFunction &F) { 00901 DEBUG(errs() << "JIT: Starting CodeGen of Function " 00902 << F.getFunction()->getName() << "\n"); 00903 00904 uintptr_t ActualSize = 0; 00905 // Set the memory writable, if it's not already 00906 MemMgr->setMemoryWritable(); 00907 if (MemMgr->NeedsExactSize()) { 00908 DOUT << "JIT: ExactSize\n"; 00909 const TargetInstrInfo* TII = F.getTarget().getInstrInfo(); 00910 MachineJumpTableInfo *MJTI = F.getJumpTableInfo(); 00911 MachineConstantPool *MCP = F.getConstantPool(); 00912 00913 // Ensure the constant pool/jump table info is at least 4-byte aligned. 00914 ActualSize = RoundUpToAlign(ActualSize, 16); 00915 00916 // Add the alignment of the constant pool 00917 ActualSize = RoundUpToAlign(ActualSize, MCP->getConstantPoolAlignment()); 00918 00919 // Add the constant pool size 00920 ActualSize += GetConstantPoolSizeInBytes(MCP, TheJIT->getTargetData()); 00921 00922 // Add the aligment of the jump table info 00923 ActualSize = RoundUpToAlign(ActualSize, MJTI->getAlignment()); 00924 00925 // Add the jump table size 00926 ActualSize += GetJumpTableSizeInBytes(MJTI); 00927 00928 // Add the alignment for the function 00929 ActualSize = RoundUpToAlign(ActualSize, 00930 std::max(F.getFunction()->getAlignment(), 8U)); 00931 00932 // Add the function size 00933 ActualSize += TII->GetFunctionSizeInBytes(F); 00934 00935 DOUT << "JIT: ActualSize before globals " << ActualSize << "\n"; 00936 // Add the size of the globals that will be allocated after this function. 00937 // These are all the ones referenced from this function that were not 00938 // previously allocated. 00939 ActualSize += GetSizeOfGlobalsInBytes(F); 00940 DOUT << "JIT: ActualSize after globals " << ActualSize << "\n"; 00941 } else if (SizeEstimate > 0) { 00942 // SizeEstimate will be non-zero on reallocation attempts. 00943 ActualSize = SizeEstimate; 00944 } 00945 00946 BufferBegin = CurBufferPtr = MemMgr->startFunctionBody(F.getFunction(), 00947 ActualSize); 00948 BufferEnd = BufferBegin+ActualSize; 00949 00950 // Ensure the constant pool/jump table info is at least 4-byte aligned. 00951 emitAlignment(16); 00952 00953 emitConstantPool(F.getConstantPool()); 00954 initJumpTableInfo(F.getJumpTableInfo()); 00955 00956 // About to start emitting the machine code for the function. 00957 emitAlignment(std::max(F.getFunction()->getAlignment(), 8U)); 00958 TheJIT->updateGlobalMapping(F.getFunction(), CurBufferPtr); 00959 00960 MBBLocations.clear(); 00961 00962 EmissionDetails.MF = &F; 00963 EmissionDetails.LineStarts.clear(); 00964 } 00965 00966 bool JITEmitter::finishFunction(MachineFunction &F) { 00967 if (CurBufferPtr == BufferEnd) { 00968 // We must call endFunctionBody before retrying, because 00969 // deallocateMemForFunction requires it. 00970 MemMgr->endFunctionBody(F.getFunction(), BufferBegin, CurBufferPtr); 00971 retryWithMoreMemory(F); 00972 return true; 00973 } 00974 00975 emitJumpTableInfo(F.getJumpTableInfo()); 00976 00977 // FnStart is the start of the text, not the start of the constant pool and 00978 // other per-function data. 00979 uint8_t *FnStart = 00980 (uint8_t *)TheJIT->getPointerToGlobalIfAvailable(F.getFunction()); 00981 00982 // FnEnd is the end of the function's machine code. 00983 uint8_t *FnEnd = CurBufferPtr; 00984 00985 if (!Relocations.empty()) { 00986 CurFn = F.getFunction(); 00987 NumRelos += Relocations.size(); 00988 00989 // Resolve the relocations to concrete pointers. 00990 for (unsigned i = 0, e = Relocations.size(); i != e; ++i) { 00991 MachineRelocation &MR = Relocations[i]; 00992 void *ResultPtr = 0; 00993 if (!MR.letTargetResolve()) { 00994 if (MR.isExternalSymbol()) { 00995 ResultPtr = TheJIT->getPointerToNamedFunction(MR.getExternalSymbol(), 00996 false); 00997 DOUT << "JIT: Map \'" << MR.getExternalSymbol() << "\' to [" 00998 << ResultPtr << "]\n"; 00999 01000 // If the target REALLY wants a stub for this function, emit it now. 01001 if (!MR.doesntNeedStub()) { 01002 if (!TheJIT->areDlsymStubsEnabled()) { 01003 ResultPtr = Resolver.getExternalFunctionStub(ResultPtr); 01004 } else { 01005 void *&Stub = ExtFnStubs[MR.getExternalSymbol()]; 01006 if (!Stub) { 01007 Stub = Resolver.getExternalFunctionStub((void *)&Stub); 01008 AddStubToCurrentFunction(Stub); 01009 } 01010 ResultPtr = Stub; 01011 } 01012 } 01013 } else if (MR.isGlobalValue()) { 01014 ResultPtr = getPointerToGlobal(MR.getGlobalValue(), 01015 BufferBegin+MR.getMachineCodeOffset(), 01016 MR.doesntNeedStub()); 01017 } else if (MR.isIndirectSymbol()) { 01018 ResultPtr = getPointerToGVIndirectSym(MR.getGlobalValue(), 01019 BufferBegin+MR.getMachineCodeOffset(), 01020 MR.doesntNeedStub()); 01021 } else if (MR.isBasicBlock()) { 01022 ResultPtr = (void*)getMachineBasicBlockAddress(MR.getBasicBlock()); 01023 } else if (MR.isConstantPoolIndex()) { 01024 ResultPtr = (void*)getConstantPoolEntryAddress(MR.getConstantPoolIndex()); 01025 } else { 01026 assert(MR.isJumpTableIndex()); 01027 ResultPtr=(void*)getJumpTableEntryAddress(MR.getJumpTableIndex()); 01028 } 01029 01030 MR.setResultPointer(ResultPtr); 01031 } 01032 01033 // if we are managing the GOT and the relocation wants an index, 01034 // give it one 01035 if (MR.isGOTRelative() && MemMgr->isManagingGOT()) { 01036 unsigned idx = Resolver.getGOTIndexForAddr(ResultPtr); 01037 MR.setGOTIndex(idx); 01038 if (((void**)MemMgr->getGOTBase())[idx] != ResultPtr) { 01039 DOUT << "JIT: GOT was out of date for " << ResultPtr 01040 << " pointing at " << ((void**)MemMgr->getGOTBase())[idx] 01041 << "\n"; 01042 ((void**)MemMgr->getGOTBase())[idx] = ResultPtr; 01043 } 01044 } 01045 } 01046 01047 CurFn = 0; 01048 TheJIT->getJITInfo().relocate(BufferBegin, &Relocations[0], 01049 Relocations.size(), MemMgr->getGOTBase()); 01050 } 01051 01052 // Update the GOT entry for F to point to the new code. 01053 if (MemMgr->isManagingGOT()) { 01054 unsigned idx = Resolver.getGOTIndexForAddr((void*)BufferBegin); 01055 if (((void**)MemMgr->getGOTBase())[idx] != (void*)BufferBegin) { 01056 DOUT << "JIT: GOT was out of date for " << (void*)BufferBegin 01057 << " pointing at " << ((void**)MemMgr->getGOTBase())[idx] << "\n"; 01058 ((void**)MemMgr->getGOTBase())[idx] = (void*)BufferBegin; 01059 } 01060 } 01061 01062 // CurBufferPtr may have moved beyond FnEnd, due to memory allocation for 01063 // global variables that were referenced in the relocations. 01064 MemMgr->endFunctionBody(F.getFunction(), BufferBegin, CurBufferPtr); 01065 01066 if (CurBufferPtr == BufferEnd) { 01067 retryWithMoreMemory(F); 01068 return true; 01069 } else { 01070 // Now that we've succeeded in emitting the function, reset the 01071 // SizeEstimate back down to zero. 01072 SizeEstimate = 0; 01073 } 01074 01075 BufferBegin = CurBufferPtr = 0; 01076 NumBytes += FnEnd-FnStart; 01077 01078 // Invalidate the icache if necessary. 01079 sys::Memory::InvalidateInstructionCache(FnStart, FnEnd-FnStart); 01080 01081 TheJIT->NotifyFunctionEmitted(*F.getFunction(), FnStart, FnEnd-FnStart, 01082 EmissionDetails); 01083 01084 DEBUG(errs() << "JIT: Finished CodeGen of [" << (void*)FnStart 01085 << "] Function: " << F.getFunction()->getName() 01086 << ": " << (FnEnd-FnStart) << " bytes of text, " 01087 << Relocations.size() << " relocations\n"); 01088 01089 Relocations.clear(); 01090 ConstPoolAddresses.clear(); 01091 01092 // Mark code region readable and executable if it's not so already. 01093 MemMgr->setMemoryExecutable(); 01094 01095 #ifndef NDEBUG 01096 { 01097 if (sys::hasDisassembler()) { 01098 DOUT << "JIT: Disassembled code:\n"; 01099 DOUT << sys::disassembleBuffer(FnStart, FnEnd-FnStart, (uintptr_t)FnStart); 01100 } else { 01101 DOUT << "JIT: Binary code:\n"; 01102 DOUT << std::hex; 01103 uint8_t* q = FnStart; 01104 for (int i = 0; q < FnEnd; q += 4, ++i) { 01105 if (i == 4) 01106 i = 0; 01107 if (i == 0) 01108 DOUT << "JIT: " << std::setw(8) << std::setfill('0') 01109 << (long)(q - FnStart) << ": "; 01110 bool Done = false; 01111 for (int j = 3; j >= 0; --j) { 01112 if (q + j >= FnEnd) 01113 Done = true; 01114 else 01115 DOUT << std::setw(2) << std::setfill('0') << (unsigned short)q[j]; 01116 } 01117 if (Done) 01118 break; 01119 DOUT << ' '; 01120 if (i == 3) 01121 DOUT << '\n'; 01122 } 01123 DOUT << std::dec; 01124 DOUT<< '\n'; 01125 } 01126 } 01127 #endif 01128 if (DwarfExceptionHandling) { 01129 uintptr_t ActualSize = 0; 01130 SavedBufferBegin = BufferBegin; 01131 SavedBufferEnd = BufferEnd; 01132 SavedCurBufferPtr = CurBufferPtr; 01133 01134 if (MemMgr->NeedsExactSize()) { 01135 ActualSize = DE->GetDwarfTableSizeInBytes(F, *this, FnStart, FnEnd); 01136 } 01137 01138 BufferBegin = CurBufferPtr = MemMgr->startExceptionTable(F.getFunction(), 01139 ActualSize); 01140 BufferEnd = BufferBegin+ActualSize; 01141 uint8_t* FrameRegister = DE->EmitDwarfTable(F, *this, FnStart, FnEnd); 01142 MemMgr->endExceptionTable(F.getFunction(), BufferBegin, CurBufferPtr, 01143 FrameRegister); 01144 BufferBegin = SavedBufferBegin; 01145 BufferEnd = SavedBufferEnd; 01146 CurBufferPtr = SavedCurBufferPtr; 01147 01148 TheJIT->RegisterTable(FrameRegister); 01149 } 01150 01151 if (MMI) 01152 MMI->EndFunction(); 01153 01154 return false; 01155 } 01156 01157 void JITEmitter::retryWithMoreMemory(MachineFunction &F) { 01158 DOUT << "JIT: Ran out of space for native code. Reattempting.\n"; 01159 Relocations.clear(); // Clear the old relocations or we'll reapply them. 01160 ConstPoolAddresses.clear(); 01161 ++NumRetries; 01162 deallocateMemForFunction(F.getFunction()); 01163 // Try again with at least twice as much free space. 01164 SizeEstimate = (uintptr_t)(2 * (BufferEnd - BufferBegin)); 01165 } 01166 01167 /// deallocateMemForFunction - Deallocate all memory for the specified 01168 /// function body. Also drop any references the function has to stubs. 01169 void JITEmitter::deallocateMemForFunction(const Function *F) { 01170 MemMgr->deallocateMemForFunction(F); 01171 01172 // If the function did not reference any stubs, return. 01173 if (CurFnStubUses.find(F) == CurFnStubUses.end()) 01174 return; 01175 01176 // For each referenced stub, erase the reference to this function, and then 01177 // erase the list of referenced stubs. 01178 SmallVectorImpl<void *> &StubList = CurFnStubUses[F]; 01179 for (unsigned i = 0, e = StubList.size(); i != e; ++i) { 01180 void *Stub = StubList[i]; 01181 01182 // If we already invalidated this stub for this function, continue. 01183 if (StubFnRefs.count(Stub) == 0) 01184 continue; 01185 01186 SmallPtrSet<const Function *, 1> &FnRefs = StubFnRefs[Stub]; 01187 FnRefs.erase(F); 01188 01189 // If this function was the last reference to the stub, invalidate the stub 01190 // in the JITResolver. Were there a memory manager deallocateStub routine, 01191 // we could call that at this point too. 01192 if (FnRefs.empty()) { 01193 DOUT << "\nJIT: Invalidated Stub at [" << Stub << "]\n"; 01194 StubFnRefs.erase(Stub); 01195 01196 // Invalidate the stub. If it is a GV stub, update the JIT's global 01197 // mapping for that GV to zero, otherwise, search the string map of 01198 // external function names to stubs and remove the entry for this stub. 01199 GlobalValue *GV = Resolver.invalidateStub(Stub); 01200 if (GV) { 01201 TheJIT->updateGlobalMapping(GV, 0); 01202 } else { 01203 for (StringMapIterator<void*> i = ExtFnStubs.begin(), 01204 e = ExtFnStubs.end(); i != e; ++i) { 01205 if (i->second == Stub) { 01206 ExtFnStubs.erase(i); 01207 break; 01208 } 01209 } 01210 } 01211 } 01212 } 01213 CurFnStubUses.erase(F); 01214 } 01215 01216 01217 void* JITEmitter::allocateSpace(uintptr_t Size, unsigned Alignment) { 01218 if (BufferBegin) 01219 return JITCodeEmitter::allocateSpace(Size, Alignment); 01220 01221 // create a new memory block if there is no active one. 01222 // care must be taken so that BufferBegin is invalidated when a 01223 // block is trimmed 01224 BufferBegin = CurBufferPtr = MemMgr->allocateSpace(Size, Alignment); 01225 BufferEnd = BufferBegin+Size; 01226 return CurBufferPtr; 01227 } 01228 01229 void* JITEmitter::allocateGlobal(uintptr_t Size, unsigned Alignment) { 01230 // Delegate this call through the memory manager. 01231 return MemMgr->allocateGlobal(Size, Alignment); 01232 } 01233 01234 void JITEmitter::emitConstantPool(MachineConstantPool *MCP) { 01235 if (TheJIT->getJITInfo().hasCustomConstantPool()) 01236 return; 01237 01238 const std::vector<MachineConstantPoolEntry> &Constants = MCP->getConstants(); 01239 if (Constants.empty()) return; 01240 01241 unsigned Size = GetConstantPoolSizeInBytes(MCP, TheJIT->getTargetData()); 01242 unsigned Align = MCP->getConstantPoolAlignment(); 01243 ConstantPoolBase = allocateSpace(Size, Align); 01244 ConstantPool = MCP; 01245 01246 if (ConstantPoolBase == 0) return; // Buffer overflow. 01247 01248 DOUT << "JIT: Emitted constant pool at [" << ConstantPoolBase 01249 << "] (size: " << Size << ", alignment: " << Align << ")\n"; 01250 01251 // Initialize the memory for all of the constant pool entries. 01252 unsigned Offset = 0; 01253 for (unsigned i = 0, e = Constants.size(); i != e; ++i) { 01254 MachineConstantPoolEntry CPE = Constants[i]; 01255 unsigned AlignMask = CPE.getAlignment() - 1; 01256 Offset = (Offset + AlignMask) & ~AlignMask; 01257 01258 uintptr_t CAddr = (uintptr_t)ConstantPoolBase + Offset; 01259 ConstPoolAddresses.push_back(CAddr); 01260 if (CPE.isMachineConstantPoolEntry()) { 01261 // FIXME: add support to lower machine constant pool values into bytes! 01262 llvm_report_error("Initialize memory with machine specific constant pool" 01263 "entry has not been implemented!"); 01264 } 01265 TheJIT->InitializeMemory(CPE.Val.ConstVal, (void*)CAddr); 01266 DOUT << "JIT: CP" << i << " at [0x" 01267 << std::hex << CAddr << std::dec << "]\n"; 01268 01269 const Type *Ty = CPE.Val.ConstVal->getType(); 01270 Offset += TheJIT->getTargetData()->getTypeAllocSize(Ty); 01271 } 01272 } 01273 01274 void JITEmitter::initJumpTableInfo(MachineJumpTableInfo *MJTI) { 01275 if (TheJIT->getJITInfo().hasCustomJumpTables()) 01276 return; 01277 01278 const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables(); 01279 if (JT.empty()) return; 01280 01281 unsigned NumEntries = 0; 01282 for (unsigned i = 0, e = JT.size(); i != e; ++i) 01283 NumEntries += JT[i].MBBs.size(); 01284 01285 unsigned EntrySize = MJTI->getEntrySize(); 01286 01287 // Just allocate space for all the jump tables now. We will fix up the actual 01288 // MBB entries in the tables after we emit the code for each block, since then 01289 // we will know the final locations of the MBBs in memory. 01290 JumpTable = MJTI; 01291 JumpTableBase = allocateSpace(NumEntries * EntrySize, MJTI->getAlignment()); 01292 } 01293 01294 void JITEmitter::emitJumpTableInfo(MachineJumpTableInfo *MJTI) { 01295 if (TheJIT->getJITInfo().hasCustomJumpTables()) 01296 return; 01297 01298 const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables(); 01299 if (JT.empty() || JumpTableBase == 0) return; 01300 01301 if (TargetMachine::getRelocationModel() == Reloc::PIC_) { 01302 assert(MJTI->getEntrySize() == 4 && "Cross JIT'ing?"); 01303 // For each jump table, place the offset from the beginning of the table 01304 // to the target address. 01305 int *SlotPtr = (int*)JumpTableBase; 01306 01307 for (unsigned i = 0, e = JT.size(); i != e; ++i) { 01308 const std::vector<MachineBasicBlock*> &MBBs = JT[i].MBBs; 01309 // Store the offset of the basic block for this jump table slot in the 01310 // memory we allocated for the jump table in 'initJumpTableInfo' 01311 uintptr_t Base = (uintptr_t)SlotPtr; 01312 for (unsigned mi = 0, me = MBBs.size(); mi != me; ++mi) { 01313 uintptr_t MBBAddr = getMachineBasicBlockAddress(MBBs[mi]); 01314 *SlotPtr++ = TheJIT->getJITInfo().getPICJumpTableEntry(MBBAddr, Base); 01315 } 01316 } 01317 } else { 01318 assert(MJTI->getEntrySize() == sizeof(void*) && "Cross JIT'ing?"); 01319 01320 // For each jump table, map each target in the jump table to the address of 01321 // an emitted MachineBasicBlock. 01322 intptr_t *SlotPtr = (intptr_t*)JumpTableBase; 01323 01324 for (unsigned i = 0, e = JT.size(); i != e; ++i) { 01325 const std::vector<MachineBasicBlock*> &MBBs = JT[i].MBBs; 01326 // Store the address of the basic block for this jump table slot in the 01327 // memory we allocated for the jump table in 'initJumpTableInfo' 01328 for (unsigned mi = 0, me = MBBs.size(); mi != me; ++mi) 01329 *SlotPtr++ = getMachineBasicBlockAddress(MBBs[mi]); 01330 } 01331 } 01332 } 01333 01334 void JITEmitter::startGVStub(const GlobalValue* GV, unsigned StubSize, 01335 unsigned Alignment) { 01336 SavedBufferBegin = BufferBegin; 01337 SavedBufferEnd = BufferEnd; 01338 SavedCurBufferPtr = CurBufferPtr; 01339 01340 BufferBegin = CurBufferPtr = MemMgr->allocateStub(GV, StubSize, Alignment); 01341 BufferEnd = BufferBegin+StubSize+1; 01342 } 01343 01344 void JITEmitter::startGVStub(const GlobalValue* GV, void *Buffer, 01345 unsigned StubSize) { 01346 SavedBufferBegin = BufferBegin; 01347 SavedBufferEnd = BufferEnd; 01348 SavedCurBufferPtr = CurBufferPtr; 01349 01350 BufferBegin = CurBufferPtr = (uint8_t *)Buffer; 01351 BufferEnd = BufferBegin+StubSize+1; 01352 } 01353 01354 void *JITEmitter::finishGVStub(const GlobalValue* GV) { 01355 NumBytes += getCurrentPCOffset(); 01356 std::swap(SavedBufferBegin, BufferBegin); 01357 BufferEnd = SavedBufferEnd; 01358 CurBufferPtr = SavedCurBufferPtr; 01359 return SavedBufferBegin; 01360 } 01361 01362 // getConstantPoolEntryAddress - Return the address of the 'ConstantNum' entry 01363 // in the constant pool that was last emitted with the 'emitConstantPool' 01364 // method. 01365 // 01366 uintptr_t JITEmitter::getConstantPoolEntryAddress(unsigned ConstantNum) const { 01367 assert(ConstantNum < ConstantPool->getConstants().size() && 01368 "Invalid ConstantPoolIndex!"); 01369 return ConstPoolAddresses[ConstantNum]; 01370 } 01371 01372 // getJumpTableEntryAddress - Return the address of the JumpTable with index 01373 // 'Index' in the jumpp table that was last initialized with 'initJumpTableInfo' 01374 // 01375 uintptr_t JITEmitter::getJumpTableEntryAddress(unsigned Index) const { 01376 const std::vector<MachineJumpTableEntry> &JT = JumpTable->getJumpTables(); 01377 assert(Index < JT.size() && "Invalid jump table index!"); 01378 01379 unsigned Offset = 0; 01380 unsigned EntrySize = JumpTable->getEntrySize(); 01381 01382 for (unsigned i = 0; i < Index; ++i) 01383 Offset += JT[i].MBBs.size(); 01384 01385 Offset *= EntrySize; 01386 01387 return (uintptr_t)((char *)JumpTableBase + Offset); 01388 } 01389 01390 //===----------------------------------------------------------------------===// 01391 // Public interface to this file 01392 //===----------------------------------------------------------------------===// 01393 01394 JITCodeEmitter *JIT::createEmitter(JIT &jit, JITMemoryManager *JMM) { 01395 return new JITEmitter(jit, JMM); 01396 } 01397 01398 // getPointerToNamedFunction - This function is used as a global wrapper to 01399 // JIT::getPointerToNamedFunction for the purpose of resolving symbols when 01400 // bugpoint is debugging the JIT. In that scenario, we are loading an .so and 01401 // need to resolve function(s) that are being mis-codegenerated, so we need to 01402 // resolve their addresses at runtime, and this is the way to do it. 01403 extern "C" { 01404 void *getPointerToNamedFunction(const char *Name) { 01405 if (Function *F = TheJIT->FindFunctionNamed(Name)) 01406 return TheJIT->getPointerToFunction(F); 01407 return TheJIT->getPointerToNamedFunction(Name); 01408 } 01409 } 01410 01411 // getPointerToFunctionOrStub - If the specified function has been 01412 // code-gen'd, return a pointer to the function. If not, compile it, or use 01413 // a stub to implement lazy compilation if available. 01414 // 01415 void *JIT::getPointerToFunctionOrStub(Function *F) { 01416 // If we have already code generated the function, just return the address. 01417 if (void *Addr = getPointerToGlobalIfAvailable(F)) 01418 return Addr; 01419 01420 // Get a stub if the target supports it. 01421 assert(isa<JITEmitter>(JCE) && "Unexpected MCE?"); 01422 JITEmitter *JE = cast<JITEmitter>(getCodeEmitter()); 01423 return JE->getJITResolver().getFunctionStub(F); 01424 } 01425 01426 void JIT::updateFunctionStub(Function *F) { 01427 // Get the empty stub we generated earlier. 01428 assert(isa<JITEmitter>(JCE) && "Unexpected MCE?"); 01429 JITEmitter *JE = cast<JITEmitter>(getCodeEmitter()); 01430 void *Stub = JE->getJITResolver().getFunctionStub(F); 01431 01432 // Tell the target jit info to rewrite the stub at the specified address, 01433 // rather than creating a new one. 01434 void *Addr = getPointerToGlobalIfAvailable(F); 01435 getJITInfo().emitFunctionStubAtAddr(F, Addr, Stub, *getCodeEmitter()); 01436 } 01437 01438 /// updateDlsymStubTable - Emit the data necessary to relocate the stubs 01439 /// that were emitted during code generation. 01440 /// 01441 void JIT::updateDlsymStubTable() { 01442 assert(isa<JITEmitter>(JCE) && "Unexpected MCE?"); 01443 JITEmitter *JE = cast<JITEmitter>(getCodeEmitter()); 01444 01445 SmallVector<GlobalValue*, 8> GVs; 01446 SmallVector<void*, 8> Ptrs; 01447 const StringMap<void *> &ExtFns = JE->getExternalFnStubs(); 01448 01449 JE->getJITResolver().getRelocatableGVs(GVs, Ptrs); 01450 01451 unsigned nStubs = GVs.size() + ExtFns.size(); 01452 01453 // If there are no relocatable stubs, return. 01454 if (nStubs == 0) 01455 return; 01456 01457 // If there are no new relocatable stubs, return. 01458 void *CurTable = JE->getMemMgr()->getDlsymTable(); 01459 if (CurTable && (*(unsigned *)CurTable == nStubs)) 01460 return; 01461 01462 // Calculate the size of the stub info 01463 unsigned offset = 4 + 4 * nStubs + sizeof(intptr_t) * nStubs; 01464 01465 SmallVector<unsigned, 8> Offsets; 01466 for (unsigned i = 0; i != GVs.size(); ++i) { 01467 Offsets.push_back(offset); 01468 offset += GVs[i]->getName().size() + 1; 01469 } 01470 for (StringMapConstIterator<void*> i = ExtFns.begin(), e = ExtFns.end(); 01471 i != e; ++i) { 01472 Offsets.push_back(offset); 01473 offset += strlen(i->first()) + 1; 01474 } 01475 01476 // Allocate space for the new "stub", which contains the dlsym table. 01477 JE->startGVStub(0, offset, 4); 01478 01479 // Emit the number of records 01480 JE->emitInt32(nStubs); 01481 01482 // Emit the string offsets 01483 for (unsigned i = 0; i != nStubs; ++i) 01484 JE->emitInt32(Offsets[i]); 01485 01486 // Emit the pointers. Verify that they are at least 2-byte aligned, and set 01487 // the low bit to 0 == GV, 1 == Function, so that the client code doing the 01488 // relocation can write the relocated pointer at the appropriate place in 01489 // the stub. 01490 for (unsigned i = 0; i != GVs.size(); ++i) { 01491 intptr_t Ptr = (intptr_t)Ptrs[i]; 01492 assert((Ptr & 1) == 0 && "Stub pointers must be at least 2-byte aligned!"); 01493 01494 if (isa<Function>(GVs[i])) 01495 Ptr |= (intptr_t)1; 01496 01497 if (sizeof(Ptr) == 8) 01498 JE->emitInt64(Ptr); 01499 else 01500 JE->emitInt32(Ptr); 01501 } 01502 for (StringMapConstIterator<void*> i = ExtFns.begin(), e = ExtFns.end(); 01503 i != e; ++i) { 01504 intptr_t Ptr = (intptr_t)i->second | 1; 01505 01506 if (sizeof(Ptr) == 8) 01507 JE->emitInt64(Ptr); 01508 else 01509 JE->emitInt32(Ptr); 01510 } 01511 01512 // Emit the strings. 01513 for (unsigned i = 0; i != GVs.size(); ++i) 01514 JE->emitString(GVs[i]->getName()); 01515 for (StringMapConstIterator<void*> i = ExtFns.begin(), e = ExtFns.end(); 01516 i != e; ++i) 01517 JE->emitString(i->first()); 01518 01519 // Tell the JIT memory manager where it is. The JIT Memory Manager will 01520 // deallocate space for the old one, if one existed. 01521 JE->getMemMgr()->SetDlsymTable(JE->finishGVStub(0)); 01522 } 01523 01524 /// freeMachineCodeForFunction - release machine code memory for given Function. 01525 /// 01526 void JIT::freeMachineCodeForFunction(Function *F) { 01527 01528 // Delete translation for this from the ExecutionEngine, so it will get 01529 // retranslated next time it is used. 01530 void *OldPtr = updateGlobalMapping(F, 0); 01531 01532 if (OldPtr) 01533 TheJIT->NotifyFreeingMachineCode(*F, OldPtr); 01534 01535 // Free the actual memory for the function body and related stuff. 01536 assert(isa<JITEmitter>(JCE) && "Unexpected MCE?"); 01537 cast<JITEmitter>(JCE)->deallocateMemForFunction(F); 01538 }