Subversion Repositories HelenOS

Rev

Rev 945 | Rev 958 | Go to most recent revision | Blame | Compare with Previous | Last modification | View Log | Download | RSS feed

  1. /*
  2.  * Copyright (C) 2006 Jakub Jermar
  3.  * All rights reserved.
  4.  *
  5.  * Redistribution and use in source and binary forms, with or without
  6.  * modification, are permitted provided that the following conditions
  7.  * are met:
  8.  *
  9.  * - Redistributions of source code must retain the above copyright
  10.  *   notice, this list of conditions and the following disclaimer.
  11.  * - Redistributions in binary form must reproduce the above copyright
  12.  *   notice, this list of conditions and the following disclaimer in the
  13.  *   documentation and/or other materials provided with the distribution.
  14.  * - The name of the author may not be used to endorse or promote products
  15.  *   derived from this software without specific prior written permission.
  16.  *
  17.  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18.  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19.  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20.  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21.  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22.  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23.  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24.  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25.  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26.  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27.  */
  28.  
  29. /*
  30.  * TLB management.
  31.  */
  32.  
  33. #include <mm/tlb.h>
  34. #include <mm/asid.h>
  35. #include <mm/page.h>
  36. #include <mm/as.h>
  37. #include <arch/mm/tlb.h>
  38. #include <arch/mm/page.h>
  39. #include <arch/barrier.h>
  40. #include <arch/interrupt.h>
  41. #include <arch/pal/pal.h>
  42. #include <arch/asm.h>
  43. #include <typedefs.h>
  44. #include <panic.h>
  45. #include <arch.h>
  46.  
  47.  
  48.  
  49. /** Invalidate all TLB entries. */
  50. void tlb_invalidate_all(void)
  51. {
  52.         __address adr;
  53.         __u32 count1,count2,stride1,stride2;
  54.        
  55.         int i,j;
  56.        
  57.         adr=PAL_PTCE_INFO_BASE();
  58.         count1=PAL_PTCE_INFO_COUNT1();
  59.         count2=PAL_PTCE_INFO_COUNT2();
  60.         stride1=PAL_PTCE_INFO_STRIDE1();
  61.         stride2=PAL_PTCE_INFO_STRIDE2();
  62.        
  63.         interrupts_disable();
  64.  
  65.         for(i=0;i<count1;i++)
  66.         {
  67.             for(j=0;j<count2;j++)
  68.             {
  69.                 asm volatile
  70.                 (
  71.                     "ptc.e %0;;"
  72.                     :
  73.                     :"r" (adr)
  74.                 );
  75.                 adr+=stride2;
  76.             }
  77.             adr+=stride1;
  78.         }
  79.  
  80.         interrupts_enable();
  81.  
  82.         srlz_d();
  83.         srlz_i();
  84. }
  85.  
  86. /** Invalidate entries belonging to an address space.
  87.  *
  88.  * @param asid Address space identifier.
  89.  */
  90. void tlb_invalidate_asid(asid_t asid)
  91. {
  92.     /* TODO */
  93.     tlb_invalidate_all();
  94. }
  95.  
  96.  
  97. void tlb_invalidate_pages(asid_t asid, __address page, count_t cnt)
  98. {
  99.  
  100.  
  101.     region_register rr;
  102.     bool restore_rr = false;
  103.     int b=0;
  104.     int c=cnt;
  105.  
  106.     __address va;
  107.     va=page;
  108.  
  109.     rr.word = rr_read(VA2VRN(va));
  110.     if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
  111.         /*
  112.          * The selected region register does not contain required RID.
  113.          * Save the old content of the register and replace the RID.
  114.          */
  115.         region_register rr0;
  116.  
  117.         rr0 = rr;
  118.         rr0.map.rid = ASID2RID(asid, VA2VRN(va));
  119.         rr_write(VA2VRN(va), rr0.word);
  120.         srlz_d();
  121.         srlz_i();
  122.     }
  123.    
  124.     while(c>>=1)    b++;
  125.     b>>=1;
  126.     __u64 ps;
  127.    
  128.     switch(b)
  129.     {
  130.         case 0: /*cnt 1-3*/
  131.         {
  132.             ps=PAGE_WIDTH;
  133.             break;
  134.         }
  135.         case 1: /*cnt 4-15*/
  136.         {
  137.             /*cnt=((cnt-1)/4)+1;*/
  138.             ps=PAGE_WIDTH+2;
  139.             va&=~((1<<ps)-1);
  140.             break;
  141.         }
  142.         case 2: /*cnt 16-63*/
  143.         {
  144.             /*cnt=((cnt-1)/16)+1;*/
  145.             ps=PAGE_WIDTH+4;
  146.             va&=~((1<<ps)-1);
  147.             break;
  148.         }
  149.         case 3: /*cnt 64-255*/
  150.         {
  151.             /*cnt=((cnt-1)/64)+1;*/
  152.             ps=PAGE_WIDTH+6;
  153.             va&=~((1<<ps)-1);
  154.             break;
  155.         }
  156.         case 4: /*cnt 256-1023*/
  157.         {
  158.             /*cnt=((cnt-1)/256)+1;*/
  159.             ps=PAGE_WIDTH+8;
  160.             va&=~((1<<ps)-1);
  161.             break;
  162.         }
  163.         case 5: /*cnt 1024-4095*/
  164.         {
  165.             /*cnt=((cnt-1)/1024)+1;*/
  166.             ps=PAGE_WIDTH+10;
  167.             va&=~((1<<ps)-1);
  168.             break;
  169.         }
  170.         case 6: /*cnt 4096-16383*/
  171.         {
  172.             /*cnt=((cnt-1)/4096)+1;*/
  173.             ps=PAGE_WIDTH+12;
  174.             va&=~((1<<ps)-1);
  175.             break;
  176.         }
  177.         case 7: /*cnt 16384-65535*/
  178.         case 8: /*cnt 65536-(256K-1)*/
  179.         {
  180.             /*cnt=((cnt-1)/16384)+1;*/
  181.             ps=PAGE_WIDTH+14;
  182.             va&=~((1<<ps)-1);
  183.             break;
  184.         }
  185.         default:
  186.         {
  187.             /*cnt=((cnt-1)/(16384*16))+1;*/
  188.             ps=PAGE_WIDTH+18;
  189.             va&=~((1<<ps)-1);
  190.             break;
  191.         }
  192.     }
  193.     /*cnt+=(page!=va);*/
  194.     for(;va<(page+cnt*(PAGE_SIZE));va+=(1<<ps)) {
  195.         __asm__ volatile
  196.         (
  197.             "ptc.l %0,%1;;"
  198.             :
  199.             : "r"(va), "r"(ps<<2)
  200.         );
  201.     }
  202.     srlz_d();
  203.     srlz_i();
  204.    
  205.    
  206.     if (restore_rr) {
  207.         rr_write(VA2VRN(va), rr.word);
  208.         srlz_d();
  209.         srlz_i();
  210.     }
  211.  
  212.  
  213. }
  214.  
  215.  
  216. /** Insert data into data translation cache.
  217.  *
  218.  * @param va Virtual page address.
  219.  * @param asid Address space identifier.
  220.  * @param entry The rest of TLB entry as required by TLB insertion format.
  221.  */
  222. void dtc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry)
  223. {
  224.     tc_mapping_insert(va, asid, entry, true);
  225. }
  226.  
  227. /** Insert data into instruction translation cache.
  228.  *
  229.  * @param va Virtual page address.
  230.  * @param asid Address space identifier.
  231.  * @param entry The rest of TLB entry as required by TLB insertion format.
  232.  */
  233. void itc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry)
  234. {
  235.     tc_mapping_insert(va, asid, entry, false);
  236. }
  237.  
  238. /** Insert data into instruction or data translation cache.
  239.  *
  240.  * @param va Virtual page address.
  241.  * @param asid Address space identifier.
  242.  * @param entry The rest of TLB entry as required by TLB insertion format.
  243.  * @param dtc If true, insert into data translation cache, use instruction translation cache otherwise.
  244.  */
  245. void tc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, bool dtc)
  246. {
  247.     region_register rr;
  248.     bool restore_rr = false;
  249.  
  250.     rr.word = rr_read(VA2VRN(va));
  251.     if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
  252.         /*
  253.          * The selected region register does not contain required RID.
  254.          * Save the old content of the register and replace the RID.
  255.          */
  256.         region_register rr0;
  257.  
  258.         rr0 = rr;
  259.         rr0.map.rid = ASID2RID(asid, VA2VRN(va));
  260.         rr_write(VA2VRN(va), rr0.word);
  261.         srlz_d();
  262.         srlz_i();
  263.     }
  264.    
  265.     __asm__ volatile (
  266.         "mov r8=psr;;\n"
  267.         "rsm %0;;\n"            /* PSR_IC_MASK */
  268.         "srlz.d;;\n"
  269.         "srlz.i;;\n"
  270.         "mov cr.ifa=%1\n"       /* va */
  271.         "mov cr.itir=%2;;\n"        /* entry.word[1] */
  272.         "cmp.eq p6,p7 = %4,r0;;\n"  /* decide between itc and dtc */
  273.         "(p6) itc.i %3;;\n"
  274.         "(p7) itc.d %3;;\n"
  275.         "mov psr.l=r8;;\n"
  276.         "srlz.d;;\n"
  277.         :
  278.         : "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]), "r" (entry.word[0]), "r" (dtc)
  279.         : "p6", "p7", "r8"
  280.     );
  281.    
  282.     if (restore_rr) {
  283.         rr_write(VA2VRN(va), rr.word);
  284.         srlz_d();
  285.         srlz_i();
  286.     }
  287. }
  288.  
  289. /** Insert data into instruction translation register.
  290.  *
  291.  * @param va Virtual page address.
  292.  * @param asid Address space identifier.
  293.  * @param entry The rest of TLB entry as required by TLB insertion format.
  294.  * @param tr Translation register.
  295.  */
  296. void itr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, index_t tr)
  297. {
  298.     tr_mapping_insert(va, asid, entry, false, tr);
  299. }
  300.  
  301. /** Insert data into data translation register.
  302.  *
  303.  * @param va Virtual page address.
  304.  * @param asid Address space identifier.
  305.  * @param entry The rest of TLB entry as required by TLB insertion format.
  306.  * @param tr Translation register.
  307.  */
  308. void dtr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, index_t tr)
  309. {
  310.     tr_mapping_insert(va, asid, entry, true, tr);
  311. }
  312.  
  313. /** Insert data into instruction or data translation register.
  314.  *
  315.  * @param va Virtual page address.
  316.  * @param asid Address space identifier.
  317.  * @param entry The rest of TLB entry as required by TLB insertion format.
  318.  * @param dtc If true, insert into data translation register, use instruction translation register otherwise.
  319.  * @param tr Translation register.
  320.  */
  321. void tr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, bool dtr, index_t tr)
  322. {
  323.     region_register rr;
  324.     bool restore_rr = false;
  325.  
  326.     rr.word = rr_read(VA2VRN(va));
  327.     if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
  328.         /*
  329.          * The selected region register does not contain required RID.
  330.          * Save the old content of the register and replace the RID.
  331.          */
  332.         region_register rr0;
  333.  
  334.         rr0 = rr;
  335.         rr0.map.rid = ASID2RID(asid, VA2VRN(va));
  336.         rr_write(VA2VRN(va), rr0.word);
  337.         srlz_d();
  338.         srlz_i();
  339.     }
  340.  
  341.     __asm__ volatile (
  342.         "mov r8=psr;;\n"
  343.         "rsm %0;;\n"            /* PSR_IC_MASK */
  344.         "srlz.d;;\n"
  345.         "srlz.i;;\n"
  346.         "mov cr.ifa=%1\n"           /* va */         
  347.         "mov cr.itir=%2;;\n"        /* entry.word[1] */
  348.         "cmp.eq p6,p7=%5,r0;;\n"    /* decide between itr and dtr */
  349.         "(p6) itr.i itr[%4]=%3;;\n"
  350.         "(p7) itr.d dtr[%4]=%3;;\n"
  351.         "mov psr.l=r8;;\n"
  352.         "srlz.d;;\n"
  353.         :
  354.         : "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]), "r" (entry.word[0]), "r" (tr), "r" (dtr)
  355.         : "p6", "p7", "r8"
  356.     );
  357.    
  358.     if (restore_rr) {
  359.         rr_write(VA2VRN(va), rr.word);
  360.         srlz_d();
  361.         srlz_i();
  362.     }
  363. }
  364.  
  365. /** Insert data into DTLB.
  366.  *
  367.  * @param va Virtual page address.
  368.  * @param asid Address space identifier.
  369.  * @param entry The rest of TLB entry as required by TLB insertion format.
  370.  * @param dtr If true, insert into data translation register, use data translation cache otherwise.
  371.  * @param tr Translation register if dtr is true, ignored otherwise.
  372.  */
  373. void dtlb_kernel_mapping_insert(__address page, __address frame, bool dtr, index_t tr)
  374. {
  375.     tlb_entry_t entry;
  376.    
  377.     entry.word[0] = 0;
  378.     entry.word[1] = 0;
  379.    
  380.     entry.p = true;         /* present */
  381.     entry.ma = MA_WRITEBACK;
  382.     entry.a = true;         /* already accessed */
  383.     entry.d = true;         /* already dirty */
  384.     entry.pl = PL_KERNEL;
  385.     entry.ar = AR_READ | AR_WRITE;
  386.     entry.ppn = frame >> PPN_SHIFT;
  387.     entry.ps = PAGE_WIDTH;
  388.    
  389.     if (dtr)
  390.         dtr_mapping_insert(page, ASID_KERNEL, entry, tr);
  391.     else
  392.         dtc_mapping_insert(page, ASID_KERNEL, entry);
  393. }
  394.  
  395. /** Copy content of PTE into data translation cache.
  396.  *
  397.  * @param t PTE.
  398.  */
  399. void dtc_pte_copy(pte_t *t)
  400. {
  401.     tlb_entry_t entry;
  402.  
  403.     entry.word[0] = 0;
  404.     entry.word[1] = 0;
  405.    
  406.     entry.p = t->p;
  407.     entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE;
  408.     entry.a = t->a;
  409.     entry.d = t->d;
  410.     entry.pl = t->k ? PL_KERNEL : PL_USER;
  411.     entry.ar = t->w ? AR_WRITE : AR_READ;
  412.     entry.ppn = t->frame >> PPN_SHIFT;
  413.     entry.ps = PAGE_WIDTH;
  414.    
  415.     dtc_mapping_insert(t->page, t->as->asid, entry);
  416. }
  417.  
  418. /** Copy content of PTE into instruction translation cache.
  419.  *
  420.  * @param t PTE.
  421.  */
  422. void itc_pte_copy(pte_t *t)
  423. {
  424.     tlb_entry_t entry;
  425.  
  426.     entry.word[0] = 0;
  427.     entry.word[1] = 0;
  428.    
  429.     ASSERT(t->x);
  430.    
  431.     entry.p = t->p;
  432.     entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE;
  433.     entry.a = t->a;
  434.     entry.pl = t->k ? PL_KERNEL : PL_USER;
  435.     entry.ar = t->x ? (AR_EXECUTE | AR_READ) : AR_READ;
  436.     entry.ppn = t->frame >> PPN_SHIFT;
  437.     entry.ps = PAGE_WIDTH;
  438.    
  439.     itc_mapping_insert(t->page, t->as->asid, entry);
  440. }
  441.  
  442. /** Instruction TLB fault handler for faults with VHPT turned off.
  443.  *
  444.  * @param vector Interruption vector.
  445.  * @param pstate Structure with saved interruption state.
  446.  */
  447. void alternate_instruction_tlb_fault(__u64 vector, struct exception_regdump *pstate)
  448. {
  449.     region_register rr;
  450.     __address va;
  451.     pte_t *t;
  452.    
  453.     va = pstate->cr_ifa;    /* faulting address */
  454.     t = page_mapping_find(AS, va);
  455.     if (t) {
  456.         /*
  457.          * The mapping was found in software page hash table.
  458.          * Insert it into data translation cache.
  459.          */
  460.         itc_pte_copy(t);
  461.     } else {
  462.         /*
  463.          * Forward the page fault to address space page fault handler.
  464.          */
  465.         if (!as_page_fault(va)) {
  466.             panic("%s: va=%P, rid=%d\n", __FUNCTION__, pstate->cr_ifa, rr.map.rid);
  467.         }
  468.     }
  469. }
  470.  
  471. /** Data TLB fault handler for faults with VHPT turned off.
  472.  *
  473.  * @param vector Interruption vector.
  474.  * @param pstate Structure with saved interruption state.
  475.  */
  476. void alternate_data_tlb_fault(__u64 vector, struct exception_regdump *pstate)
  477. {
  478.     region_register rr;
  479.     rid_t rid;
  480.     __address va;
  481.     pte_t *t;
  482.    
  483.     va = pstate->cr_ifa;    /* faulting address */
  484.     rr.word = rr_read(VA2VRN(va));
  485.     rid = rr.map.rid;
  486.     if (RID2ASID(rid) == ASID_KERNEL) {
  487.         if (VA2VRN(va) == VRN_KERNEL) {
  488.             /*
  489.              * Provide KA2PA(identity) mapping for faulting piece of
  490.              * kernel address space.
  491.              */
  492.             dtlb_kernel_mapping_insert(va, KA2PA(va), false, 0);
  493.             return;
  494.         }
  495.     }
  496.  
  497.     t = page_mapping_find(AS, va);
  498.     if (t) {
  499.         /*
  500.          * The mapping was found in software page hash table.
  501.          * Insert it into data translation cache.
  502.          */
  503.         dtc_pte_copy(t);
  504.     } else {
  505.         /*
  506.          * Forward the page fault to address space page fault handler.
  507.          */
  508.         if (!as_page_fault(va)) {
  509.             panic("%s: va=%P, rid=%d\n", __FUNCTION__, pstate->cr_ifa, rr.map.rid);
  510.         }
  511.     }
  512. }
  513.  
  514. /** Data nested TLB fault handler.
  515.  *
  516.  * This fault should not occur.
  517.  *
  518.  * @param vector Interruption vector.
  519.  * @param pstate Structure with saved interruption state.
  520.  */
  521. void data_nested_tlb_fault(__u64 vector, struct exception_regdump *pstate)
  522. {
  523.     panic("%s\n", __FUNCTION__);
  524. }
  525.  
  526. /** Data Dirty bit fault handler.
  527.  *
  528.  * @param vector Interruption vector.
  529.  * @param pstate Structure with saved interruption state.
  530.  */
  531. void data_dirty_bit_fault(__u64 vector, struct exception_regdump *pstate)
  532. {
  533.     pte_t *t;
  534.  
  535.     t = page_mapping_find(AS, pstate->cr_ifa);
  536.     ASSERT(t && t->p);
  537.     if (t && t->p) {
  538.         /*
  539.          * Update the Dirty bit in page tables and reinsert
  540.          * the mapping into DTC.
  541.          */
  542.         t->d = true;
  543.         dtc_pte_copy(t);
  544.     }
  545. }
  546.  
  547. /** Instruction access bit fault handler.
  548.  *
  549.  * @param vector Interruption vector.
  550.  * @param pstate Structure with saved interruption state.
  551.  */
  552. void instruction_access_bit_fault(__u64 vector, struct exception_regdump *pstate)
  553. {
  554.     pte_t *t;
  555.  
  556.     t = page_mapping_find(AS, pstate->cr_ifa);
  557.     ASSERT(t && t->p);
  558.     if (t && t->p) {
  559.         /*
  560.          * Update the Accessed bit in page tables and reinsert
  561.          * the mapping into ITC.
  562.          */
  563.         t->a = true;
  564.         itc_pte_copy(t);
  565.     }
  566. }
  567.  
  568. /** Data access bit fault handler.
  569.  *
  570.  * @param vector Interruption vector.
  571.  * @param pstate Structure with saved interruption state.
  572.  */
  573. void data_access_bit_fault(__u64 vector, struct exception_regdump *pstate)
  574. {
  575.     pte_t *t;
  576.  
  577.     t = page_mapping_find(AS, pstate->cr_ifa);
  578.     ASSERT(t && t->p);
  579.     if (t && t->p) {
  580.         /*
  581.          * Update the Accessed bit in page tables and reinsert
  582.          * the mapping into DTC.
  583.          */
  584.         t->a = true;
  585.         dtc_pte_copy(t);
  586.     }
  587. }
  588.  
  589. /** Page not present fault handler.
  590.  *
  591.  * @param vector Interruption vector.
  592.  * @param pstate Structure with saved interruption state.
  593.  */
  594. void page_not_present(__u64 vector, struct exception_regdump *pstate)
  595. {
  596.     region_register rr;
  597.     __address va;
  598.     pte_t *t;
  599.    
  600.     va = pstate->cr_ifa;    /* faulting address */
  601.     t = page_mapping_find(AS, va);
  602.     ASSERT(t);
  603.    
  604.     if (t->p) {
  605.         /*
  606.          * If the Present bit is set in page hash table, just copy it
  607.          * and update ITC/DTC.
  608.          */
  609.         if (t->x)
  610.             itc_pte_copy(t);
  611.         else
  612.             dtc_pte_copy(t);
  613.     } else {
  614.         if (!as_page_fault(va)) {
  615.             panic("%s: va=%P, rid=%d\n", __FUNCTION__, pstate->cr_ifa, rr.map.rid);
  616.         }
  617.     }
  618. }
  619.